View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to you under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.hadoop.hbase.quotas;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertFalse;
21  import static org.junit.Assert.assertNull;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Map.Entry;
30  import java.util.concurrent.atomic.AtomicLong;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.DoNotRetryIOException;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.Waiter;
43  import org.apache.hadoop.hbase.client.Admin;
44  import org.apache.hadoop.hbase.client.Append;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.Delete;
47  import org.apache.hadoop.hbase.client.Increment;
48  import org.apache.hadoop.hbase.client.Mutation;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.client.RegionServerCallable;
51  import org.apache.hadoop.hbase.client.Result;
52  import org.apache.hadoop.hbase.client.ResultScanner;
53  import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
54  import org.apache.hadoop.hbase.client.Scan;
55  import org.apache.hadoop.hbase.client.Table;
56  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
57  import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
58  import org.apache.hadoop.hbase.master.HMaster;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.quotas.policies.DefaultViolationPolicyEnforcement;
61  import org.apache.hadoop.hbase.regionserver.HRegionServer;
62  import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
63  import org.apache.hadoop.hbase.security.AccessDeniedException;
64  import org.apache.hadoop.hbase.testclassification.LargeTests;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.apache.hadoop.hbase.util.Pair;
67  import org.apache.hadoop.util.StringUtils;
68  import org.junit.AfterClass;
69  import org.junit.Before;
70  import org.junit.BeforeClass;
71  import org.junit.Rule;
72  import org.junit.Test;
73  import org.junit.experimental.categories.Category;
74  import org.junit.rules.TestName;
75  
76  /**
77   * End-to-end test class for filesystem space quotas.
78   */
79  @Category(LargeTests.class)
80  public class TestSpaceQuotas {
81    private static final Log LOG = LogFactory.getLog(TestSpaceQuotas.class);
82    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
83    private static final AtomicLong COUNTER = new AtomicLong(0);
84    private static final int NUM_RETRIES = 10;
85  
86    @Rule
87    public TestName testName = new TestName();
88    private SpaceQuotaHelperForTests helper;
89  
90    @BeforeClass
91    public static void setUp() throws Exception {
92      Configuration conf = TEST_UTIL.getConfiguration();
93      // Hack to work around HBASE-17534
94      conf.setInt("hbase.client.retries.number", 5);
95      conf.set(
96          CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
97          "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
98      // Increase the frequency of some of the chores for responsiveness of the test
99      conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
100     conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
101     conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
102     conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
103     conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_DELAY_KEY, 1000);
104     conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_PERIOD_KEY, 1000);
105     conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
106     TEST_UTIL.startMiniCluster(1);
107   }
108 
109   @AfterClass
110   public static void tearDown() throws Exception {
111     TEST_UTIL.shutdownMiniCluster();
112   }
113 
114   @Before
115   public void removeAllQuotas() throws Exception {
116     final Connection conn = TEST_UTIL.getConnection();
117     if (helper == null) {
118       helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
119     }
120     // Wait for the quota table to be created
121     if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
122       helper.waitForQuotaTable(conn);
123     } else {
124       // Or, clean up any quotas from previous test runs.
125       helper.removeAllQuotas(conn);
126       assertEquals(0, helper.listNumDefinedQuotas(conn));
127     }
128   }
129 
130   @Test
131   public void testNoInsertsWithPut() throws Exception {
132     Put p = new Put(Bytes.toBytes("to_reject"));
133     p.addColumn(
134         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
135     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
136   }
137 
138   @Test
139   public void testNoInsertsWithAppend() throws Exception {
140     Append a = new Append(Bytes.toBytes("to_reject"));
141     a.add(
142         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
143     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
144   }
145 
146   @Test
147   public void testNoInsertsWithIncrement() throws Exception {
148     Increment i = new Increment(Bytes.toBytes("to_reject"));
149     i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
150     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
151   }
152 
153   @Test
154   public void testDeletesAfterNoInserts() throws Exception {
155     final TableName tn = writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
156     // Try a couple of times to verify that the quota never gets enforced, same as we
157     // do when we're trying to catch the failure.
158     Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
159     for (int i = 0; i < NUM_RETRIES; i++) {
160       try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
161         t.delete(d);
162       }
163     }
164   }
165 
166   @Test
167   public void testNoWritesWithPut() throws Exception {
168     Put p = new Put(Bytes.toBytes("to_reject"));
169     p.addColumn(
170         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
171     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
172   }
173 
174   @Test
175   public void testNoWritesWithAppend() throws Exception {
176     Append a = new Append(Bytes.toBytes("to_reject"));
177     a.add(
178         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
179     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
180   }
181 
182   @Test
183   public void testNoWritesWithIncrement() throws Exception {
184     Increment i = new Increment(Bytes.toBytes("to_reject"));
185     i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
186     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
187   }
188 
189   @Test
190   public void testNoWritesWithDelete() throws Exception {
191     Delete d = new Delete(Bytes.toBytes("to_reject"));
192     writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
193   }
194 
195   @Test
196   public void testNoCompactions() throws Exception {
197     Put p = new Put(Bytes.toBytes("to_reject"));
198     p.addColumn(
199         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
200     final TableName tn = writeUntilViolationAndVerifyViolation(
201         SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
202     // We know the policy is active at this point
203 
204     // Major compactions should be rejected
205     try {
206       TEST_UTIL.getHBaseAdmin().majorCompact(tn);
207       fail("Expected that invoking the compaction should throw an Exception");
208     } catch (DoNotRetryIOException e) {
209       // Expected!
210     }
211     // Minor compactions should also be rejected.
212     try {
213       TEST_UTIL.getHBaseAdmin().compact(tn);
214       fail("Expected that invoking the compaction should throw an Exception");
215     } catch (DoNotRetryIOException e) {
216       // Expected!
217     }
218   }
219 
220   @Test
221   public void testNoEnableAfterDisablePolicy() throws Exception {
222     Put p = new Put(Bytes.toBytes("to_reject"));
223     p.addColumn(
224         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
225     final TableName tn = writeUntilViolation(SpaceViolationPolicy.DISABLE);
226     final Admin admin = TEST_UTIL.getHBaseAdmin();
227     // Disabling a table relies on some external action (over the other policies), so wait a bit
228     // more than the other tests.
229     for (int i = 0; i < NUM_RETRIES * 2; i++) {
230       if (admin.isTableEnabled(tn)) {
231         LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
232         Thread.sleep(2000);
233       }
234     }
235     assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
236     try {
237       admin.enableTable(tn);
238     } catch (AccessDeniedException e) {
239       String exceptionContents = StringUtils.stringifyException(e);
240       final String expectedText = "violated space quota";
241       assertTrue("Expected the exception to contain " + expectedText + ", but was: "
242           + exceptionContents, exceptionContents.contains(expectedText));
243     }
244   }
245 
246   @Test(timeout=120000)
247   public void testNoBulkLoadsWithNoWrites() throws Exception {
248     Put p = new Put(Bytes.toBytes("to_reject"));
249     p.addColumn(
250         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
251     TableName tableName = writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
252 
253     // The table is now in violation. Try to do a bulk load
254     FileSystem fs = TEST_UTIL.getTestFileSystem();
255     Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
256     fs.mkdirs(baseDir);
257     RegionServerCallable<byte[]> callable = generateFileToLoad(tableName, 1, 50, baseDir);
258     try {
259       RpcRetryingCallerFactory.instantiate(TEST_UTIL.getConfiguration(), null).<byte[]> newCaller()
260           .callWithRetries(callable, Integer.MAX_VALUE);
261       fail("Expected the bulk load call to fail!");
262     } catch (SpaceLimitingException e) {
263       // Pass
264       LOG.trace("Caught expected exception", e);
265     }
266   }
267 
268   @Test(timeout=120000)
269   public void testAtomicBulkLoadUnderQuota() throws Exception {
270     // Need to verify that if the batch of hfiles cannot be loaded, none are loaded.
271     TableName tn = helper.createTableWithRegions(10);
272 
273     final long sizeLimit = 50L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
274     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
275     TEST_UTIL.getHBaseAdmin().setQuota(settings);
276 
277     HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
278     RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
279     Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
280     Map<HRegionInfo,Long> regionSizes = getReportedSizesForTable(tn);
281     while (true) {
282       SpaceQuotaSnapshot snapshot = snapshots.get(tn);
283       if (snapshot != null && snapshot.getLimit() > 0) {
284         break;
285       }
286       LOG.debug("Snapshot does not yet realize quota limit: " + snapshots + ", regionsizes: " + regionSizes);
287       Thread.sleep(3000);
288       snapshots = spaceQuotaManager.copyQuotaSnapshots();
289       regionSizes = getReportedSizesForTable(tn);
290     }
291     // Our quota limit should be reflected in the latest snapshot
292     SpaceQuotaSnapshot snapshot = snapshots.get(tn);
293     assertEquals(0L, snapshot.getUsage());
294     assertEquals(sizeLimit, snapshot.getLimit());
295 
296     // We would also not have a "real" policy in violation
297     ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements();
298     SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn);
299     assertTrue(
300         "Expected to find Noop policy, but got " + enforcement.getClass().getSimpleName(),
301         enforcement instanceof DefaultViolationPolicyEnforcement);
302 
303     // Should generate two files, each of which is over 25KB each
304     FileSystem fs = TEST_UTIL.getTestFileSystem();
305     Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
306     fs.mkdirs(baseDir);
307     RegionServerCallable<byte[]> callable = generateFileToLoad(tn, 2, 500, baseDir);
308     FileStatus[] files = fs.listStatus(baseDir);
309     for (FileStatus file : files) {
310       assertTrue(
311           "Expected the file, " + file.getPath() + ",  length to be larger than 25KB, but was " 
312               + file.getLen(),
313           file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
314       LOG.debug(file.getPath() + " -> " + file.getLen() +"B");
315     }
316 
317     try {
318       RpcRetryingCallerFactory.instantiate(TEST_UTIL.getConfiguration(), null).<byte[]> newCaller()
319           .callWithRetries(callable, Integer.MAX_VALUE);
320       fail("Expected the bulk load call to fail!");
321     } catch (SpaceLimitingException e) {
322       // Pass
323       LOG.trace("Caught expected exception", e);
324     }
325     // Verify that we have no data in the table because neither file should have been
326     // loaded even though one of the files could have.
327     Table table = TEST_UTIL.getConnection().getTable(tn);
328     ResultScanner scanner = table.getScanner(new Scan());
329     try {
330       assertNull("Expected no results", scanner.next());
331     } finally{
332       scanner.close();
333     }
334   }
335 
336   @Test(timeout=120000)
337   public void testTableQuotaOverridesNamespaceQuota() throws Exception {
338     final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
339     final TableName tn = helper.createTableWithRegions(10);
340 
341     // 2MB limit on the table, 1GB limit on the namespace
342     final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
343     final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
344     TEST_UTIL.getHBaseAdmin().setQuota(
345         QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
346     TEST_UTIL.getHBaseAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace(
347         tn.getNamespaceAsString(), namespaceLimit, policy));
348 
349     // Write more data than should be allowed and flush it to disk
350     helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
351 
352     // This should be sufficient time for the chores to run and see the change.
353     Thread.sleep(5000);
354 
355     // The write should be rejected because the table quota takes priority over the namespace
356     Put p = new Put(Bytes.toBytes("to_reject"));
357     p.addColumn(
358         Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
359     verifyViolation(policy, tn, p);
360   }
361 
362   @Test
363   public void testSecureBulkLoads() throws Exception {
364     final TableName tn = helper.createTableWithRegions(10);
365 
366     // Set a very small limit
367     final long sizeLimit = 1L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
368     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
369         tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
370     TEST_UTIL.getHBaseAdmin().setQuota(settings);
371 
372     // Wait for the RS to acknowledge this small limit
373     HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
374     final RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
375     TEST_UTIL.waitFor(60000, 3000, new Waiter.Predicate<Exception>() {
376       @Override
377       public boolean evaluate() throws Exception {
378         Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
379         SpaceQuotaSnapshot snapshot = snapshots.get(tn);
380         LOG.debug("Snapshots: " + snapshots);
381         return null != snapshot && snapshot.getLimit() > 0;
382       }
383     });
384     // Our quota limit should be reflected in the latest snapshot
385     Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
386     SpaceQuotaSnapshot snapshot = snapshots.get(tn);
387     assertEquals(0L, snapshot.getUsage());
388     assertEquals(sizeLimit, snapshot.getLimit());
389 
390     // Generate a file that is ~25KB
391     FileSystem fs = TEST_UTIL.getTestFileSystem();
392     Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
393     fs.mkdirs(baseDir);
394     Path hfilesDir = new Path(baseDir, SpaceQuotaHelperForTests.F1);
395     fs.mkdirs(hfilesDir);
396     List<Pair<byte[], String>> filesToLoad = createFiles(tn, 1, 500, hfilesDir);
397     // Verify that they are the size we expecte
398     for (Pair<byte[], String> pair : filesToLoad) {
399       String file = pair.getSecond();
400       FileStatus[] statuses = fs.listStatus(new Path(file));
401       assertEquals(1, statuses.length);
402       FileStatus status = statuses[0];
403       assertTrue(
404           "Expected the file, " + file + ",  length to be larger than 25KB, but was " 
405               + status.getLen(),
406               status.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
407       LOG.debug(file + " -> " + status.getLen() +"B");
408     }
409 
410     // Use LoadIncrementalHFiles to load the file which should be rejected since
411     // it would violate the quota.
412     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(TEST_UTIL.getConfiguration());
413     try {
414       loader.run(new String[] {new Path(
415           fs.getHomeDirectory(), testName.getMethodName() + "_files").toString(), tn.toString()});
416       fail("Expected the bulk load to be rejected, but it was not");
417     } catch (Exception e) {
418       LOG.debug("Caught expected exception", e);
419       String stringifiedException = StringUtils.stringifyException(e);
420       assertTrue(
421           "Expected exception message to contain the SpaceLimitingException class name: "
422               + stringifiedException,
423           stringifiedException.contains(SpaceLimitingException.class.getName()));
424     }
425   }
426 
427   private Map<HRegionInfo,Long> getReportedSizesForTable(TableName tn) {
428     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
429     MasterQuotaManager quotaManager = master.getMasterQuotaManager();
430     Map<HRegionInfo,Long> filteredRegionSizes = new HashMap<>();
431     for (Entry<HRegionInfo,Long> entry : quotaManager.snapshotRegionSizes().entrySet()) {
432       if (entry.getKey().getTable().equals(tn)) {
433         filteredRegionSizes.put(entry.getKey(), entry.getValue());
434       }
435     }
436     return filteredRegionSizes;
437   }
438 
439   private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception {
440     TableName tn = helper.createTableWithRegions(10);
441 
442     final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
443     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policyToViolate);
444     TEST_UTIL.getHBaseAdmin().setQuota(settings);
445 
446     // Write more data than should be allowed and flush it to disk
447     helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
448 
449     // This should be sufficient time for the chores to run and see the change.
450     Thread.sleep(5000);
451 
452     return tn;
453   }
454 
455   private TableName writeUntilViolationAndVerifyViolation(SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
456     final TableName tn = writeUntilViolation(policyToViolate);
457     verifyViolation(policyToViolate, tn, m);
458     return tn;
459   }
460 
461   private void verifyViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m) throws Exception {
462     // But let's try a few times to get the exception before failing
463     boolean sawError = false;
464     for (int i = 0; i < NUM_RETRIES && !sawError; i++) {
465       try (Table table = TEST_UTIL.getConnection().getTable(tn)) {
466         if (m instanceof Put) {
467           table.put((Put) m);
468         } else if (m instanceof Delete) {
469           table.delete((Delete) m);
470         } else if (m instanceof Append) {
471           table.append((Append) m);
472         } else if (m instanceof Increment) {
473           table.increment((Increment) m);
474         } else {
475           fail("Failed to apply " + m.getClass().getSimpleName() + " to the table. Programming error");
476         }
477         LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry");
478         Thread.sleep(2000);
479       } catch (Exception e) {
480         String msg = StringUtils.stringifyException(e);
481         assertTrue("Expected exception message to contain the word '" + policyToViolate.name() + "', but was " + msg,
482             msg.contains(policyToViolate.name()));
483         sawError = true;
484       }
485     }
486     if (!sawError) {
487       try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
488         ResultScanner scanner = quotaTable.getScanner(new Scan());
489         Result result = null;
490         LOG.info("Dumping contents of hbase:quota table");
491         while ((result = scanner.next()) != null) {
492           LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
493         }
494         scanner.close();
495       }
496     }
497     assertTrue("Expected to see an exception writing data to a table exceeding its quota", sawError);
498   }
499 
500   private List<Pair<byte[], String>> createFiles(
501       TableName tn, int numFiles, int numRowsPerFile, Path baseDir) throws Exception {
502     FileSystem fs = TEST_UTIL.getTestFileSystem();
503     fs.mkdirs(baseDir);
504     final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>();
505     for (int i = 1; i <= numFiles; i++) {
506       Path hfile = new Path(baseDir, "file" + i);
507       TestHRegionServerBulkLoad.createHFile(
508           fs, hfile, Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
509           Bytes.toBytes("reject"), numRowsPerFile);
510       famPaths.add(new Pair<>(Bytes.toBytes(SpaceQuotaHelperForTests.F1), hfile.toString()));
511     }
512     return famPaths;
513   }
514 
515   private RegionServerCallable<byte[]> generateFileToLoad(
516       TableName tn, int numFiles, int numRowsPerFile, Path baseDir) throws Exception {
517     final Connection conn = TEST_UTIL.getConnection();
518     final List<Pair<byte[], String>> famPaths = createFiles(tn, numFiles, numRowsPerFile, baseDir);
519     // bulk load HFiles
520     return new RegionServerCallable<byte[]>(conn, tn, Bytes.toBytes("row")) {
521       @Override
522       public byte[] call(int callTimeout) throws Exception {
523         byte[] regionName = getLocation().getRegionInfo().getRegionName();
524         boolean success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, true);
525         return success ? regionName : null;
526       }
527     };
528   }
529 }