View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  
12  package org.apache.hadoop.hbase.quotas;
13  
14  import static org.junit.Assert.assertEquals;
15  
16  import java.util.concurrent.TimeUnit;
17  
18  import org.apache.commons.logging.Log;
19  import org.apache.commons.logging.LogFactory;
20  import org.apache.hadoop.hbase.HBaseTestingUtility;
21  import org.apache.hadoop.hbase.HConstants;
22  import org.apache.hadoop.hbase.TableName;
23  import org.apache.hadoop.hbase.client.Admin;
24  import org.apache.hadoop.hbase.client.Get;
25  import org.apache.hadoop.hbase.client.HTable;
26  import org.apache.hadoop.hbase.client.Put;
27  import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
28  import org.apache.hadoop.hbase.security.User;
29  import org.apache.hadoop.hbase.testclassification.MediumTests;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
32  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
33  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
34  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
35  import org.junit.After;
36  import org.junit.AfterClass;
37  import org.junit.BeforeClass;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  
41  @Category({ MediumTests.class })
42  public class TestQuotaThrottle {
43    final Log LOG = LogFactory.getLog(getClass());
44  
45    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
46  
47    private final static byte[] FAMILY = Bytes.toBytes("cf");
48    private final static byte[] QUALIFIER = Bytes.toBytes("q");
49  
50    private final static TableName[] TABLE_NAMES = new TableName[] {
51        TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
52        TableName.valueOf("TestQuotaAdmin2") };
53  
54    private static HTable[] tables;
55  
56    @BeforeClass
57    public static void setUpBeforeClass() throws Exception {
58      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
59      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
60      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
61      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
62      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
63      TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
64      TEST_UTIL.startMiniCluster(1);
65      TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
66      QuotaCache.setTEST_FORCE_REFRESH(true);
67  
68      tables = new HTable[TABLE_NAMES.length];
69      for (int i = 0; i < TABLE_NAMES.length; ++i) {
70        tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
71      }
72    }
73  
74    @AfterClass
75    public static void tearDownAfterClass() throws Exception {
76      for (int i = 0; i < tables.length; ++i) {
77        if (tables[i] != null) {
78          tables[i].close();
79          TEST_UTIL.deleteTable(TABLE_NAMES[i]);
80        }
81      }
82  
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    @After
87    public void tearDown() throws Exception {
88      for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
89        RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
90        QuotaCache quotaCache = quotaManager.getQuotaCache();
91        quotaCache.getNamespaceQuotaCache().clear();
92        quotaCache.getTableQuotaCache().clear();
93        quotaCache.getUserQuotaCache().clear();
94      }
95    }
96  
97    @Test(timeout = 60000)
98    public void testUserGlobalThrottle() throws Exception {
99      final Admin admin = TEST_UTIL.getHBaseAdmin();
100     final String userName = User.getCurrent().getShortName();
101 
102     // Add 6req/min limit
103     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
104       TimeUnit.MINUTES));
105     triggerUserCacheRefresh(false, TABLE_NAMES);
106 
107     // should execute at max 6 requests
108     assertEquals(6, doPuts(100, tables));
109 
110     // wait a minute and you should get other 6 requests executed
111     waitMinuteQuota();
112     assertEquals(6, doPuts(100, tables));
113 
114     // Remove all the limits
115     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
116     triggerUserCacheRefresh(true, TABLE_NAMES);
117     assertEquals(60, doPuts(60, tables));
118     assertEquals(60, doGets(60, tables));
119   }
120 
121   @Test(timeout=60000)
122   public void testUserGlobalReadAndWriteThrottle() throws Exception {
123     final Admin admin = TEST_UTIL.getHBaseAdmin();
124     final String userName = User.getCurrent().getShortName();
125 
126     // Add 6req/min limit for read request
127     admin.setQuota(QuotaSettingsFactory
128       .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
129     triggerUserCacheRefresh(false, TABLE_NAMES);
130 
131     // not limit for write request and should execute at max 6 read requests
132     assertEquals(60, doPuts(60, tables));
133     assertEquals(6, doGets(100, tables));
134 
135     waitMinuteQuota();
136 
137     // Add 6req/min limit for write request
138     admin.setQuota(QuotaSettingsFactory
139       .throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
140     triggerUserCacheRefresh(false, TABLE_NAMES);
141 
142     // should execute at max 6 read requests and at max 6 write write requests
143     assertEquals(6, doGets(100, tables));
144     assertEquals(6, doPuts(60, tables));
145 
146     // Remove all the limits
147     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
148     triggerUserCacheRefresh(true, TABLE_NAMES);
149     assertEquals(60, doPuts(60, tables));
150     assertEquals(60, doGets(60, tables));
151   }
152 
153   @Test(timeout = 60000)
154   public void testUserTableThrottle() throws Exception {
155     final Admin admin = TEST_UTIL.getHBaseAdmin();
156     final String userName = User.getCurrent().getShortName();
157 
158     // Add 6req/min limit
159     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
160       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
161     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
162 
163     // should execute at max 6 requests on tables[0] and have no limit on tables[1]
164     assertEquals(6, doPuts(100, tables[0]));
165     assertEquals(30, doPuts(30, tables[1]));
166 
167     // wait a minute and you should get other 6 requests executed
168     waitMinuteQuota();
169     assertEquals(6, doPuts(100, tables[0]));
170 
171     // Remove all the limits
172     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
173     triggerUserCacheRefresh(true, TABLE_NAMES);
174     assertEquals(60, doPuts(60, tables));
175     assertEquals(60, doGets(60, tables));
176   }
177 
178   @Test(timeout=60000)
179   public void testUserTableReadAndWriteThrottle() throws Exception {
180     final Admin admin = TEST_UTIL.getHBaseAdmin();
181     final String userName = User.getCurrent().getShortName();
182 
183     // Add 6req/min limit for write request on tables[0]
184     admin.setQuota(QuotaSettingsFactory
185       .throttleUser(userName, TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
186     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
187 
188     // should execute at max 6 write requests and have no limit for read request
189     assertEquals(6, doPuts(100, tables[0]));
190     assertEquals(60, doGets(60, tables[0]));
191 
192     // no limit on tables[1]
193     assertEquals(60, doPuts(60, tables[1]));
194     assertEquals(60, doGets(60, tables[1]));
195 
196     // wait a minute and you should get other 6  write requests executed
197     waitMinuteQuota();
198 
199     // Add 6req/min limit for read request on tables[0]
200     admin.setQuota(QuotaSettingsFactory
201       .throttleUser(userName, TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
202     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
203 
204     // should execute at max 6 read requests and at max 6 write requests
205     assertEquals(6, doPuts(100, tables[0]));
206     assertEquals(6, doGets(60, tables[0]));
207 
208     // no limit on tables[1]
209     assertEquals(30, doPuts(30, tables[1]));
210     assertEquals(30, doGets(30, tables[1]));
211 
212     // Remove all the limits
213     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
214     triggerUserCacheRefresh(true, TABLE_NAMES);
215     assertEquals(60, doPuts(60, tables));
216     assertEquals(60, doGets(60, tables));
217   }
218 
219   @Test(timeout = 60000)
220   public void testUserNamespaceThrottle() throws Exception {
221     final Admin admin = TEST_UTIL.getHBaseAdmin();
222     final String userName = User.getCurrent().getShortName();
223     final String NAMESPACE = "default";
224 
225     // Add 6req/min limit
226     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, NAMESPACE,
227       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
228     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
229 
230     // should execute at max 6 requests on tables[0] and have no limit on tables[1]
231     assertEquals(6, doPuts(100, tables[0]));
232 
233     // wait a minute and you should get other 6 requests executed
234     waitMinuteQuota();
235     assertEquals(6, doPuts(100, tables[1]));
236 
237     // Remove all the limits
238     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
239     triggerUserCacheRefresh(true, TABLE_NAMES);
240     assertEquals(60, doPuts(60, tables));
241     assertEquals(60, doGets(60, tables));
242   }
243 
244   @Test(timeout=60000)
245   public void testUserNamespaceReadAndWriteThrottle() throws Exception {
246     final Admin admin = TEST_UTIL.getHBaseAdmin();
247     final String userName = User.getCurrent().getShortName();
248     final String NAMESPACE = "default";
249 
250     // Add 6req/min limit for read request
251     admin.setQuota(QuotaSettingsFactory
252       .throttleUser(userName, NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
253     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
254 
255     // should execute at max 6 read requests and have no limit for write request
256     assertEquals(6, doGets(60, tables[0]));
257     assertEquals(60, doPuts(60, tables[0]));
258 
259     waitMinuteQuota();
260 
261     // Add 6req/min limit for write request, too
262     admin.setQuota(QuotaSettingsFactory
263       .throttleUser(userName, NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
264     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
265 
266     // should execute at max 6 read requests and at max 6 write requests
267     assertEquals(6, doGets(60, tables[0]));
268     assertEquals(6, doPuts(60, tables[0]));
269 
270     // Remove all the limits
271     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
272     triggerUserCacheRefresh(true, TABLE_NAMES);
273     assertEquals(60, doPuts(60, tables));
274     assertEquals(60, doGets(60, tables));
275   }
276 
277   @Test(timeout = 60000)
278   public void testTableGlobalThrottle() throws Exception {
279     final Admin admin = TEST_UTIL.getHBaseAdmin();
280 
281     // Add 6req/min limit
282     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
283       6, TimeUnit.MINUTES));
284     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
285 
286     // should execute at max 6 requests
287     assertEquals(6, doPuts(100, tables[0]));
288     // should have no limits
289     assertEquals(30, doPuts(30, tables[1]));
290 
291     // wait a minute and you should get other 6 requests executed
292     waitMinuteQuota();
293     assertEquals(6, doPuts(100, tables[0]));
294 
295     // Remove all the limits
296     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
297     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
298     assertEquals(80, doGets(80, tables[0], tables[1]));
299   }
300 
301   @Test(timeout=60000)
302   public void testTableGlobalReadAndWriteThrottle() throws Exception {
303     final Admin admin = TEST_UTIL.getHBaseAdmin();
304 
305     // Add 6req/min limit for read request
306     admin.setQuota(QuotaSettingsFactory
307       .throttleTable(TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
308     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
309 
310     // should execute at max 6 read requests and have no limit for write request
311     assertEquals(6, doGets(100, tables[0]));
312     assertEquals(100, doPuts(100, tables[0]));
313     // should have no limits on tables[1]
314     assertEquals(30, doPuts(30, tables[1]));
315     assertEquals(30, doGets(30, tables[1]));
316 
317     // wait a minute and you should get other 6 requests executed
318     waitMinuteQuota();
319 
320     // Add 6req/min limit for write request, too
321     admin.setQuota(QuotaSettingsFactory
322       .throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
323     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
324 
325     // should execute at max 6 read requests and at max 6 write requests
326     assertEquals(6, doGets(100, tables[0]));
327     assertEquals(6, doPuts(100, tables[0]));
328     // should have no limits on tables[1]
329     assertEquals(30, doPuts(30, tables[1]));
330     assertEquals(30, doGets(30, tables[1]));
331 
332     // Remove all the limits
333     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
334     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
335     assertEquals(80, doGets(80, tables[0], tables[1]));
336   }
337 
338   @Test(timeout = 60000)
339   public void testNamespaceGlobalThrottle() throws Exception {
340     final Admin admin = TEST_UTIL.getHBaseAdmin();
341     final String NAMESPACE = "default";
342 
343     // Add 6req/min limit
344     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
345       6, TimeUnit.MINUTES));
346     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
347 
348     // should execute at max 6 requests
349     assertEquals(6, doPuts(100, tables[0]));
350 
351     // wait a minute and you should get other 6 requests executed
352     waitMinuteQuota();
353     assertEquals(6, doPuts(100, tables[1]));
354 
355     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
356     triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
357     assertEquals(40, doPuts(40, tables[0]));
358   }
359 
360   @Test(timeout=60000)
361   public void testNamespaceGlobalReadAndWriteThrottle() throws Exception {
362     final Admin admin = TEST_UTIL.getHBaseAdmin();
363     final String NAMESPACE = "default";
364 
365     // Add 6req/min limit for write request
366     admin.setQuota(QuotaSettingsFactory
367       .throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
368     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
369 
370     // should execute at max 6 write requests and no limit for read request
371     assertEquals(6, doPuts(100, tables[0]));
372     assertEquals(100, doGets(100, tables[0]));
373 
374     // wait a minute and you should get other 6 requests executed
375     waitMinuteQuota();
376 
377     // Add 6req/min limit for read request, too
378     admin.setQuota(QuotaSettingsFactory
379       .throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
380     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
381 
382     // should execute at max 6 write requests and at max 6 read requests
383     assertEquals(6, doPuts(100, tables[0]));
384     assertEquals(6, doGets(100, tables[0]));
385 
386     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
387     triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
388     assertEquals(40, doPuts(40, tables[0]));
389   }
390 
391   @Test(timeout = 60000)
392   public void testUserAndTableThrottle() throws Exception {
393     final Admin admin = TEST_UTIL.getHBaseAdmin();
394     final String userName = User.getCurrent().getShortName();
395 
396     // Add 6req/min limit for the user on tables[0]
397     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
398       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
399     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
400     // Add 12req/min limit for the user
401     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 12,
402       TimeUnit.MINUTES));
403     triggerUserCacheRefresh(false, TABLE_NAMES[1], TABLE_NAMES[2]);
404     // Add 8req/min limit for the tables[1]
405     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[1], ThrottleType.REQUEST_NUMBER,
406       8, TimeUnit.MINUTES));
407     triggerTableCacheRefresh(false, TABLE_NAMES[1]);
408     // Add a lower table level throttle on tables[0]
409     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
410       3, TimeUnit.MINUTES));
411     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
412 
413     // should execute at max 12 requests
414     assertEquals(12, doGets(100, tables[2]));
415 
416     // should execute at max 8 requests
417     waitMinuteQuota();
418     assertEquals(8, doGets(100, tables[1]));
419 
420     // should execute at max 3 requests
421     waitMinuteQuota();
422     assertEquals(3, doPuts(100, tables[0]));
423 
424     // Remove all the throttling rules
425     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
426     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
427     triggerUserCacheRefresh(true, TABLE_NAMES[0], TABLE_NAMES[1]);
428 
429     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[1]));
430     triggerTableCacheRefresh(true, TABLE_NAMES[1]);
431     waitMinuteQuota();
432     assertEquals(40, doGets(40, tables[1]));
433 
434     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
435     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
436     waitMinuteQuota();
437     assertEquals(40, doGets(40, tables[0]));
438   }
439 
440   @Test(timeout = 60000)
441   public void testUserGlobalBypassThrottle() throws Exception {
442     final Admin admin = TEST_UTIL.getHBaseAdmin();
443     final String userName = User.getCurrent().getShortName();
444     final String NAMESPACE = "default";
445 
446     // Add 6req/min limit for tables[0]
447     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
448       6, TimeUnit.MINUTES));
449     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
450     // Add 13req/min limit for the user
451     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
452       13, TimeUnit.MINUTES));
453     triggerNamespaceCacheRefresh(false, TABLE_NAMES[1]);
454 
455     // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
456     assertEquals(6, doPuts(100, tables[0]));
457     assertEquals(7, doGets(100, tables[1]));
458     waitMinuteQuota();
459 
460     // Set the global bypass for the user
461     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
462     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[2],
463       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
464     triggerUserCacheRefresh(false, TABLE_NAMES[2]);
465     assertEquals(30, doGets(30, tables[0]));
466     assertEquals(30, doGets(30, tables[1]));
467     waitMinuteQuota();
468 
469     // Remove the global bypass
470     // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
471     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
472     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[2]));
473     triggerUserCacheRefresh(true, TABLE_NAMES[2]);
474     assertEquals(6, doPuts(100, tables[0]));
475     assertEquals(7, doGets(100, tables[1]));
476 
477     // unset throttle
478     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
479     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
480     waitMinuteQuota();
481     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
482     triggerNamespaceCacheRefresh(true, TABLE_NAMES[1]);
483     assertEquals(30, doGets(30, tables[0]));
484     assertEquals(30, doGets(30, tables[1]));
485   }
486 
487   private int doPuts(int maxOps, final HTable... tables) throws Exception {
488     int count = 0;
489     try {
490       while (count < maxOps) {
491         Put put = new Put(Bytes.toBytes("row-" + count));
492         put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes("data-" + count));
493         for (final HTable table : tables) {
494           table.put(put);
495         }
496         count += tables.length;
497       }
498     } catch (RetriesExhaustedWithDetailsException e) {
499       for (Throwable t : e.getCauses()) {
500         if (!(t instanceof ThrottlingException)) {
501           throw e;
502         }
503       }
504       LOG.error("put failed after nRetries=" + count, e);
505     }
506     return count;
507   }
508 
509   private long doGets(int maxOps, final HTable... tables) throws Exception {
510     int count = 0;
511     try {
512       while (count < maxOps) {
513         Get get = new Get(Bytes.toBytes("row-" + count));
514         for (final HTable table : tables) {
515           table.get(get);
516         }
517         count += tables.length;
518       }
519     } catch (ThrottlingException e) {
520       LOG.error("get failed after nRetries=" + count, e);
521     }
522     return count;
523   }
524 
525   private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception {
526     triggerCacheRefresh(bypass, true, false, false, tables);
527   }
528 
529   private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception {
530     triggerCacheRefresh(bypass, false, true, false, tables);
531   }
532 
533   private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception {
534     triggerCacheRefresh(bypass, false, false, true, tables);
535   }
536 
537   private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter,
538       boolean nsLimiter, final TableName... tables) throws Exception {
539     for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
540       RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
541       QuotaCache quotaCache = quotaManager.getQuotaCache();
542 
543       quotaCache.triggerCacheRefresh();
544       Thread.sleep(250);
545 
546       for (TableName table : tables) {
547         quotaCache.getTableLimiter(table);
548       }
549 
550       boolean isUpdated = false;
551       while (!isUpdated) {
552         isUpdated = true;
553         for (TableName table : tables) {
554           boolean isBypass = true;
555           if (userLimiter) {
556             isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
557           }
558           if (tableLimiter) {
559             isBypass &= quotaCache.getTableLimiter(table).isBypass();
560           }
561           if (nsLimiter) {
562             isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
563           }
564           if (isBypass != bypass) {
565             isUpdated = false;
566             Thread.sleep(250);
567             break;
568           }
569         }
570       }
571 
572       LOG.debug("QuotaCache");
573       LOG.debug(quotaCache.getNamespaceQuotaCache());
574       LOG.debug(quotaCache.getTableQuotaCache());
575       LOG.debug(quotaCache.getUserQuotaCache());
576     }
577   }
578 
579   private void waitMinuteQuota() {
580     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge(
581         EnvironmentEdgeManager.currentTime() + 70000));
582   }
583 }