1
2
3
4
5
6
7
8
9
10
11
12 package org.apache.hadoop.hbase.quotas;
13
14 import static org.junit.Assert.assertEquals;
15
16 import java.util.concurrent.TimeUnit;
17
18 import org.apache.commons.logging.Log;
19 import org.apache.commons.logging.LogFactory;
20 import org.apache.hadoop.hbase.HBaseTestingUtility;
21 import org.apache.hadoop.hbase.HConstants;
22 import org.apache.hadoop.hbase.TableName;
23 import org.apache.hadoop.hbase.client.Admin;
24 import org.apache.hadoop.hbase.client.Get;
25 import org.apache.hadoop.hbase.client.HTable;
26 import org.apache.hadoop.hbase.client.Put;
27 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
28 import org.apache.hadoop.hbase.security.User;
29 import org.apache.hadoop.hbase.testclassification.MediumTests;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
32 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
33 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
34 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
35 import org.junit.After;
36 import org.junit.AfterClass;
37 import org.junit.BeforeClass;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40
41 @Category({ MediumTests.class })
42 public class TestQuotaThrottle {
43 final Log LOG = LogFactory.getLog(getClass());
44
45 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
46
47 private final static byte[] FAMILY = Bytes.toBytes("cf");
48 private final static byte[] QUALIFIER = Bytes.toBytes("q");
49
50 private final static TableName[] TABLE_NAMES = new TableName[] {
51 TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
52 TableName.valueOf("TestQuotaAdmin2") };
53
54 private static HTable[] tables;
55
56 @BeforeClass
57 public static void setUpBeforeClass() throws Exception {
58 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
59 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
60 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
61 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
62 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
63 TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
64 TEST_UTIL.startMiniCluster(1);
65 TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
66 QuotaCache.setTEST_FORCE_REFRESH(true);
67
68 tables = new HTable[TABLE_NAMES.length];
69 for (int i = 0; i < TABLE_NAMES.length; ++i) {
70 tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
71 }
72 }
73
74 @AfterClass
75 public static void tearDownAfterClass() throws Exception {
76 for (int i = 0; i < tables.length; ++i) {
77 if (tables[i] != null) {
78 tables[i].close();
79 TEST_UTIL.deleteTable(TABLE_NAMES[i]);
80 }
81 }
82
83 TEST_UTIL.shutdownMiniCluster();
84 }
85
86 @After
87 public void tearDown() throws Exception {
88 for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
89 RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
90 QuotaCache quotaCache = quotaManager.getQuotaCache();
91 quotaCache.getNamespaceQuotaCache().clear();
92 quotaCache.getTableQuotaCache().clear();
93 quotaCache.getUserQuotaCache().clear();
94 }
95 }
96
97 @Test(timeout = 60000)
98 public void testUserGlobalThrottle() throws Exception {
99 final Admin admin = TEST_UTIL.getHBaseAdmin();
100 final String userName = User.getCurrent().getShortName();
101
102
103 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
104 TimeUnit.MINUTES));
105 triggerUserCacheRefresh(false, TABLE_NAMES);
106
107
108 assertEquals(6, doPuts(100, tables));
109
110
111 waitMinuteQuota();
112 assertEquals(6, doPuts(100, tables));
113
114
115 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
116 triggerUserCacheRefresh(true, TABLE_NAMES);
117 assertEquals(60, doPuts(60, tables));
118 assertEquals(60, doGets(60, tables));
119 }
120
121 @Test(timeout=60000)
122 public void testUserGlobalReadAndWriteThrottle() throws Exception {
123 final Admin admin = TEST_UTIL.getHBaseAdmin();
124 final String userName = User.getCurrent().getShortName();
125
126
127 admin.setQuota(QuotaSettingsFactory
128 .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
129 triggerUserCacheRefresh(false, TABLE_NAMES);
130
131
132 assertEquals(60, doPuts(60, tables));
133 assertEquals(6, doGets(100, tables));
134
135 waitMinuteQuota();
136
137
138 admin.setQuota(QuotaSettingsFactory
139 .throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
140 triggerUserCacheRefresh(false, TABLE_NAMES);
141
142
143 assertEquals(6, doGets(100, tables));
144 assertEquals(6, doPuts(60, tables));
145
146
147 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
148 triggerUserCacheRefresh(true, TABLE_NAMES);
149 assertEquals(60, doPuts(60, tables));
150 assertEquals(60, doGets(60, tables));
151 }
152
153 @Test(timeout = 60000)
154 public void testUserTableThrottle() throws Exception {
155 final Admin admin = TEST_UTIL.getHBaseAdmin();
156 final String userName = User.getCurrent().getShortName();
157
158
159 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
160 ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
161 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
162
163
164 assertEquals(6, doPuts(100, tables[0]));
165 assertEquals(30, doPuts(30, tables[1]));
166
167
168 waitMinuteQuota();
169 assertEquals(6, doPuts(100, tables[0]));
170
171
172 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
173 triggerUserCacheRefresh(true, TABLE_NAMES);
174 assertEquals(60, doPuts(60, tables));
175 assertEquals(60, doGets(60, tables));
176 }
177
178 @Test(timeout=60000)
179 public void testUserTableReadAndWriteThrottle() throws Exception {
180 final Admin admin = TEST_UTIL.getHBaseAdmin();
181 final String userName = User.getCurrent().getShortName();
182
183
184 admin.setQuota(QuotaSettingsFactory
185 .throttleUser(userName, TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
186 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
187
188
189 assertEquals(6, doPuts(100, tables[0]));
190 assertEquals(60, doGets(60, tables[0]));
191
192
193 assertEquals(60, doPuts(60, tables[1]));
194 assertEquals(60, doGets(60, tables[1]));
195
196
197 waitMinuteQuota();
198
199
200 admin.setQuota(QuotaSettingsFactory
201 .throttleUser(userName, TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
202 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
203
204
205 assertEquals(6, doPuts(100, tables[0]));
206 assertEquals(6, doGets(60, tables[0]));
207
208
209 assertEquals(30, doPuts(30, tables[1]));
210 assertEquals(30, doGets(30, tables[1]));
211
212
213 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
214 triggerUserCacheRefresh(true, TABLE_NAMES);
215 assertEquals(60, doPuts(60, tables));
216 assertEquals(60, doGets(60, tables));
217 }
218
219 @Test(timeout = 60000)
220 public void testUserNamespaceThrottle() throws Exception {
221 final Admin admin = TEST_UTIL.getHBaseAdmin();
222 final String userName = User.getCurrent().getShortName();
223 final String NAMESPACE = "default";
224
225
226 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, NAMESPACE,
227 ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
228 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
229
230
231 assertEquals(6, doPuts(100, tables[0]));
232
233
234 waitMinuteQuota();
235 assertEquals(6, doPuts(100, tables[1]));
236
237
238 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
239 triggerUserCacheRefresh(true, TABLE_NAMES);
240 assertEquals(60, doPuts(60, tables));
241 assertEquals(60, doGets(60, tables));
242 }
243
244 @Test(timeout=60000)
245 public void testUserNamespaceReadAndWriteThrottle() throws Exception {
246 final Admin admin = TEST_UTIL.getHBaseAdmin();
247 final String userName = User.getCurrent().getShortName();
248 final String NAMESPACE = "default";
249
250
251 admin.setQuota(QuotaSettingsFactory
252 .throttleUser(userName, NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
253 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
254
255
256 assertEquals(6, doGets(60, tables[0]));
257 assertEquals(60, doPuts(60, tables[0]));
258
259 waitMinuteQuota();
260
261
262 admin.setQuota(QuotaSettingsFactory
263 .throttleUser(userName, NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
264 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
265
266
267 assertEquals(6, doGets(60, tables[0]));
268 assertEquals(6, doPuts(60, tables[0]));
269
270
271 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
272 triggerUserCacheRefresh(true, TABLE_NAMES);
273 assertEquals(60, doPuts(60, tables));
274 assertEquals(60, doGets(60, tables));
275 }
276
277 @Test(timeout = 60000)
278 public void testTableGlobalThrottle() throws Exception {
279 final Admin admin = TEST_UTIL.getHBaseAdmin();
280
281
282 admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
283 6, TimeUnit.MINUTES));
284 triggerTableCacheRefresh(false, TABLE_NAMES[0]);
285
286
287 assertEquals(6, doPuts(100, tables[0]));
288
289 assertEquals(30, doPuts(30, tables[1]));
290
291
292 waitMinuteQuota();
293 assertEquals(6, doPuts(100, tables[0]));
294
295
296 admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
297 triggerTableCacheRefresh(true, TABLE_NAMES[0]);
298 assertEquals(80, doGets(80, tables[0], tables[1]));
299 }
300
301 @Test(timeout=60000)
302 public void testTableGlobalReadAndWriteThrottle() throws Exception {
303 final Admin admin = TEST_UTIL.getHBaseAdmin();
304
305
306 admin.setQuota(QuotaSettingsFactory
307 .throttleTable(TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
308 triggerTableCacheRefresh(false, TABLE_NAMES[0]);
309
310
311 assertEquals(6, doGets(100, tables[0]));
312 assertEquals(100, doPuts(100, tables[0]));
313
314 assertEquals(30, doPuts(30, tables[1]));
315 assertEquals(30, doGets(30, tables[1]));
316
317
318 waitMinuteQuota();
319
320
321 admin.setQuota(QuotaSettingsFactory
322 .throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
323 triggerTableCacheRefresh(false, TABLE_NAMES[0]);
324
325
326 assertEquals(6, doGets(100, tables[0]));
327 assertEquals(6, doPuts(100, tables[0]));
328
329 assertEquals(30, doPuts(30, tables[1]));
330 assertEquals(30, doGets(30, tables[1]));
331
332
333 admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
334 triggerTableCacheRefresh(true, TABLE_NAMES[0]);
335 assertEquals(80, doGets(80, tables[0], tables[1]));
336 }
337
338 @Test(timeout = 60000)
339 public void testNamespaceGlobalThrottle() throws Exception {
340 final Admin admin = TEST_UTIL.getHBaseAdmin();
341 final String NAMESPACE = "default";
342
343
344 admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
345 6, TimeUnit.MINUTES));
346 triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
347
348
349 assertEquals(6, doPuts(100, tables[0]));
350
351
352 waitMinuteQuota();
353 assertEquals(6, doPuts(100, tables[1]));
354
355 admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
356 triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
357 assertEquals(40, doPuts(40, tables[0]));
358 }
359
360 @Test(timeout=60000)
361 public void testNamespaceGlobalReadAndWriteThrottle() throws Exception {
362 final Admin admin = TEST_UTIL.getHBaseAdmin();
363 final String NAMESPACE = "default";
364
365
366 admin.setQuota(QuotaSettingsFactory
367 .throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
368 triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
369
370
371 assertEquals(6, doPuts(100, tables[0]));
372 assertEquals(100, doGets(100, tables[0]));
373
374
375 waitMinuteQuota();
376
377
378 admin.setQuota(QuotaSettingsFactory
379 .throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
380 triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
381
382
383 assertEquals(6, doPuts(100, tables[0]));
384 assertEquals(6, doGets(100, tables[0]));
385
386 admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
387 triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
388 assertEquals(40, doPuts(40, tables[0]));
389 }
390
391 @Test(timeout = 60000)
392 public void testUserAndTableThrottle() throws Exception {
393 final Admin admin = TEST_UTIL.getHBaseAdmin();
394 final String userName = User.getCurrent().getShortName();
395
396
397 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
398 ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
399 triggerUserCacheRefresh(false, TABLE_NAMES[0]);
400
401 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 12,
402 TimeUnit.MINUTES));
403 triggerUserCacheRefresh(false, TABLE_NAMES[1], TABLE_NAMES[2]);
404
405 admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[1], ThrottleType.REQUEST_NUMBER,
406 8, TimeUnit.MINUTES));
407 triggerTableCacheRefresh(false, TABLE_NAMES[1]);
408
409 admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
410 3, TimeUnit.MINUTES));
411 triggerTableCacheRefresh(false, TABLE_NAMES[0]);
412
413
414 assertEquals(12, doGets(100, tables[2]));
415
416
417 waitMinuteQuota();
418 assertEquals(8, doGets(100, tables[1]));
419
420
421 waitMinuteQuota();
422 assertEquals(3, doPuts(100, tables[0]));
423
424
425 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
426 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
427 triggerUserCacheRefresh(true, TABLE_NAMES[0], TABLE_NAMES[1]);
428
429 admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[1]));
430 triggerTableCacheRefresh(true, TABLE_NAMES[1]);
431 waitMinuteQuota();
432 assertEquals(40, doGets(40, tables[1]));
433
434 admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
435 triggerTableCacheRefresh(true, TABLE_NAMES[0]);
436 waitMinuteQuota();
437 assertEquals(40, doGets(40, tables[0]));
438 }
439
440 @Test(timeout = 60000)
441 public void testUserGlobalBypassThrottle() throws Exception {
442 final Admin admin = TEST_UTIL.getHBaseAdmin();
443 final String userName = User.getCurrent().getShortName();
444 final String NAMESPACE = "default";
445
446
447 admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
448 6, TimeUnit.MINUTES));
449 triggerTableCacheRefresh(false, TABLE_NAMES[0]);
450
451 admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
452 13, TimeUnit.MINUTES));
453 triggerNamespaceCacheRefresh(false, TABLE_NAMES[1]);
454
455
456 assertEquals(6, doPuts(100, tables[0]));
457 assertEquals(7, doGets(100, tables[1]));
458 waitMinuteQuota();
459
460
461 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
462 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[2],
463 ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
464 triggerUserCacheRefresh(false, TABLE_NAMES[2]);
465 assertEquals(30, doGets(30, tables[0]));
466 assertEquals(30, doGets(30, tables[1]));
467 waitMinuteQuota();
468
469
470
471 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
472 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[2]));
473 triggerUserCacheRefresh(true, TABLE_NAMES[2]);
474 assertEquals(6, doPuts(100, tables[0]));
475 assertEquals(7, doGets(100, tables[1]));
476
477
478 admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
479 admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
480 waitMinuteQuota();
481 triggerTableCacheRefresh(true, TABLE_NAMES[0]);
482 triggerNamespaceCacheRefresh(true, TABLE_NAMES[1]);
483 assertEquals(30, doGets(30, tables[0]));
484 assertEquals(30, doGets(30, tables[1]));
485 }
486
487 private int doPuts(int maxOps, final HTable... tables) throws Exception {
488 int count = 0;
489 try {
490 while (count < maxOps) {
491 Put put = new Put(Bytes.toBytes("row-" + count));
492 put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes("data-" + count));
493 for (final HTable table : tables) {
494 table.put(put);
495 }
496 count += tables.length;
497 }
498 } catch (RetriesExhaustedWithDetailsException e) {
499 for (Throwable t : e.getCauses()) {
500 if (!(t instanceof ThrottlingException)) {
501 throw e;
502 }
503 }
504 LOG.error("put failed after nRetries=" + count, e);
505 }
506 return count;
507 }
508
509 private long doGets(int maxOps, final HTable... tables) throws Exception {
510 int count = 0;
511 try {
512 while (count < maxOps) {
513 Get get = new Get(Bytes.toBytes("row-" + count));
514 for (final HTable table : tables) {
515 table.get(get);
516 }
517 count += tables.length;
518 }
519 } catch (ThrottlingException e) {
520 LOG.error("get failed after nRetries=" + count, e);
521 }
522 return count;
523 }
524
525 private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception {
526 triggerCacheRefresh(bypass, true, false, false, tables);
527 }
528
529 private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception {
530 triggerCacheRefresh(bypass, false, true, false, tables);
531 }
532
533 private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception {
534 triggerCacheRefresh(bypass, false, false, true, tables);
535 }
536
537 private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter,
538 boolean nsLimiter, final TableName... tables) throws Exception {
539 for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
540 RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
541 QuotaCache quotaCache = quotaManager.getQuotaCache();
542
543 quotaCache.triggerCacheRefresh();
544 Thread.sleep(250);
545
546 for (TableName table : tables) {
547 quotaCache.getTableLimiter(table);
548 }
549
550 boolean isUpdated = false;
551 while (!isUpdated) {
552 isUpdated = true;
553 for (TableName table : tables) {
554 boolean isBypass = true;
555 if (userLimiter) {
556 isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
557 }
558 if (tableLimiter) {
559 isBypass &= quotaCache.getTableLimiter(table).isBypass();
560 }
561 if (nsLimiter) {
562 isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
563 }
564 if (isBypass != bypass) {
565 isUpdated = false;
566 Thread.sleep(250);
567 break;
568 }
569 }
570 }
571
572 LOG.debug("QuotaCache");
573 LOG.debug(quotaCache.getNamespaceQuotaCache());
574 LOG.debug(quotaCache.getTableQuotaCache());
575 LOG.debug(quotaCache.getUserQuotaCache());
576 }
577 }
578
579 private void waitMinuteQuota() {
580 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge(
581 EnvironmentEdgeManager.currentTime() + 70000));
582 }
583 }