1
2
3
4
5
6
7
8
9
10
11
12 package org.apache.hadoop.hbase.quotas;
13
14 import static org.junit.Assert.assertEquals;
15 import static org.junit.Assert.fail;
16
17 import java.util.concurrent.TimeUnit;
18
19 import org.apache.commons.logging.Log;
20 import org.apache.commons.logging.LogFactory;
21 import org.apache.hadoop.hbase.Cell;
22 import org.apache.hadoop.hbase.CellScanner;
23 import org.apache.hadoop.hbase.HBaseTestingUtility;
24 import org.apache.hadoop.hbase.HConstants;
25 import org.apache.hadoop.hbase.TableName;
26 import org.apache.hadoop.hbase.client.Admin;
27 import org.apache.hadoop.hbase.client.Result;
28 import org.apache.hadoop.hbase.client.ResultScanner;
29 import org.apache.hadoop.hbase.client.Scan;
30 import org.apache.hadoop.hbase.client.Table;
31 import org.apache.hadoop.hbase.security.User;
32 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
33 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
34 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
35 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceLimitRequest;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.junit.AfterClass;
38 import org.junit.BeforeClass;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 import com.google.common.collect.Iterables;
43
44 import static org.junit.Assert.assertEquals;
45 import static org.junit.Assert.assertNull;
46 import static org.junit.Assert.assertTrue;
47 import static org.junit.Assert.fail;
48
49
50
51
52 @Category({ MediumTests.class })
53 public class TestQuotaAdmin {
54 final Log LOG = LogFactory.getLog(getClass());
55
56 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57
58 @BeforeClass
59 public static void setUpBeforeClass() throws Exception {
60 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
61 TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
62 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
63 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
64 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
65 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
66 TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
67 TEST_UTIL.startMiniCluster(1);
68 TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
69 }
70
71 @AfterClass
72 public static void tearDownAfterClass() throws Exception {
73 TEST_UTIL.shutdownMiniCluster();
74 }
75
76 @Test
77 public void testThrottleType() throws Exception {
78 Admin admin = TEST_UTIL.getHBaseAdmin();
79 String userName = User.getCurrent().getShortName();
80
81 admin.setQuota(QuotaSettingsFactory
82 .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
83 admin.setQuota(QuotaSettingsFactory
84 .throttleUser(userName, ThrottleType.WRITE_NUMBER, 12, TimeUnit.MINUTES));
85 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
86
87 try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
88 int countThrottle = 0;
89 int countGlobalBypass = 0;
90 for (QuotaSettings settings: scanner) {
91 switch (settings.getQuotaType()) {
92 case THROTTLE:
93 ThrottleSettings throttle = (ThrottleSettings)settings;
94 if (throttle.getSoftLimit() == 6) {
95 assertEquals(ThrottleType.READ_NUMBER, throttle.getThrottleType());
96 } else if (throttle.getSoftLimit() == 12) {
97 assertEquals(ThrottleType.WRITE_NUMBER, throttle.getThrottleType());
98 } else {
99 fail("should not come here, because don't set quota with this limit");
100 }
101 assertEquals(userName, throttle.getUserName());
102 assertEquals(null, throttle.getTableName());
103 assertEquals(null, throttle.getNamespace());
104 assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
105 countThrottle++;
106 break;
107 case GLOBAL_BYPASS:
108 countGlobalBypass++;
109 break;
110 default:
111 fail("unexpected settings type: " + settings.getQuotaType());
112 }
113 }
114 assertEquals(2, countThrottle);
115 assertEquals(1, countGlobalBypass);
116 }
117
118 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
119 assertNumResults(1, null);
120 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
121 assertNumResults(0, null);
122 }
123
124 @Test
125 public void testSimpleScan() throws Exception {
126 Admin admin = TEST_UTIL.getHBaseAdmin();
127 String userName = User.getCurrent().getShortName();
128
129 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
130 TimeUnit.MINUTES));
131 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
132
133 try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
134 int countThrottle = 0;
135 int countGlobalBypass = 0;
136 for (QuotaSettings settings : scanner) {
137 LOG.debug(settings);
138 switch (settings.getQuotaType()) {
139 case THROTTLE:
140 ThrottleSettings throttle = (ThrottleSettings) settings;
141 assertEquals(userName, throttle.getUserName());
142 assertEquals(null, throttle.getTableName());
143 assertEquals(null, throttle.getNamespace());
144 assertEquals(6, throttle.getSoftLimit());
145 assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
146 countThrottle++;
147 break;
148 case GLOBAL_BYPASS:
149 countGlobalBypass++;
150 break;
151 default:
152 fail("unexpected settings type: " + settings.getQuotaType());
153 }
154 }
155 assertEquals(1, countThrottle);
156 assertEquals(1, countGlobalBypass);
157 }
158
159 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
160 assertNumResults(1, null);
161 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
162 assertNumResults(0, null);
163 }
164
165 @Test
166 public void testQuotaRetrieverFilter() throws Exception {
167 Admin admin = TEST_UTIL.getHBaseAdmin();
168 TableName[] tables =
169 new TableName[] { TableName.valueOf("T0"), TableName.valueOf("T01"),
170 TableName.valueOf("NS0:T2"), };
171 String[] namespaces = new String[] { "NS0", "NS01", "NS2" };
172 String[] users = new String[] { "User0", "User01", "User2" };
173
174 for (String user : users) {
175 admin.setQuota(QuotaSettingsFactory.throttleUser(user, ThrottleType.REQUEST_NUMBER, 1,
176 TimeUnit.MINUTES));
177
178 for (TableName table : tables) {
179 admin.setQuota(QuotaSettingsFactory.throttleUser(user, table, ThrottleType.REQUEST_NUMBER,
180 2, TimeUnit.MINUTES));
181 }
182
183 for (String ns : namespaces) {
184 admin.setQuota(QuotaSettingsFactory.throttleUser(user, ns, ThrottleType.REQUEST_NUMBER, 3,
185 TimeUnit.MINUTES));
186 }
187 }
188 assertNumResults(21, null);
189
190 for (TableName table : tables) {
191 admin.setQuota(QuotaSettingsFactory.throttleTable(table, ThrottleType.REQUEST_NUMBER, 4,
192 TimeUnit.MINUTES));
193 }
194 assertNumResults(24, null);
195
196 for (String ns : namespaces) {
197 admin.setQuota(QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_NUMBER, 5,
198 TimeUnit.MINUTES));
199 }
200 assertNumResults(27, null);
201
202 assertNumResults(7, new QuotaFilter().setUserFilter("User0"));
203 assertNumResults(0, new QuotaFilter().setUserFilter("User"));
204 assertNumResults(21, new QuotaFilter().setUserFilter("User.*"));
205 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0"));
206 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("NS.*"));
207 assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setTableFilter("T"));
208 assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T.*"));
209 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS0"));
210 assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS"));
211 assertNumResults(9, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS.*"));
212 assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0")
213 .setNamespaceFilter("NS0"));
214 assertNumResults(1, new QuotaFilter().setTableFilter("T0"));
215 assertNumResults(0, new QuotaFilter().setTableFilter("T"));
216 assertNumResults(2, new QuotaFilter().setTableFilter("T.*"));
217 assertNumResults(3, new QuotaFilter().setTableFilter(".*T.*"));
218 assertNumResults(1, new QuotaFilter().setNamespaceFilter("NS0"));
219 assertNumResults(0, new QuotaFilter().setNamespaceFilter("NS"));
220 assertNumResults(3, new QuotaFilter().setNamespaceFilter("NS.*"));
221
222 for (String user : users) {
223 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user));
224 for (TableName table : tables) {
225 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, table));
226 }
227 for (String ns : namespaces) {
228 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, ns));
229 }
230 }
231 assertNumResults(6, null);
232
233 for (TableName table : tables) {
234 admin.setQuota(QuotaSettingsFactory.unthrottleTable(table));
235 }
236 assertNumResults(3, null);
237
238 for (String ns : namespaces) {
239 admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(ns));
240 }
241 assertNumResults(0, null);
242 }
243
244 @Test
245 public void testSetGetRemoveSpaceQuota() throws Exception {
246 Admin admin = TEST_UTIL.getHBaseAdmin();
247 final TableName tn = TableName.valueOf("table1");
248 final long sizeLimit = 1024L * 1024L * 1024L * 1024L * 5L;
249 final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_WRITES;
250 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy);
251 admin.setQuota(settings);
252
253
254 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
255 ResultScanner scanner = quotaTable.getScanner(new Scan());
256 try {
257 Result r = Iterables.getOnlyElement(scanner);
258 CellScanner cells = r.cellScanner();
259 assertTrue("Expected to find a cell", cells.advance());
260 assertSpaceQuota(sizeLimit, violationPolicy, cells.current());
261 } finally {
262 scanner.close();
263 }
264 }
265
266
267 QuotaRetriever scanner = QuotaRetriever.open(admin.getConfiguration());
268 try {
269 assertSpaceQuota(sizeLimit, violationPolicy, Iterables.getOnlyElement(scanner));
270 } finally {
271 scanner.close();
272 }
273
274
275 QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
276 admin.setQuota(removeQuota);
277
278
279 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
280 ResultScanner rs = quotaTable.getScanner(new Scan());
281 try {
282 assertNull("Did not expect to find a quota entry", rs.next());
283 } finally {
284 rs.close();
285 }
286 }
287
288
289 scanner = QuotaRetriever.open(admin.getConfiguration());
290 try {
291 assertNull("Did not expect to find a quota entry", scanner.next());
292 } finally {
293 scanner.close();
294 }
295 }
296
297 @Test
298 public void testSetModifyRemoveQuota() throws Exception {
299 Admin admin = TEST_UTIL.getHBaseAdmin();
300 final TableName tn = TableName.valueOf("table1");
301 final long originalSizeLimit = 1024L * 1024L * 1024L * 1024L * 5L;
302 final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_WRITES;
303 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
304 tn, originalSizeLimit, violationPolicy);
305 admin.setQuota(settings);
306
307
308 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
309 ResultScanner scanner = quotaTable.getScanner(new Scan());
310 try {
311 Result r = Iterables.getOnlyElement(scanner);
312 CellScanner cells = r.cellScanner();
313 assertTrue("Expected to find a cell", cells.advance());
314 assertSpaceQuota(originalSizeLimit, violationPolicy, cells.current());
315 } finally {
316 scanner.close();
317 }
318 }
319
320
321 QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration());
322 try {
323 assertSpaceQuota(originalSizeLimit, violationPolicy, Iterables.getOnlyElement(quotaScanner));
324 } finally {
325 quotaScanner.close();
326 }
327
328
329 final long newSizeLimit = 1024L * 1024L * 1024L * 1024L;
330 final SpaceViolationPolicy newViolationPolicy = SpaceViolationPolicy.NO_WRITES_COMPACTIONS;
331 QuotaSettings newSettings = QuotaSettingsFactory.limitTableSpace(
332 tn, newSizeLimit, newViolationPolicy);
333 admin.setQuota(newSettings);
334
335
336 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
337 ResultScanner scanner = quotaTable.getScanner(new Scan());
338 try {
339 Result r = Iterables.getOnlyElement(scanner);
340 CellScanner cells = r.cellScanner();
341 assertTrue("Expected to find a cell", cells.advance());
342 assertSpaceQuota(newSizeLimit, newViolationPolicy, cells.current());
343 } finally {
344 scanner.close();
345 }
346 }
347
348
349 quotaScanner = QuotaRetriever.open(admin.getConfiguration());
350 try {
351 assertSpaceQuota(newSizeLimit, newViolationPolicy, Iterables.getOnlyElement(quotaScanner));
352 } finally {
353 quotaScanner.close();
354 }
355
356
357 QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
358 admin.setQuota(removeQuota);
359
360
361 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
362 ResultScanner scanner = quotaTable.getScanner(new Scan());
363 try {
364 assertNull("Did not expect to find a quota entry", scanner.next());
365 } finally {
366 scanner.close();
367 }
368 }
369
370
371 quotaScanner = QuotaRetriever.open(admin.getConfiguration());
372 try {
373 assertNull("Did not expect to find a quota entry", quotaScanner.next());
374 } finally {
375 quotaScanner.close();
376 }
377 }
378
379 private void assertNumResults(int expected, final QuotaFilter filter) throws Exception {
380 assertEquals(expected, countResults(filter));
381 }
382
383 private void assertSpaceQuota(
384 long sizeLimit, SpaceViolationPolicy violationPolicy, Cell cell) throws Exception {
385 Quotas q = QuotaTableUtil.quotasFromData(
386 cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
387 assertTrue("Quota should have space quota defined", q.hasSpace());
388 QuotaProtos.SpaceQuota spaceQuota = q.getSpace();
389 assertEquals(sizeLimit, spaceQuota.getSoftLimit());
390 assertEquals(violationPolicy, ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy()));
391 }
392
393 private void assertSpaceQuota(
394 long sizeLimit, SpaceViolationPolicy violationPolicy, QuotaSettings actualSettings) {
395 assertTrue("The actual QuotaSettings was not an instance of " + SpaceLimitSettings.class
396 + " but of " + actualSettings.getClass(), actualSettings instanceof SpaceLimitSettings);
397 SpaceLimitRequest spaceLimitRequest = ((SpaceLimitSettings) actualSettings).getProto();
398 assertEquals(sizeLimit, spaceLimitRequest.getQuota().getSoftLimit());
399 assertEquals(violationPolicy,
400 ProtobufUtil.toViolationPolicy(spaceLimitRequest.getQuota().getViolationPolicy()));
401 }
402
403 private int countResults(final QuotaFilter filter) throws Exception {
404 QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration(), filter);
405 try {
406 int count = 0;
407 for (QuotaSettings settings : scanner) {
408 LOG.debug(settings);
409 count++;
410 }
411 return count;
412 } finally {
413 scanner.close();
414 }
415 }
416 }