View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  
12  package org.apache.hadoop.hbase.quotas;
13  
14  import static org.junit.Assert.assertEquals;
15  import static org.junit.Assert.fail;
16  
17  import java.util.concurrent.TimeUnit;
18  
19  import org.apache.commons.logging.Log;
20  import org.apache.commons.logging.LogFactory;
21  import org.apache.hadoop.hbase.Cell;
22  import org.apache.hadoop.hbase.CellScanner;
23  import org.apache.hadoop.hbase.HBaseTestingUtility;
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.TableName;
26  import org.apache.hadoop.hbase.client.Admin;
27  import org.apache.hadoop.hbase.client.Result;
28  import org.apache.hadoop.hbase.client.ResultScanner;
29  import org.apache.hadoop.hbase.client.Scan;
30  import org.apache.hadoop.hbase.client.Table;
31  import org.apache.hadoop.hbase.security.User;
32  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
33  import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
34  import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
35  import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceLimitRequest;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.junit.AfterClass;
38  import org.junit.BeforeClass;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  import com.google.common.collect.Iterables;
43  
44  import static org.junit.Assert.assertEquals;
45  import static org.junit.Assert.assertNull;
46  import static org.junit.Assert.assertTrue;
47  import static org.junit.Assert.fail;
48  
49  /**
50   * minicluster tests that validate that quota entries are properly set in the quota table
51   */
52  @Category({ MediumTests.class })
53  public class TestQuotaAdmin {
54    final Log LOG = LogFactory.getLog(getClass());
55  
56    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57  
58    @BeforeClass
59    public static void setUpBeforeClass() throws Exception {
60      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
61      TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
62      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
63      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
64      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
65      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
66      TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
67      TEST_UTIL.startMiniCluster(1);
68      TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
69    }
70  
71    @AfterClass
72    public static void tearDownAfterClass() throws Exception {
73      TEST_UTIL.shutdownMiniCluster();
74    }
75  
76    @Test
77    public void testThrottleType() throws Exception {
78      Admin admin = TEST_UTIL.getHBaseAdmin();
79      String userName = User.getCurrent().getShortName();
80  
81      admin.setQuota(QuotaSettingsFactory
82        .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
83      admin.setQuota(QuotaSettingsFactory
84        .throttleUser(userName, ThrottleType.WRITE_NUMBER, 12, TimeUnit.MINUTES));
85      admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
86  
87      try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
88        int countThrottle = 0;
89        int countGlobalBypass = 0;
90        for (QuotaSettings settings: scanner) {
91          switch (settings.getQuotaType()) {
92            case THROTTLE:
93              ThrottleSettings throttle = (ThrottleSettings)settings;
94              if (throttle.getSoftLimit() == 6) {
95                assertEquals(ThrottleType.READ_NUMBER, throttle.getThrottleType());
96              } else if (throttle.getSoftLimit() == 12) {
97                assertEquals(ThrottleType.WRITE_NUMBER, throttle.getThrottleType());
98              } else {
99                fail("should not come here, because don't set quota with this limit");
100             }
101             assertEquals(userName, throttle.getUserName());
102             assertEquals(null, throttle.getTableName());
103             assertEquals(null, throttle.getNamespace());
104             assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
105             countThrottle++;
106             break;
107           case GLOBAL_BYPASS:
108             countGlobalBypass++;
109             break;
110           default:
111             fail("unexpected settings type: " + settings.getQuotaType());
112         }
113       }
114       assertEquals(2, countThrottle);
115       assertEquals(1, countGlobalBypass);
116     }
117 
118     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
119     assertNumResults(1, null);
120     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
121     assertNumResults(0, null);
122   }
123 
124   @Test
125   public void testSimpleScan() throws Exception {
126     Admin admin = TEST_UTIL.getHBaseAdmin();
127     String userName = User.getCurrent().getShortName();
128 
129     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
130       TimeUnit.MINUTES));
131     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
132 
133     try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
134       int countThrottle = 0;
135       int countGlobalBypass = 0;
136       for (QuotaSettings settings : scanner) {
137         LOG.debug(settings);
138         switch (settings.getQuotaType()) {
139         case THROTTLE:
140           ThrottleSettings throttle = (ThrottleSettings) settings;
141           assertEquals(userName, throttle.getUserName());
142           assertEquals(null, throttle.getTableName());
143           assertEquals(null, throttle.getNamespace());
144           assertEquals(6, throttle.getSoftLimit());
145           assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
146           countThrottle++;
147           break;
148         case GLOBAL_BYPASS:
149           countGlobalBypass++;
150           break;
151         default:
152           fail("unexpected settings type: " + settings.getQuotaType());
153         }
154       }
155       assertEquals(1, countThrottle);
156       assertEquals(1, countGlobalBypass);
157     }
158 
159     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
160     assertNumResults(1, null);
161     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
162     assertNumResults(0, null);
163   }
164 
165   @Test
166   public void testQuotaRetrieverFilter() throws Exception {
167     Admin admin = TEST_UTIL.getHBaseAdmin();
168     TableName[] tables =
169         new TableName[] { TableName.valueOf("T0"), TableName.valueOf("T01"),
170             TableName.valueOf("NS0:T2"), };
171     String[] namespaces = new String[] { "NS0", "NS01", "NS2" };
172     String[] users = new String[] { "User0", "User01", "User2" };
173 
174     for (String user : users) {
175       admin.setQuota(QuotaSettingsFactory.throttleUser(user, ThrottleType.REQUEST_NUMBER, 1,
176         TimeUnit.MINUTES));
177 
178       for (TableName table : tables) {
179         admin.setQuota(QuotaSettingsFactory.throttleUser(user, table, ThrottleType.REQUEST_NUMBER,
180           2, TimeUnit.MINUTES));
181       }
182 
183       for (String ns : namespaces) {
184         admin.setQuota(QuotaSettingsFactory.throttleUser(user, ns, ThrottleType.REQUEST_NUMBER, 3,
185           TimeUnit.MINUTES));
186       }
187     }
188     assertNumResults(21, null);
189 
190     for (TableName table : tables) {
191       admin.setQuota(QuotaSettingsFactory.throttleTable(table, ThrottleType.REQUEST_NUMBER, 4,
192         TimeUnit.MINUTES));
193     }
194     assertNumResults(24, null);
195 
196     for (String ns : namespaces) {
197       admin.setQuota(QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_NUMBER, 5,
198         TimeUnit.MINUTES));
199     }
200     assertNumResults(27, null);
201 
202     assertNumResults(7, new QuotaFilter().setUserFilter("User0"));
203     assertNumResults(0, new QuotaFilter().setUserFilter("User"));
204     assertNumResults(21, new QuotaFilter().setUserFilter("User.*"));
205     assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0"));
206     assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("NS.*"));
207     assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setTableFilter("T"));
208     assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T.*"));
209     assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS0"));
210     assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS"));
211     assertNumResults(9, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS.*"));
212     assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0")
213         .setNamespaceFilter("NS0"));
214     assertNumResults(1, new QuotaFilter().setTableFilter("T0"));
215     assertNumResults(0, new QuotaFilter().setTableFilter("T"));
216     assertNumResults(2, new QuotaFilter().setTableFilter("T.*"));
217     assertNumResults(3, new QuotaFilter().setTableFilter(".*T.*"));
218     assertNumResults(1, new QuotaFilter().setNamespaceFilter("NS0"));
219     assertNumResults(0, new QuotaFilter().setNamespaceFilter("NS"));
220     assertNumResults(3, new QuotaFilter().setNamespaceFilter("NS.*"));
221 
222     for (String user : users) {
223       admin.setQuota(QuotaSettingsFactory.unthrottleUser(user));
224       for (TableName table : tables) {
225         admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, table));
226       }
227       for (String ns : namespaces) {
228         admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, ns));
229       }
230     }
231     assertNumResults(6, null);
232 
233     for (TableName table : tables) {
234       admin.setQuota(QuotaSettingsFactory.unthrottleTable(table));
235     }
236     assertNumResults(3, null);
237 
238     for (String ns : namespaces) {
239       admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(ns));
240     }
241     assertNumResults(0, null);
242   }
243 
244   @Test
245   public void testSetGetRemoveSpaceQuota() throws Exception {
246     Admin admin = TEST_UTIL.getHBaseAdmin();
247     final TableName tn = TableName.valueOf("table1");
248     final long sizeLimit = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB
249     final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_WRITES;
250     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy);
251     admin.setQuota(settings);
252 
253     // Verify the Quotas in the table
254     try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
255       ResultScanner scanner = quotaTable.getScanner(new Scan());
256       try {
257         Result r = Iterables.getOnlyElement(scanner);
258         CellScanner cells = r.cellScanner();
259         assertTrue("Expected to find a cell", cells.advance());
260         assertSpaceQuota(sizeLimit, violationPolicy, cells.current());
261       } finally {
262         scanner.close();
263       }
264     }
265 
266     // Verify we can retrieve it via the QuotaRetriever API
267     QuotaRetriever scanner = QuotaRetriever.open(admin.getConfiguration());
268     try {
269       assertSpaceQuota(sizeLimit, violationPolicy, Iterables.getOnlyElement(scanner));
270     } finally {
271       scanner.close();
272     }
273 
274     // Now, remove the quota
275     QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
276     admin.setQuota(removeQuota);
277 
278     // Verify that the record doesn't exist in the table
279     try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
280       ResultScanner rs = quotaTable.getScanner(new Scan());
281       try {
282         assertNull("Did not expect to find a quota entry", rs.next());
283       } finally {
284         rs.close();
285       }
286     }
287 
288     // Verify that we can also not fetch it via the API
289     scanner = QuotaRetriever.open(admin.getConfiguration());
290     try {
291       assertNull("Did not expect to find a quota entry", scanner.next());
292     } finally {
293       scanner.close();
294     }
295   }
296 
297   @Test
298   public void testSetModifyRemoveQuota() throws Exception {
299     Admin admin = TEST_UTIL.getHBaseAdmin();
300     final TableName tn = TableName.valueOf("table1");
301     final long originalSizeLimit = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB
302     final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_WRITES;
303     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
304         tn, originalSizeLimit, violationPolicy);
305     admin.setQuota(settings);
306 
307     // Verify the Quotas in the table
308     try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
309       ResultScanner scanner = quotaTable.getScanner(new Scan());
310       try {
311         Result r = Iterables.getOnlyElement(scanner);
312         CellScanner cells = r.cellScanner();
313         assertTrue("Expected to find a cell", cells.advance());
314         assertSpaceQuota(originalSizeLimit, violationPolicy, cells.current());
315       } finally {
316         scanner.close();
317       }
318     }
319 
320     // Verify we can retrieve it via the QuotaRetriever API
321     QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration());
322     try {
323       assertSpaceQuota(originalSizeLimit, violationPolicy, Iterables.getOnlyElement(quotaScanner));
324     } finally {
325       quotaScanner.close();
326     }
327 
328     // Setting a new size and policy should be reflected
329     final long newSizeLimit = 1024L * 1024L * 1024L * 1024L; // 1TB
330     final SpaceViolationPolicy newViolationPolicy = SpaceViolationPolicy.NO_WRITES_COMPACTIONS;
331     QuotaSettings newSettings = QuotaSettingsFactory.limitTableSpace(
332         tn, newSizeLimit, newViolationPolicy);
333     admin.setQuota(newSettings);
334 
335     // Verify the new Quotas in the table
336     try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
337       ResultScanner scanner = quotaTable.getScanner(new Scan());
338       try {
339         Result r = Iterables.getOnlyElement(scanner);
340         CellScanner cells = r.cellScanner();
341         assertTrue("Expected to find a cell", cells.advance());
342         assertSpaceQuota(newSizeLimit, newViolationPolicy, cells.current());
343       } finally {
344         scanner.close();
345       }
346     }
347 
348     // Verify we can retrieve the new quota via the QuotaRetriever API
349     quotaScanner = QuotaRetriever.open(admin.getConfiguration());
350     try {
351       assertSpaceQuota(newSizeLimit, newViolationPolicy, Iterables.getOnlyElement(quotaScanner));
352     } finally {
353       quotaScanner.close();
354     }
355 
356     // Now, remove the quota
357     QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
358     admin.setQuota(removeQuota);
359 
360     // Verify that the record doesn't exist in the table
361     try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
362       ResultScanner scanner = quotaTable.getScanner(new Scan());
363       try {
364         assertNull("Did not expect to find a quota entry", scanner.next());
365       } finally {
366         scanner.close();
367       }
368     }
369 
370     // Verify that we can also not fetch it via the API
371     quotaScanner = QuotaRetriever.open(admin.getConfiguration());
372     try {
373       assertNull("Did not expect to find a quota entry", quotaScanner.next());
374     } finally {
375       quotaScanner.close();
376     }
377   }
378 
379   private void assertNumResults(int expected, final QuotaFilter filter) throws Exception {
380     assertEquals(expected, countResults(filter));
381   }
382 
383   private void assertSpaceQuota(
384       long sizeLimit, SpaceViolationPolicy violationPolicy, Cell cell) throws Exception {
385     Quotas q = QuotaTableUtil.quotasFromData(
386         cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
387     assertTrue("Quota should have space quota defined", q.hasSpace());
388     QuotaProtos.SpaceQuota spaceQuota = q.getSpace();
389     assertEquals(sizeLimit, spaceQuota.getSoftLimit());
390     assertEquals(violationPolicy, ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy()));
391   }
392 
393   private void assertSpaceQuota(
394       long sizeLimit, SpaceViolationPolicy violationPolicy, QuotaSettings actualSettings) {
395     assertTrue("The actual QuotaSettings was not an instance of " + SpaceLimitSettings.class
396         + " but of " + actualSettings.getClass(), actualSettings instanceof SpaceLimitSettings);
397     SpaceLimitRequest spaceLimitRequest = ((SpaceLimitSettings) actualSettings).getProto();
398     assertEquals(sizeLimit, spaceLimitRequest.getQuota().getSoftLimit());
399     assertEquals(violationPolicy,
400         ProtobufUtil.toViolationPolicy(spaceLimitRequest.getQuota().getViolationPolicy()));
401   }
402 
403   private int countResults(final QuotaFilter filter) throws Exception {
404     QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration(), filter);
405     try {
406       int count = 0;
407       for (QuotaSettings settings : scanner) {
408         LOG.debug(settings);
409         count++;
410       }
411       return count;
412     } finally {
413       scanner.close();
414     }
415   }
416 }