1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19 import org.junit.Ignore;
20
21 import java.io.IOException;
22 import java.util.Collections;
23 import java.util.List;
24 import java.util.Set;
25 import java.util.concurrent.CountDownLatch;
26
27 import org.apache.commons.lang.StringUtils;
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Coprocessor;
34 import org.apache.hadoop.hbase.CoprocessorEnvironment;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.NamespaceDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.Waiter;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Mutation;
50 import org.apache.hadoop.hbase.client.RegionLocator;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.master.RegionState;
64 import org.apache.hadoop.hbase.master.RegionStates;
65 import org.apache.hadoop.hbase.master.TableNamespaceManager;
66 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68 import org.apache.hadoop.hbase.quotas.QuotaUtil;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73 import org.apache.hadoop.hbase.regionserver.Store;
74 import org.apache.hadoop.hbase.regionserver.StoreFile;
75 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
76 import org.apache.hadoop.hbase.testclassification.MediumTests;
77 import org.apache.hadoop.hbase.util.Bytes;
78 import org.apache.hadoop.hbase.util.FSUtils;
79 import org.apache.zookeeper.KeeperException;
80 import org.junit.After;
81 import org.junit.AfterClass;
82 import org.junit.BeforeClass;
83 import org.junit.Rule;
84 import org.junit.Test;
85 import org.junit.experimental.categories.Category;
86
87 import com.google.common.collect.Sets;
88
89 @Category(MediumTests.class)
90 public class TestNamespaceAuditor {
91 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
92 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
93 private static HBaseAdmin ADMIN;
94 private String prefix = "TestNamespaceAuditor";
95
96 @BeforeClass
97 public static void before() throws Exception {
98 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
99 setupOnce();
100 }
101
102 public static void setupOnce() throws Exception, IOException {
103 Configuration conf = UTIL.getConfiguration();
104 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
105 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
106 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
107 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
108 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
109 RegionServerObserver.class);
110 UTIL.startMiniCluster(1, 1);
111 waitForQuotaInitialize(UTIL);
112 ADMIN = UTIL.getHBaseAdmin();
113 }
114
115 @AfterClass
116 public static void tearDown() throws Exception {
117 UTIL.shutdownMiniCluster();
118 }
119
120 @After
121 public void cleanup() throws Exception, KeeperException {
122 for (HTableDescriptor table : ADMIN.listTables()) {
123 ADMIN.disableTable(table.getTableName());
124 deleteTable(table.getTableName());
125 }
126 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
127 if (ns.getName().startsWith(prefix)) {
128 ADMIN.deleteNamespace(ns.getName());
129 }
130 }
131 assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
132 .getMasterQuotaManager().isQuotaInitialized());
133 }
134
135 @Test(timeout = 60000)
136 public void testTableOperations() throws Exception {
137 String nsp = prefix + "_np2";
138 NamespaceDescriptor nspDesc =
139 NamespaceDescriptor.create(nsp)
140 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
141 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
142 ADMIN.createNamespace(nspDesc);
143 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
144 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
145 HTableDescriptor tableDescOne =
146 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
147 HTableDescriptor tableDescTwo =
148 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
149 HTableDescriptor tableDescThree =
150 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
151 ADMIN.createTable(tableDescOne);
152 boolean constraintViolated = false;
153 try {
154 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
155 } catch (Exception exp) {
156 assertTrue(exp instanceof IOException);
157 constraintViolated = true;
158 } finally {
159 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
160 constraintViolated);
161 }
162 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
163 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
164 assertNotNull(nspState);
165 assertTrue(nspState.getTables().size() == 2);
166 assertTrue(nspState.getRegionCount() == 5);
167 constraintViolated = false;
168 try {
169 ADMIN.createTable(tableDescThree);
170 } catch (Exception exp) {
171 assertTrue(exp instanceof IOException);
172 constraintViolated = true;
173 } finally {
174 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
175 constraintViolated);
176 }
177 }
178
179 @Test
180 public void testValidQuotas() throws Exception {
181 boolean exceptionCaught = false;
182 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
183 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
184 NamespaceDescriptor nspDesc =
185 NamespaceDescriptor.create(prefix + "vq1")
186 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
187 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
188 try {
189 ADMIN.createNamespace(nspDesc);
190 } catch (Exception exp) {
191 LOG.warn(exp);
192 exceptionCaught = true;
193 } finally {
194 assertTrue(exceptionCaught);
195 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
196 }
197 nspDesc =
198 NamespaceDescriptor.create(prefix + "vq2")
199 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
200 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
201 try {
202 ADMIN.createNamespace(nspDesc);
203 } catch (Exception exp) {
204 LOG.warn(exp);
205 exceptionCaught = true;
206 } finally {
207 assertTrue(exceptionCaught);
208 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
209 }
210 nspDesc =
211 NamespaceDescriptor.create(prefix + "vq3")
212 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
213 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
214 try {
215 ADMIN.createNamespace(nspDesc);
216 } catch (Exception exp) {
217 LOG.warn(exp);
218 exceptionCaught = true;
219 } finally {
220 assertTrue(exceptionCaught);
221 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
222 }
223 nspDesc =
224 NamespaceDescriptor.create(prefix + "vq4")
225 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
226 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
227 try {
228 ADMIN.createNamespace(nspDesc);
229 } catch (Exception exp) {
230 LOG.warn(exp);
231 exceptionCaught = true;
232 } finally {
233 assertTrue(exceptionCaught);
234 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
235 }
236 }
237
238 @Test
239 public void testDeleteTable() throws Exception {
240 String namespace = prefix + "_dummy";
241 NamespaceDescriptor nspDesc =
242 NamespaceDescriptor.create(namespace)
243 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
244 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
245 ADMIN.createNamespace(nspDesc);
246 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
247 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
248 assertNotNull("Namespace state found null for " + namespace, stateInfo);
249 HTableDescriptor tableDescOne =
250 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
251 HTableDescriptor tableDescTwo =
252 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
253 ADMIN.createTable(tableDescOne);
254 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
255 stateInfo = getNamespaceState(nspDesc.getName());
256 assertNotNull("Namespace state found to be null.", stateInfo);
257 assertEquals(2, stateInfo.getTables().size());
258 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
259 assertEquals(6, stateInfo.getRegionCount());
260 ADMIN.disableTable(tableDescOne.getTableName());
261 deleteTable(tableDescOne.getTableName());
262 stateInfo = getNamespaceState(nspDesc.getName());
263 assertNotNull("Namespace state found to be null.", stateInfo);
264 assertEquals(5, stateInfo.getRegionCount());
265 assertEquals(1, stateInfo.getTables().size());
266 ADMIN.disableTable(tableDescTwo.getTableName());
267 deleteTable(tableDescTwo.getTableName());
268 ADMIN.deleteNamespace(namespace);
269 stateInfo = getNamespaceState(namespace);
270 assertNull("Namespace state not found to be null.", stateInfo);
271 }
272
273 public static class CPRegionServerObserver extends BaseRegionServerObserver {
274 private volatile boolean shouldFailMerge = false;
275
276 public void failMerge(boolean fail) {
277 shouldFailMerge = fail;
278 }
279
280 private boolean triggered = false;
281
282 public synchronized void waitUtilTriggered() throws InterruptedException {
283 while (!triggered) {
284 wait();
285 }
286 }
287
288 @Override
289 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
290 Region regionA, Region regionB) throws IOException {
291 triggered = true;
292 notifyAll();
293 if (shouldFailMerge) {
294 throw new IOException("fail merge");
295 }
296 }
297 }
298
299 @Test
300 public void testRegionMerge() throws Exception {
301 String nsp1 = prefix + "_regiontest";
302 NamespaceDescriptor nspDesc =
303 NamespaceDescriptor.create(nsp1)
304 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
305 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
306 ADMIN.createNamespace(nspDesc);
307 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
308 byte[] columnFamily = Bytes.toBytes("info");
309 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
310 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
311 final int initialRegions = 3;
312 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
313 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
314 Table table = connection.getTable(tableTwo)) {
315 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
316 }
317 ADMIN.flush(tableTwo);
318 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
319 Collections.sort(hris);
320
321 final Set<String> encodedRegionNamesToMerge =
322 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
323 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
324 false);
325 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
326 hris = ADMIN.getTableRegions(tableTwo);
327 assertEquals(initialRegions - 1, hris.size());
328 Collections.sort(hris);
329
330 final HRegionInfo hriToSplit = hris.get(1);
331 ADMIN.split(tableTwo, Bytes.toBytes("500"));
332
333 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
334
335 @Override
336 public boolean evaluate() throws Exception {
337 RegionStates regionStates =
338 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
339 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
340 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
341 return false;
342 }
343 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
344 return false;
345 }
346 }
347 return true;
348 }
349
350 @Override
351 public String explainFailure() throws Exception {
352 RegionStates regionStates =
353 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
354 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
355 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
356 return hriToSplit + " which is expected to be split is still online";
357 }
358 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
359 return hri + " is still in not opened";
360 }
361 }
362 return "Unknown";
363 }
364 });
365 hris = ADMIN.getTableRegions(tableTwo);
366 assertEquals(initialRegions, hris.size());
367 Collections.sort(hris);
368
369
370 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
371 HRegionServer regionServer = cluster.getRegionServer(0);
372 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
373 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
374 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
375 regionServerObserver.failMerge(true);
376 regionServerObserver.triggered = false;
377
378 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
379 false);
380 regionServerObserver.waitUtilTriggered();
381 hris = ADMIN.getTableRegions(tableTwo);
382 assertEquals(initialRegions, hris.size());
383 Collections.sort(hris);
384
385 HRegionInfo hriToSplit2 = hris.get(1);
386 ADMIN.split(tableTwo,
387 TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
388 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
389 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
390 }
391
392 private void waitForMergeToComplete(final TableName tableTwo,
393 final Set<String> encodedRegionNamesToMerge) throws Exception {
394 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
395
396 @Override
397 public boolean evaluate() throws Exception {
398 RegionStates regionStates =
399 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
400 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
401 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
402 return false;
403 }
404 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
405 return false;
406 }
407 }
408 return true;
409 }
410
411 @Override
412 public String explainFailure() throws Exception {
413 RegionStates regionStates =
414 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
415 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
416 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
417 return hri + " which is expected to be merged is still online";
418 }
419 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
420 return hri + " is still in not opened";
421 }
422 }
423 return "Unknown";
424 }
425 });
426 }
427
428 @Ignore("Hangs on occasion waiting on countdown latch") @Test
429 public void testRegionOperations() throws Exception {
430 String nsp1 = prefix + "_regiontest";
431 NamespaceDescriptor nspDesc =
432 NamespaceDescriptor.create(nsp1)
433 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
434 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
435 ADMIN.createNamespace(nspDesc);
436 boolean constraintViolated = false;
437 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
438 byte[] columnFamily = Bytes.toBytes("info");
439 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
440 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
441 NamespaceTableAndRegionInfo stateInfo;
442 try {
443 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
444 } catch (Exception exp) {
445 assertTrue(exp instanceof DoNotRetryIOException);
446 LOG.info(exp);
447 constraintViolated = true;
448 } finally {
449 assertTrue(constraintViolated);
450 }
451 assertFalse(ADMIN.tableExists(tableOne));
452
453 ADMIN.createTable(tableDescOne);
454 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
455 HTable htable = (HTable) connection.getTable(tableOne);
456 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
457 ADMIN.flush(tableOne);
458 stateInfo = getNamespaceState(nsp1);
459 assertEquals(1, stateInfo.getTables().size());
460 assertEquals(1, stateInfo.getRegionCount());
461 restartMaster();
462
463 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
464 CustomObserver observer =
465 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
466 CustomObserver.class.getName());
467 assertNotNull(observer);
468
469 ADMIN.split(tableOne, Bytes.toBytes("500"));
470 observer.postSplit.await();
471 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
472 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473 observer =
474 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475 CustomObserver.class.getName());
476 assertNotNull(observer);
477
478
479 ADMIN.compact(tableOne);
480 observer.postCompact.await();
481
482 ADMIN.split(
483 tableOne,
484 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
485 .getEndKey()));
486 observer.postSplit.await();
487
488 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
489 assertEquals(2, hris.size());
490 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
491
492 htable.close();
493 }
494
495
496
497
498
499
500 @Test(timeout = 180000)
501 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
502 String nsp1 = prefix + "_testRecreateTable";
503 NamespaceDescriptor nspDesc =
504 NamespaceDescriptor.create(nsp1)
505 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
506 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
507 ADMIN.createNamespace(nspDesc);
508 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
509 byte[] columnFamily = Bytes.toBytes("info");
510 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
511 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
512 MasterSyncObserver.throwExceptionInPreCreateTable = true;
513 try {
514 try {
515 ADMIN.createTable(tableDescOne);
516 fail("Table " + tableOne.toString() + "creation should fail.");
517 } catch (Exception exp) {
518 LOG.error(exp);
519 }
520 assertFalse(ADMIN.tableExists(tableOne));
521
522 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
523 assertEquals("First table creation failed in namespace so number of tables in namespace "
524 + "should be 0.", 0, nstate.getTables().size());
525
526 MasterSyncObserver.throwExceptionInPreCreateTable = false;
527 try {
528 ADMIN.createTable(tableDescOne);
529 } catch (Exception e) {
530 fail("Table " + tableOne.toString() + "creation should succeed.");
531 LOG.error(e);
532 }
533 assertTrue(ADMIN.tableExists(tableOne));
534 nstate = getNamespaceState(nsp1);
535 assertEquals("First table was created successfully so table size in namespace should "
536 + "be one now.", 1, nstate.getTables().size());
537 } finally {
538 MasterSyncObserver.throwExceptionInPreCreateTable = false;
539 if (ADMIN.tableExists(tableOne)) {
540 ADMIN.disableTable(tableOne);
541 deleteTable(tableOne);
542 }
543 ADMIN.deleteNamespace(nsp1);
544 }
545 }
546
547 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
548 IOException {
549 return getQuotaManager().getState(namespace);
550 }
551
552 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
553 String skey = Bytes.toString(startKey);
554 int key;
555 if (StringUtils.isBlank(skey)) {
556 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
557 } else {
558 key = (int) (Integer.parseInt(skey) * 1.5);
559 }
560 return Bytes.toBytes("" + key);
561 }
562
563 public static class CustomObserver extends BaseRegionObserver {
564 volatile CountDownLatch postSplit;
565 volatile CountDownLatch preSplitBeforePONR;
566 volatile CountDownLatch postCompact;
567
568 @Override
569 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
570 throws IOException {
571 postSplit.countDown();
572 }
573
574 @Override
575 public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
576 Store store, StoreFile resultFile) throws IOException {
577 postCompact.countDown();
578 }
579
580 @Override
581 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
582 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
583 preSplitBeforePONR.countDown();
584 }
585
586
587 @Override
588 public void start(CoprocessorEnvironment e) throws IOException {
589 postSplit = new CountDownLatch(1);
590 preSplitBeforePONR = new CountDownLatch(1);
591 postCompact = new CountDownLatch(1);
592 }
593 }
594
595 @Test
596 public void testStatePreserve() throws Exception {
597 final String nsp1 = prefix + "_testStatePreserve";
598 NamespaceDescriptor nspDesc =
599 NamespaceDescriptor.create(nsp1)
600 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
601 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
602 ADMIN.createNamespace(nspDesc);
603 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
604 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
605 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
606 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
607 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
608 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
609 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
610 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
611 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
612 ADMIN.disableTable(tableThree);
613 deleteTable(tableThree);
614
615 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
616 @Override
617 public boolean evaluate() throws Exception {
618 return (getNamespaceState(nsp1).getTables().size() == 2);
619 }
620 });
621 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
622 restartMaster();
623 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
624 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
625 .getTables().size(), after.getTables().size());
626 }
627
628 public static void waitForQuotaInitialize(final HBaseTestingUtility util) throws Exception {
629 util.waitFor(60000, new Waiter.Predicate<Exception>() {
630 @Override
631 public boolean evaluate() throws Exception {
632 HMaster master = util.getHBaseCluster().getMaster();
633 if (master == null) {
634 return false;
635 }
636 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
637 return quotaManager != null && quotaManager.isQuotaInitialized();
638 }
639 });
640 }
641
642 private void restartMaster() throws Exception {
643 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
644 UTIL.getHBaseCluster().waitOnMaster(0);
645 UTIL.getHBaseCluster().startMaster();
646 waitForQuotaInitialize(UTIL);
647 }
648
649 private NamespaceAuditor getQuotaManager() {
650 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
651 }
652
653 public static class MasterSyncObserver extends BaseMasterObserver {
654 volatile CountDownLatch tableDeletionLatch;
655 static boolean throwExceptionInPreCreateTable;
656
657 @Override
658 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
659 TableName tableName) throws IOException {
660 tableDeletionLatch = new CountDownLatch(1);
661 }
662
663 @Override
664 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
665 TableName tableName) throws IOException {
666 tableDeletionLatch.countDown();
667 }
668
669 @Override
670 public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
671 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
672 if (throwExceptionInPreCreateTable) {
673 throw new IOException("Throw exception as it is demanded.");
674 }
675 }
676 }
677
678 private void deleteTable(final TableName tableName) throws Exception {
679
680
681 MasterSyncObserver observer =
682 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
683 .findCoprocessor(MasterSyncObserver.class.getName());
684 ADMIN.deleteTable(tableName);
685 observer.tableDeletionLatch.await();
686 }
687
688 @Test(expected = QuotaExceededException.class, timeout = 30000)
689 public void testExceedTableQuotaInNamespace() throws Exception {
690 String nsp = prefix + "_testExceedTableQuotaInNamespace";
691 NamespaceDescriptor nspDesc =
692 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
693 .build();
694 ADMIN.createNamespace(nspDesc);
695 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
696 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
697 HTableDescriptor tableDescOne =
698 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
699 HTableDescriptor tableDescTwo =
700 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701 ADMIN.createTable(tableDescOne);
702 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
703 }
704
705 @Test(expected = QuotaExceededException.class, timeout = 30000)
706 public void testCloneSnapshotQuotaExceed() throws Exception {
707 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
708 NamespaceDescriptor nspDesc =
709 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
710 .build();
711 ADMIN.createNamespace(nspDesc);
712 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
713 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
714 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
715 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
716 ADMIN.createTable(tableDescOne);
717 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
718 ADMIN.snapshot(snapshot, tableName);
719 ADMIN.cloneSnapshot(snapshot, cloneTableName);
720 ADMIN.deleteSnapshot(snapshot);
721 }
722
723 @Test(timeout = 180000)
724 public void testCloneSnapshot() throws Exception {
725 String nsp = prefix + "_testCloneSnapshot";
726 NamespaceDescriptor nspDesc =
727 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
728 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
729 ADMIN.createNamespace(nspDesc);
730 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
731 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
732 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
733 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
734
735 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
736 String snapshot = "snapshot_testCloneSnapshot";
737 ADMIN.snapshot(snapshot, tableName);
738 ADMIN.cloneSnapshot(snapshot, cloneTableName);
739
740 int tableLength;
741 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
742 tableLength = locator.getStartKeys().length;
743 }
744 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
745
746 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
747 tableLength = locator.getStartKeys().length;
748 }
749 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
750
751 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
752 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
753 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
754
755 ADMIN.deleteSnapshot(snapshot);
756 }
757
758 @Test(timeout = 180000)
759 public void testRestoreSnapshot() throws Exception {
760 String nsp = prefix + "_testRestoreSnapshot";
761 NamespaceDescriptor nspDesc =
762 NamespaceDescriptor.create(nsp)
763 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
764 ADMIN.createNamespace(nspDesc);
765 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
766 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
767 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
768 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
769
770 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
771 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
772
773 String snapshot = "snapshot_testRestoreSnapshot";
774 ADMIN.snapshot(snapshot, tableName1);
775
776 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
777 Collections.sort(regions);
778
779 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
780 Thread.sleep(2000);
781 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
782
783 ADMIN.disableTable(tableName1);
784 ADMIN.restoreSnapshot(snapshot);
785
786 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
787
788 ADMIN.enableTable(tableName1);
789 ADMIN.deleteSnapshot(snapshot);
790 }
791
792 @Test(timeout = 180000)
793 public void testRestoreSnapshotQuotaExceed() throws Exception {
794 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
795 NamespaceDescriptor nspDesc =
796 NamespaceDescriptor.create(nsp)
797 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
798 ADMIN.createNamespace(nspDesc);
799 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
800 assertNotNull("Namespace descriptor found null.", ndesc);
801 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
802 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
803 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
804
805 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
806 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
807
808 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
809
810 ADMIN.snapshot(snapshot, tableName1);
811
812 ADMIN.disableTable(tableName1);
813 ADMIN.deleteTable(tableName1);
814 ADMIN.createTable(tableDescOne);
815 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
816 ADMIN.modifyNamespace(ndesc);
817
818 ADMIN.disableTable(tableName1);
819 try {
820 ADMIN.restoreSnapshot(snapshot);
821 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
822 + " wraps IOException into RestoreSnapshotException");
823 } catch (RestoreSnapshotException ignore) {
824 assertTrue(ignore.getCause() instanceof QuotaExceededException);
825 }
826 assertEquals(1, getNamespaceState(nsp).getRegionCount());
827 ADMIN.enableTable(tableName1);
828 ADMIN.deleteSnapshot(snapshot);
829 }
830 }