View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  package org.apache.hadoop.hbase.namespace;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertFalse;
15  import static org.junit.Assert.assertNotNull;
16  import static org.junit.Assert.assertNull;
17  import static org.junit.Assert.assertTrue;
18  import static org.junit.Assert.fail;
19  import org.junit.Ignore;
20  
21  import java.io.IOException;
22  import java.util.Collections;
23  import java.util.List;
24  import java.util.Set;
25  import java.util.concurrent.CountDownLatch;
26  
27  import org.apache.commons.lang.StringUtils;
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.Coprocessor;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MiniHBaseCluster;
42  import org.apache.hadoop.hbase.NamespaceDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.Waiter;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.ConnectionFactory;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Mutation;
50  import org.apache.hadoop.hbase.client.RegionLocator;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54  import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60  import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61  import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62  import org.apache.hadoop.hbase.master.HMaster;
63  import org.apache.hadoop.hbase.master.RegionState;
64  import org.apache.hadoop.hbase.master.RegionStates;
65  import org.apache.hadoop.hbase.master.TableNamespaceManager;
66  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67  import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68  import org.apache.hadoop.hbase.quotas.QuotaUtil;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.Region;
72  import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73  import org.apache.hadoop.hbase.regionserver.Store;
74  import org.apache.hadoop.hbase.regionserver.StoreFile;
75  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
76  import org.apache.hadoop.hbase.testclassification.MediumTests;
77  import org.apache.hadoop.hbase.util.Bytes;
78  import org.apache.hadoop.hbase.util.FSUtils;
79  import org.apache.zookeeper.KeeperException;
80  import org.junit.After;
81  import org.junit.AfterClass;
82  import org.junit.BeforeClass;
83  import org.junit.Rule;
84  import org.junit.Test;
85  import org.junit.experimental.categories.Category;
86  
87  import com.google.common.collect.Sets;
88  
89  @Category(MediumTests.class)
90  public class TestNamespaceAuditor {
91    private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
92    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
93    private static HBaseAdmin ADMIN;
94    private String prefix = "TestNamespaceAuditor";
95  
96    @BeforeClass
97    public static void before() throws Exception {
98      UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
99      setupOnce();
100   }
101 
102   public static void setupOnce() throws Exception, IOException {
103     Configuration conf = UTIL.getConfiguration();
104     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
105     conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
106     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
107     conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
108     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
109       RegionServerObserver.class);
110     UTIL.startMiniCluster(1, 1);
111     waitForQuotaInitialize(UTIL);
112     ADMIN = UTIL.getHBaseAdmin();
113   }
114 
115   @AfterClass
116   public static void tearDown() throws Exception {
117     UTIL.shutdownMiniCluster();
118   }
119 
120   @After
121   public void cleanup() throws Exception, KeeperException {
122     for (HTableDescriptor table : ADMIN.listTables()) {
123       ADMIN.disableTable(table.getTableName());
124       deleteTable(table.getTableName());
125     }
126     for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
127       if (ns.getName().startsWith(prefix)) {
128         ADMIN.deleteNamespace(ns.getName());
129       }
130     }
131     assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
132         .getMasterQuotaManager().isQuotaInitialized());
133   }
134 
135   @Test(timeout = 60000)
136   public void testTableOperations() throws Exception {
137     String nsp = prefix + "_np2";
138     NamespaceDescriptor nspDesc =
139         NamespaceDescriptor.create(nsp)
140             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
141             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
142     ADMIN.createNamespace(nspDesc);
143     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
144     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
145     HTableDescriptor tableDescOne =
146         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
147     HTableDescriptor tableDescTwo =
148         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
149     HTableDescriptor tableDescThree =
150         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
151     ADMIN.createTable(tableDescOne);
152     boolean constraintViolated = false;
153     try {
154       ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
155     } catch (Exception exp) {
156       assertTrue(exp instanceof IOException);
157       constraintViolated = true;
158     } finally {
159       assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
160         constraintViolated);
161     }
162     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
163     NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
164     assertNotNull(nspState);
165     assertTrue(nspState.getTables().size() == 2);
166     assertTrue(nspState.getRegionCount() == 5);
167     constraintViolated = false;
168     try {
169       ADMIN.createTable(tableDescThree);
170     } catch (Exception exp) {
171       assertTrue(exp instanceof IOException);
172       constraintViolated = true;
173     } finally {
174       assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
175         constraintViolated);
176     }
177   }
178 
179   @Test
180   public void testValidQuotas() throws Exception {
181     boolean exceptionCaught = false;
182     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
183     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
184     NamespaceDescriptor nspDesc =
185         NamespaceDescriptor.create(prefix + "vq1")
186             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
187             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
188     try {
189       ADMIN.createNamespace(nspDesc);
190     } catch (Exception exp) {
191       LOG.warn(exp);
192       exceptionCaught = true;
193     } finally {
194       assertTrue(exceptionCaught);
195       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
196     }
197     nspDesc =
198         NamespaceDescriptor.create(prefix + "vq2")
199             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
200             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
201     try {
202       ADMIN.createNamespace(nspDesc);
203     } catch (Exception exp) {
204       LOG.warn(exp);
205       exceptionCaught = true;
206     } finally {
207       assertTrue(exceptionCaught);
208       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
209     }
210     nspDesc =
211         NamespaceDescriptor.create(prefix + "vq3")
212             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
213             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
214     try {
215       ADMIN.createNamespace(nspDesc);
216     } catch (Exception exp) {
217       LOG.warn(exp);
218       exceptionCaught = true;
219     } finally {
220       assertTrue(exceptionCaught);
221       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
222     }
223     nspDesc =
224         NamespaceDescriptor.create(prefix + "vq4")
225             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
226             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
227     try {
228       ADMIN.createNamespace(nspDesc);
229     } catch (Exception exp) {
230       LOG.warn(exp);
231       exceptionCaught = true;
232     } finally {
233       assertTrue(exceptionCaught);
234       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
235     }
236   }
237 
238   @Test
239   public void testDeleteTable() throws Exception {
240     String namespace = prefix + "_dummy";
241     NamespaceDescriptor nspDesc =
242         NamespaceDescriptor.create(namespace)
243             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
244             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
245     ADMIN.createNamespace(nspDesc);
246     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
247     NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
248     assertNotNull("Namespace state found null for " + namespace, stateInfo);
249     HTableDescriptor tableDescOne =
250         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
251     HTableDescriptor tableDescTwo =
252         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
253     ADMIN.createTable(tableDescOne);
254     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
255     stateInfo = getNamespaceState(nspDesc.getName());
256     assertNotNull("Namespace state found to be null.", stateInfo);
257     assertEquals(2, stateInfo.getTables().size());
258     assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
259     assertEquals(6, stateInfo.getRegionCount());
260     ADMIN.disableTable(tableDescOne.getTableName());
261     deleteTable(tableDescOne.getTableName());
262     stateInfo = getNamespaceState(nspDesc.getName());
263     assertNotNull("Namespace state found to be null.", stateInfo);
264     assertEquals(5, stateInfo.getRegionCount());
265     assertEquals(1, stateInfo.getTables().size());
266     ADMIN.disableTable(tableDescTwo.getTableName());
267     deleteTable(tableDescTwo.getTableName());
268     ADMIN.deleteNamespace(namespace);
269     stateInfo = getNamespaceState(namespace);
270     assertNull("Namespace state not found to be null.", stateInfo);
271   }
272 
273   public static class CPRegionServerObserver extends BaseRegionServerObserver {
274     private volatile boolean shouldFailMerge = false;
275 
276     public void failMerge(boolean fail) {
277       shouldFailMerge = fail;
278     }
279 
280     private boolean triggered = false;
281 
282     public synchronized void waitUtilTriggered() throws InterruptedException {
283       while (!triggered) {
284         wait();
285       }
286     }
287 
288     @Override
289     public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
290         Region regionA, Region regionB) throws IOException {
291       triggered = true;
292       notifyAll();
293       if (shouldFailMerge) {
294         throw new IOException("fail merge");
295       }
296     }
297   }
298 
299   @Test
300   public void testRegionMerge() throws Exception {
301     String nsp1 = prefix + "_regiontest";
302     NamespaceDescriptor nspDesc =
303         NamespaceDescriptor.create(nsp1)
304             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
305             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
306     ADMIN.createNamespace(nspDesc);
307     final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
308     byte[] columnFamily = Bytes.toBytes("info");
309     HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
310     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
311     final int initialRegions = 3;
312     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
313     try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
314         Table table = connection.getTable(tableTwo)) {
315       UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
316     }
317     ADMIN.flush(tableTwo);
318     List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
319     Collections.sort(hris);
320     // merge the two regions
321     final Set<String> encodedRegionNamesToMerge =
322         Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
323     ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
324       false);
325     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
326     hris = ADMIN.getTableRegions(tableTwo);
327     assertEquals(initialRegions - 1, hris.size());
328     Collections.sort(hris);
329 
330     final HRegionInfo hriToSplit = hris.get(1);
331     ADMIN.split(tableTwo, Bytes.toBytes("500"));
332 
333     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
334 
335       @Override
336       public boolean evaluate() throws Exception {
337         RegionStates regionStates =
338             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
339         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
340           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
341             return false;
342           }
343           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
344             return false;
345           }
346         }
347         return true;
348       }
349 
350       @Override
351       public String explainFailure() throws Exception {
352         RegionStates regionStates =
353             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
354         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
355           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
356             return hriToSplit + " which is expected to be split is still online";
357           }
358           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
359             return hri + " is still in not opened";
360           }
361         }
362         return "Unknown";
363       }
364     });
365     hris = ADMIN.getTableRegions(tableTwo);
366     assertEquals(initialRegions, hris.size());
367     Collections.sort(hris);
368 
369     // fail region merge through Coprocessor hook
370     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
371     HRegionServer regionServer = cluster.getRegionServer(0);
372     RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
373     Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
374     CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
375     regionServerObserver.failMerge(true);
376     regionServerObserver.triggered = false;
377 
378     ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
379       false);
380     regionServerObserver.waitUtilTriggered();
381     hris = ADMIN.getTableRegions(tableTwo);
382     assertEquals(initialRegions, hris.size());
383     Collections.sort(hris);
384     // verify that we cannot split
385     HRegionInfo hriToSplit2 = hris.get(1);
386     ADMIN.split(tableTwo,
387       TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
388     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
389     assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
390   }
391 
392   private void waitForMergeToComplete(final TableName tableTwo,
393       final Set<String> encodedRegionNamesToMerge) throws Exception {
394     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
395 
396       @Override
397       public boolean evaluate() throws Exception {
398         RegionStates regionStates =
399             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
400         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
401           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
402             return false;
403           }
404           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
405             return false;
406           }
407         }
408         return true;
409       }
410 
411       @Override
412       public String explainFailure() throws Exception {
413         RegionStates regionStates =
414             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
415         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
416           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
417             return hri + " which is expected to be merged is still online";
418           }
419           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
420             return hri + " is still in not opened";
421           }
422         }
423         return "Unknown";
424       }
425     });
426   }
427 
428   @Ignore("Hangs on occasion waiting on countdown latch") @Test
429   public void testRegionOperations() throws Exception {
430     String nsp1 = prefix + "_regiontest";
431     NamespaceDescriptor nspDesc =
432         NamespaceDescriptor.create(nsp1)
433             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
434             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
435     ADMIN.createNamespace(nspDesc);
436     boolean constraintViolated = false;
437     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
438     byte[] columnFamily = Bytes.toBytes("info");
439     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
440     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
441     NamespaceTableAndRegionInfo stateInfo;
442     try {
443       ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
444     } catch (Exception exp) {
445       assertTrue(exp instanceof DoNotRetryIOException);
446       LOG.info(exp);
447       constraintViolated = true;
448     } finally {
449       assertTrue(constraintViolated);
450     }
451     assertFalse(ADMIN.tableExists(tableOne));
452     // This call will pass.
453     ADMIN.createTable(tableDescOne);
454     Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
455     HTable htable = (HTable) connection.getTable(tableOne);
456     UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
457     ADMIN.flush(tableOne);
458     stateInfo = getNamespaceState(nsp1);
459     assertEquals(1, stateInfo.getTables().size());
460     assertEquals(1, stateInfo.getRegionCount());
461     restartMaster();
462 
463     HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
464     CustomObserver observer =
465         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
466           CustomObserver.class.getName());
467     assertNotNull(observer);
468 
469     ADMIN.split(tableOne, Bytes.toBytes("500"));
470     observer.postSplit.await();
471     assertEquals(2, ADMIN.getTableRegions(tableOne).size());
472     actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473     observer =
474         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475           CustomObserver.class.getName());
476     assertNotNull(observer);
477 
478     //Before we go on split, we should remove all reference store files.
479     ADMIN.compact(tableOne);
480     observer.postCompact.await();
481 
482     ADMIN.split(
483       tableOne,
484       getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
485           .getEndKey()));
486     observer.postSplit.await();
487     // Make sure no regions have been added.
488     List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
489     assertEquals(2, hris.size());
490     assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
491 
492     htable.close();
493   }
494 
495   /*
496    * Create a table and make sure that the table creation fails after adding this table entry into
497    * namespace quota cache. Now correct the failure and recreate the table with same name.
498    * HBASE-13394
499    */
500   @Test(timeout = 180000)
501   public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
502     String nsp1 = prefix + "_testRecreateTable";
503     NamespaceDescriptor nspDesc =
504         NamespaceDescriptor.create(nsp1)
505             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
506             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
507     ADMIN.createNamespace(nspDesc);
508     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
509     byte[] columnFamily = Bytes.toBytes("info");
510     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
511     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
512     MasterSyncObserver.throwExceptionInPreCreateTable = true;
513     try {
514       try {
515         ADMIN.createTable(tableDescOne);
516         fail("Table " + tableOne.toString() + "creation should fail.");
517       } catch (Exception exp) {
518         LOG.error(exp);
519       }
520       assertFalse(ADMIN.tableExists(tableOne));
521 
522       NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
523       assertEquals("First table creation failed in namespace so number of tables in namespace "
524           + "should be 0.", 0, nstate.getTables().size());
525 
526       MasterSyncObserver.throwExceptionInPreCreateTable = false;
527       try {
528         ADMIN.createTable(tableDescOne);
529       } catch (Exception e) {
530         fail("Table " + tableOne.toString() + "creation should succeed.");
531         LOG.error(e);
532       }
533       assertTrue(ADMIN.tableExists(tableOne));
534       nstate = getNamespaceState(nsp1);
535       assertEquals("First table was created successfully so table size in namespace should "
536           + "be one now.", 1, nstate.getTables().size());
537     } finally {
538       MasterSyncObserver.throwExceptionInPreCreateTable = false;
539       if (ADMIN.tableExists(tableOne)) {
540         ADMIN.disableTable(tableOne);
541         deleteTable(tableOne);
542       }
543       ADMIN.deleteNamespace(nsp1);
544     }
545   }
546 
547   private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
548       IOException {
549     return getQuotaManager().getState(namespace);
550   }
551 
552   byte[] getSplitKey(byte[] startKey, byte[] endKey) {
553     String skey = Bytes.toString(startKey);
554     int key;
555     if (StringUtils.isBlank(skey)) {
556       key = Integer.parseInt(Bytes.toString(endKey)) / 2;
557     } else {
558       key = (int) (Integer.parseInt(skey) * 1.5);
559     }
560     return Bytes.toBytes("" + key);
561   }
562 
563   public static class CustomObserver extends BaseRegionObserver {
564     volatile CountDownLatch postSplit;
565     volatile CountDownLatch preSplitBeforePONR;
566     volatile CountDownLatch postCompact;
567 
568     @Override
569     public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
570         throws IOException {
571       postSplit.countDown();
572     }
573 
574     @Override
575     public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
576                             Store store, StoreFile resultFile) throws IOException {
577       postCompact.countDown();
578     }
579 
580     @Override
581     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
582         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
583       preSplitBeforePONR.countDown();
584     }
585 
586 
587     @Override
588     public void start(CoprocessorEnvironment e) throws IOException {
589       postSplit = new CountDownLatch(1);
590       preSplitBeforePONR = new CountDownLatch(1);
591       postCompact = new CountDownLatch(1);
592     }
593   }
594 
595   @Test
596   public void testStatePreserve() throws Exception {
597     final String nsp1 = prefix + "_testStatePreserve";
598     NamespaceDescriptor nspDesc =
599         NamespaceDescriptor.create(nsp1)
600             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
601             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
602     ADMIN.createNamespace(nspDesc);
603     TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
604     TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
605     TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
606     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
607     HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
608     HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
609     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
610     ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
611     ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
612     ADMIN.disableTable(tableThree);
613     deleteTable(tableThree);
614     // wait for chore to complete
615     UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
616       @Override
617       public boolean evaluate() throws Exception {
618         return (getNamespaceState(nsp1).getTables().size() == 2);
619       }
620     });
621     NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
622     restartMaster();
623     NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
624     assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
625         .getTables().size(), after.getTables().size());
626   }
627 
628   public static void waitForQuotaInitialize(final HBaseTestingUtility util) throws Exception {
629     util.waitFor(60000, new Waiter.Predicate<Exception>() {
630       @Override
631       public boolean evaluate() throws Exception {
632         HMaster master = util.getHBaseCluster().getMaster();
633         if (master == null) {
634           return false;
635         }
636         MasterQuotaManager quotaManager = master.getMasterQuotaManager();
637         return quotaManager != null && quotaManager.isQuotaInitialized();
638       }
639     });
640   }
641 
642   private void restartMaster() throws Exception {
643     UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
644     UTIL.getHBaseCluster().waitOnMaster(0);
645     UTIL.getHBaseCluster().startMaster();
646     waitForQuotaInitialize(UTIL);
647   }
648 
649   private NamespaceAuditor getQuotaManager() {
650     return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
651   }
652 
653   public static class MasterSyncObserver extends BaseMasterObserver {
654     volatile CountDownLatch tableDeletionLatch;
655     static boolean throwExceptionInPreCreateTable;
656 
657     @Override
658     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
659         TableName tableName) throws IOException {
660       tableDeletionLatch = new CountDownLatch(1);
661     }
662 
663     @Override
664     public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
665         TableName tableName) throws IOException {
666       tableDeletionLatch.countDown();
667     }
668 
669     @Override
670     public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
671         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
672       if (throwExceptionInPreCreateTable) {
673         throw new IOException("Throw exception as it is demanded.");
674       }
675     }
676   }
677 
678   private void deleteTable(final TableName tableName) throws Exception {
679     // NOTE: We need a latch because admin is not sync,
680     // so the postOp coprocessor method may be called after the admin operation returned.
681     MasterSyncObserver observer =
682         (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
683             .findCoprocessor(MasterSyncObserver.class.getName());
684     ADMIN.deleteTable(tableName);
685     observer.tableDeletionLatch.await();
686   }
687 
688   @Test(expected = QuotaExceededException.class, timeout = 30000)
689   public void testExceedTableQuotaInNamespace() throws Exception {
690     String nsp = prefix + "_testExceedTableQuotaInNamespace";
691     NamespaceDescriptor nspDesc =
692         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
693             .build();
694     ADMIN.createNamespace(nspDesc);
695     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
696     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
697     HTableDescriptor tableDescOne =
698         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
699     HTableDescriptor tableDescTwo =
700         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701     ADMIN.createTable(tableDescOne);
702     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
703   }
704   
705   @Test(expected = QuotaExceededException.class, timeout = 30000)
706   public void testCloneSnapshotQuotaExceed() throws Exception {
707     String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
708     NamespaceDescriptor nspDesc =
709         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
710             .build();
711     ADMIN.createNamespace(nspDesc);
712     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
713     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
714     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
715     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
716     ADMIN.createTable(tableDescOne);
717     String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
718     ADMIN.snapshot(snapshot, tableName);
719     ADMIN.cloneSnapshot(snapshot, cloneTableName);
720     ADMIN.deleteSnapshot(snapshot);
721   }
722 
723   @Test(timeout = 180000)
724   public void testCloneSnapshot() throws Exception {
725     String nsp = prefix + "_testCloneSnapshot";
726     NamespaceDescriptor nspDesc =
727         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
728             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
729     ADMIN.createNamespace(nspDesc);
730     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
731     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
732     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
733     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
734 
735     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
736     String snapshot = "snapshot_testCloneSnapshot";
737     ADMIN.snapshot(snapshot, tableName);
738     ADMIN.cloneSnapshot(snapshot, cloneTableName);
739 
740     int tableLength;
741     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
742       tableLength = locator.getStartKeys().length;
743     }
744     assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
745 
746     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
747       tableLength = locator.getStartKeys().length;
748     }
749     assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
750 
751     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
752     assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
753     assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
754 
755     ADMIN.deleteSnapshot(snapshot);
756   }
757 
758   @Test(timeout = 180000)
759   public void testRestoreSnapshot() throws Exception {
760     String nsp = prefix + "_testRestoreSnapshot";
761     NamespaceDescriptor nspDesc =
762         NamespaceDescriptor.create(nsp)
763             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
764     ADMIN.createNamespace(nspDesc);
765     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
766     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
767     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
768     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
769 
770     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
771     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
772 
773     String snapshot = "snapshot_testRestoreSnapshot";
774     ADMIN.snapshot(snapshot, tableName1);
775 
776     List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
777     Collections.sort(regions);
778 
779     ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
780     Thread.sleep(2000);
781     assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
782 
783     ADMIN.disableTable(tableName1);
784     ADMIN.restoreSnapshot(snapshot);
785 
786     assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
787 
788     ADMIN.enableTable(tableName1);
789     ADMIN.deleteSnapshot(snapshot);
790   }
791 
792   @Test(timeout = 180000)
793   public void testRestoreSnapshotQuotaExceed() throws Exception {
794     String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
795     NamespaceDescriptor nspDesc =
796         NamespaceDescriptor.create(nsp)
797             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
798     ADMIN.createNamespace(nspDesc);
799     NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
800     assertNotNull("Namespace descriptor found null.", ndesc);
801     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
802     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
803     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
804 
805     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
806     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
807 
808     String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
809     // snapshot has 4 regions
810     ADMIN.snapshot(snapshot, tableName1);
811     // recreate table with 1 region and set max regions to 3 for namespace
812     ADMIN.disableTable(tableName1);
813     ADMIN.deleteTable(tableName1);
814     ADMIN.createTable(tableDescOne);
815     ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
816     ADMIN.modifyNamespace(ndesc);
817 
818     ADMIN.disableTable(tableName1);
819     try {
820       ADMIN.restoreSnapshot(snapshot);
821       fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
822           + " wraps IOException into RestoreSnapshotException");
823     } catch (RestoreSnapshotException ignore) {
824       assertTrue(ignore.getCause() instanceof QuotaExceededException);
825     }
826     assertEquals(1, getNamespaceState(nsp).getRegionCount());
827     ADMIN.enableTable(tableName1);
828     ADMIN.deleteSnapshot(snapshot);
829   }
830 }