View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.testclassification.LargeTests;
29  import org.apache.hadoop.hbase.NamespaceDescriptor;
30  import org.apache.hadoop.hbase.NamespaceNotFoundException;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
33  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
34  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.junit.After;
37  import org.junit.AfterClass;
38  import org.junit.Before;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  /**
44   * Test clone snapshots from the client
45   */
46  @Category(LargeTests.class)
47  public class TestCloneSnapshotFromClient {
48    final Log LOG = LogFactory.getLog(getClass());
49  
50    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
51  
52    protected final byte[] FAMILY = Bytes.toBytes("cf");
53  
54    protected byte[] emptySnapshot;
55    protected byte[] snapshotName0;
56    protected byte[] snapshotName1;
57    protected byte[] snapshotName2;
58    protected TableName tableName;
59    protected int snapshot0Rows;
60    protected int snapshot1Rows;
61    protected Admin admin;
62  
63    protected static void setupConfiguration() {
64      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
65      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
66      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
67      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
68      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
69      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
70      TEST_UTIL.getConfiguration().setBoolean(
71          "hbase.master.enabletable.roundrobin", true);
72      TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40);
73    }
74  
75    @BeforeClass
76    public static void setUpBeforeClass() throws Exception {
77      setupConfiguration();
78      TEST_UTIL.startMiniCluster(3);
79    }
80  
81    @AfterClass
82    public static void tearDownAfterClass() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    /**
87     * Initialize the tests with a table filled with some data
88     * and two snapshots (snapshotName0, snapshotName1) of different states.
89     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
90     */
91    @Before
92    public void setup() throws Exception {
93      this.admin = TEST_UTIL.getHBaseAdmin();
94  
95      long tid = System.currentTimeMillis();
96      tableName = TableName.valueOf("testtb-" + tid);
97      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
98      snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
99      snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
100     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
101 
102     createTableAndSnapshots();
103   }
104 
105   protected void createTableAndSnapshots() throws Exception {
106     // create Table and disable it
107     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
108     admin.disableTable(tableName);
109 
110     // take an empty snapshot
111     admin.snapshot(emptySnapshot, tableName);
112 
113     // enable table and insert data
114     admin.enableTable(tableName);
115     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
116     try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
117       snapshot0Rows = TEST_UTIL.countRows(table);
118     }
119     admin.disableTable(tableName);
120 
121     // take a snapshot
122     admin.snapshot(snapshotName0, tableName);
123 
124     // enable table and insert more data
125     admin.enableTable(tableName);
126     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
127     try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
128       snapshot1Rows = TEST_UTIL.countRows(table);
129     }
130     admin.disableTable(tableName);
131 
132     // take a snapshot of the updated table
133     admin.snapshot(snapshotName1, tableName);
134 
135     // re-enable table
136     admin.enableTable(tableName);
137   }
138 
139   protected int getNumReplicas() {
140     return 1;
141   }
142 
143   @After
144   public void tearDown() throws Exception {
145     if (admin.tableExists(tableName)) {
146       TEST_UTIL.deleteTable(tableName);
147     }
148     SnapshotTestingUtils.deleteAllSnapshots(admin);
149     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
150   }
151 
152   @Test(expected=SnapshotDoesNotExistException.class)
153   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
154     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
155     TableName tableName = TableName.valueOf("random-table-" + System.currentTimeMillis());
156     admin.cloneSnapshot(snapshotName, tableName);
157   }
158 
159   @Test(expected = NamespaceNotFoundException.class)
160   public void testCloneOnMissingNamespace() throws IOException, InterruptedException {
161     TableName clonedTableName = TableName.valueOf("unknownNS:clonetb");
162     admin.cloneSnapshot(snapshotName1, clonedTableName);
163   }
164 
165   @Test
166   public void testCloneSnapshot() throws IOException, InterruptedException {
167     TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
168     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
169     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
170     testCloneSnapshot(clonedTableName, emptySnapshot, 0);
171   }
172 
173   private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName,
174       int snapshotRows) throws IOException, InterruptedException {
175     // create a new table from snapshot
176     admin.cloneSnapshot(snapshotName, tableName);
177     verifyRowCount(TEST_UTIL, tableName, snapshotRows);
178 
179     verifyReplicasCameOnline(tableName);
180     TEST_UTIL.deleteTable(tableName);
181   }
182 
183   protected void verifyReplicasCameOnline(TableName tableName) throws IOException {
184     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
185   }
186 
187   @Test
188   public void testCloneSnapshotCrossNamespace() throws IOException, InterruptedException {
189     String nsName = "testCloneSnapshotCrossNamespace";
190     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
191     TableName clonedTableName =
192         TableName.valueOf(nsName, "clonedtb-" + System.currentTimeMillis());
193     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
194     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
195     testCloneSnapshot(clonedTableName, emptySnapshot, 0);
196   }
197 
198   /**
199    * Verify that tables created from the snapshot are still alive after source table deletion.
200    */
201   @Test
202   public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
203     // Clone a table from the first snapshot
204     TableName clonedTableName = TableName.valueOf("clonedtb1-" + System.currentTimeMillis());
205     admin.cloneSnapshot(snapshotName0, clonedTableName);
206     verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
207 
208     // Take a snapshot of this cloned table.
209     admin.disableTable(clonedTableName);
210     admin.snapshot(snapshotName2, clonedTableName);
211 
212     // Clone the snapshot of the cloned table
213     TableName clonedTableName2 = TableName.valueOf("clonedtb2-" + System.currentTimeMillis());
214     admin.cloneSnapshot(snapshotName2, clonedTableName2);
215     verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
216     admin.disableTable(clonedTableName2);
217 
218     // Remove the original table
219     TEST_UTIL.deleteTable(tableName);
220     waitCleanerRun();
221 
222     // Verify the first cloned table
223     admin.enableTable(clonedTableName);
224     verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
225 
226     // Verify the second cloned table
227     admin.enableTable(clonedTableName2);
228     verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
229     admin.disableTable(clonedTableName2);
230 
231     // Delete the first cloned table
232     TEST_UTIL.deleteTable(clonedTableName);
233     waitCleanerRun();
234 
235     // Verify the second cloned table
236     admin.enableTable(clonedTableName2);
237     verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
238 
239     // Clone a new table from cloned
240     TableName clonedTableName3 = TableName.valueOf("clonedtb3-" + System.currentTimeMillis());
241     admin.cloneSnapshot(snapshotName2, clonedTableName3);
242     verifyRowCount(TEST_UTIL, clonedTableName3, snapshot0Rows);
243 
244     // Delete the cloned tables
245     TEST_UTIL.deleteTable(clonedTableName2);
246     TEST_UTIL.deleteTable(clonedTableName3);
247     admin.deleteSnapshot(snapshotName2);
248   }
249 
250   // ==========================================================================
251   //  Helpers
252   // ==========================================================================
253 
254   private void waitCleanerRun() throws InterruptedException {
255     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
256   }
257 
258   protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
259       long expectedRows) throws IOException {
260     SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
261   }
262 }