View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import java.io.IOException;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HBaseTestingUtility;
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.hbase.testclassification.LargeTests;
28  import org.apache.hadoop.hbase.TableName;
29  import org.apache.hadoop.hbase.client.Admin;
30  import org.apache.hadoop.hbase.client.Table;
31  import org.apache.hadoop.hbase.master.MasterFileSystem;
32  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
33  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
34  import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test clone/restore snapshots from the client
46   *
47   * TODO This is essentially a clone of TestRestoreSnapshotFromClient.  This is worth refactoring
48   * this because there will be a few more flavors of snapshots that need to run these tests.
49   */
50  @Category(LargeTests.class)
51  public class TestRestoreFlushSnapshotFromClient {
52    final Log LOG = LogFactory.getLog(getClass());
53  
54    protected final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
55  
56    protected final byte[] FAMILY = Bytes.toBytes("cf");
57  
58    protected byte[] snapshotName0;
59    protected byte[] snapshotName1;
60    protected byte[] snapshotName2;
61    protected int snapshot0Rows;
62    protected int snapshot1Rows;
63    protected TableName tableName;
64    protected Admin admin;
65  
66    @BeforeClass
67    public static void setupCluster() throws Exception {
68      setupConf(UTIL.getConfiguration());
69      UTIL.startMiniCluster(3);
70    }
71  
72    protected static void setupConf(Configuration conf) {
73      UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
74      UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
75      UTIL.getConfiguration().setInt("hbase.client.pause", 250);
76      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
77      UTIL.getConfiguration().setBoolean(
78          "hbase.master.enabletable.roundrobin", true);
79  
80      // Enable snapshot
81      UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
82      UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY,
83        RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2);
84    }
85  
86    @AfterClass
87    public static void tearDownAfterClass() throws Exception {
88      UTIL.shutdownMiniCluster();
89    }
90  
91    protected void createTable() throws Exception {
92      SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
93    }
94  
95    /**
96     * Initialize the tests with a table filled with some data
97     * and two snapshots (snapshotName0, snapshotName1) of different states.
98     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
99     */
100   @Before
101   public void setup() throws Exception {
102     this.admin = UTIL.getHBaseAdmin();
103 
104     long tid = System.currentTimeMillis();
105     tableName = TableName.valueOf("testtb-" + tid);
106     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
107     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
108     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
109 
110     // create Table and disable it
111     createTable();
112     SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY);
113     Table table = UTIL.getConnection().getTable(tableName);
114     snapshot0Rows = countRows(table);
115     LOG.info("=== before snapshot with 500 rows");
116     logFSTree();
117 
118     // take a snapshot
119     admin.snapshot(Bytes.toString(snapshotName0), tableName,
120         SnapshotDescription.Type.FLUSH);
121 
122     LOG.info("=== after snapshot with 500 rows");
123     logFSTree();
124 
125     // insert more data
126     SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY);
127     snapshot1Rows = countRows(table);
128     LOG.info("=== before snapshot with 1000 rows");
129     logFSTree();
130 
131     // take a snapshot of the updated table
132     admin.snapshot(Bytes.toString(snapshotName1), tableName,
133         SnapshotDescription.Type.FLUSH);
134     LOG.info("=== after snapshot with 1000 rows");
135     logFSTree();
136     table.close();
137   }
138 
139   @After
140   public void tearDown() throws Exception {
141     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
142     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
143   }
144 
145   @Test
146   public void testTakeFlushSnapshot() throws IOException {
147     // taking happens in setup.
148   }
149 
150   @Test
151   public void testRestoreSnapshot() throws IOException {
152     verifyRowCount(UTIL, tableName, snapshot1Rows);
153 
154     // Restore from snapshot-0
155     admin.disableTable(tableName);
156     admin.restoreSnapshot(snapshotName0);
157     logFSTree();
158     admin.enableTable(tableName);
159     LOG.info("=== after restore with 500 row snapshot");
160     logFSTree();
161     verifyRowCount(UTIL, tableName, snapshot0Rows);
162 
163     // Restore from snapshot-1
164     admin.disableTable(tableName);
165     admin.restoreSnapshot(snapshotName1);
166     admin.enableTable(tableName);
167     verifyRowCount(UTIL, tableName, snapshot1Rows);
168   }
169 
170   @Test(expected=SnapshotDoesNotExistException.class)
171   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
172     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
173     TableName tableName = TableName.valueOf("random-table-" + System.currentTimeMillis());
174     admin.cloneSnapshot(snapshotName, tableName);
175   }
176 
177   @Test
178   public void testCloneSnapshot() throws IOException, InterruptedException {
179     TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
180     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
181     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
182   }
183 
184   private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName,
185       int snapshotRows) throws IOException, InterruptedException {
186     // create a new table from snapshot
187     admin.cloneSnapshot(snapshotName, tableName);
188     verifyRowCount(UTIL, tableName, snapshotRows);
189 
190     UTIL.deleteTable(tableName);
191   }
192 
193   @Test
194   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
195     TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
196     admin.cloneSnapshot(snapshotName0, clonedTableName);
197     verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
198     admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH);
199     UTIL.deleteTable(clonedTableName);
200 
201     admin.cloneSnapshot(snapshotName2, clonedTableName);
202     verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
203     UTIL.deleteTable(clonedTableName);
204   }
205 
206   // ==========================================================================
207   //  Helpers
208   // ==========================================================================
209   private void logFSTree() throws IOException {
210     MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
211     FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
212   }
213 
214   protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
215       long expectedRows) throws IOException {
216     SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
217   }
218 
219   protected int countRows(final Table table, final byte[]... families) throws IOException {
220     return UTIL.countRows(table, families);
221   }
222 }