View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HTableDescriptor;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.testclassification.LargeTests;
35  import org.apache.hadoop.hbase.TableNotFoundException;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
39  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
40  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
41  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  import com.google.common.collect.Lists;
52  
53  /**
54   * Test create/using/deleting snapshots from the client
55   * <p>
56   * This is an end-to-end test for the snapshot utility
57   */
58  @Category(LargeTests.class)
59  public class TestSnapshotFromClient {
60    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
61  
62    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
63    protected static final int NUM_RS = 2;
64    protected static final String STRING_TABLE_NAME = "test";
65    protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
66    protected static final TableName TABLE_NAME =
67        TableName.valueOf(STRING_TABLE_NAME);
68  
69    /**
70     * Setup the config for the cluster
71     * @throws Exception on failure
72     */
73    @BeforeClass
74    public static void setupCluster() throws Exception {
75      setupConf(UTIL.getConfiguration());
76      UTIL.startMiniCluster(NUM_RS);
77    }
78  
79    protected static void setupConf(Configuration conf) {
80      // disable the ui
81      conf.setInt("hbase.regionsever.info.port", -1);
82      // change the flush size to a small amount, regulating number of store files
83      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
84      // so make sure we get a compaction when doing a load, but keep around some
85      // files in the store
86      conf.setInt("hbase.hstore.compaction.min", 10);
87      conf.setInt("hbase.hstore.compactionThreshold", 10);
88      // block writes if we get to 12 store files
89      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
90      // Enable snapshot
91      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
92      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93        ConstantSizeRegionSplitPolicy.class.getName());
94    }
95  
96    @Before
97    public void setup() throws Exception {
98      createTable();
99    }
100 
101   protected void createTable() throws Exception {
102     HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
103     htd.setRegionReplication(getNumReplicas());
104     UTIL.createTable(htd, new byte[][]{TEST_FAM}, UTIL.getConfiguration());
105   }
106 
107   protected int getNumReplicas() {
108     return 1;
109   }
110 
111   @After
112   public void tearDown() throws Exception {
113     UTIL.deleteTable(TABLE_NAME);
114     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
115     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
116   }
117 
118   @AfterClass
119   public static void cleanupTest() throws Exception {
120     try {
121       UTIL.shutdownMiniCluster();
122     } catch (Exception e) {
123       LOG.warn("failure shutting down cluster", e);
124     }
125   }
126 
127   /**
128    * Test snapshotting not allowed hbase:meta and -ROOT-
129    * @throws Exception
130    */
131   @Test (timeout=300000)
132   public void testMetaTablesSnapshot() throws Exception {
133     Admin admin = UTIL.getHBaseAdmin();
134     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
135 
136     try {
137       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
138       fail("taking a snapshot of hbase:meta should not be allowed");
139     } catch (IllegalArgumentException e) {
140       // expected
141     }
142   }
143 
144   /**
145    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
146    *
147    * @throws Exception
148    */
149   @Test (timeout=300000)
150   public void testSnapshotDeletionWithRegex() throws Exception {
151     Admin admin = UTIL.getHBaseAdmin();
152     // make sure we don't fail on listing snapshots
153     SnapshotTestingUtils.assertNoSnapshots(admin);
154 
155     // put some stuff in the table
156     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
157     UTIL.loadTable(table, TEST_FAM);
158     table.close();
159 
160     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
161     admin.snapshot(snapshot1, TABLE_NAME);
162     LOG.debug("Snapshot1 completed.");
163 
164     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
165     admin.snapshot(snapshot2, TABLE_NAME);
166     LOG.debug("Snapshot2 completed.");
167 
168     String snapshot3 = "3rdTableSnapshot";
169     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
170     LOG.debug(snapshot3 + " completed.");
171 
172     // delete the first two snapshots
173     admin.deleteSnapshots("TableSnapshot.*");
174     List<SnapshotDescription> snapshots = admin.listSnapshots();
175     assertEquals(1, snapshots.size());
176     assertEquals(snapshots.get(0).getName(), snapshot3);
177 
178     admin.deleteSnapshot(snapshot3);
179     admin.close();
180   }
181   /**
182    * Test snapshotting a table that is offline
183    * @throws Exception
184    */
185   @Test (timeout=300000)
186   public void testOfflineTableSnapshot() throws Exception {
187     Admin admin = UTIL.getHBaseAdmin();
188     // make sure we don't fail on listing snapshots
189     SnapshotTestingUtils.assertNoSnapshots(admin);
190 
191     // put some stuff in the table
192     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
193     UTIL.loadTable(table, TEST_FAM, false);
194 
195     LOG.debug("FS state before disable:");
196     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
197       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
198     // XXX if this is flakey, might want to consider using the async version and looping as
199     // disableTable can succeed and still timeout.
200     admin.disableTable(TABLE_NAME);
201 
202     LOG.debug("FS state before snapshot:");
203     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
204       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
205 
206     // take a snapshot of the disabled table
207     final String SNAPSHOT_NAME = "offlineTableSnapshot";
208     byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
209 
210     SnapshotDescription desc = SnapshotDescription.newBuilder()
211       .setType(SnapshotDescription.Type.DISABLED)
212       .setTable(STRING_TABLE_NAME)
213       .setName(SNAPSHOT_NAME)
214       .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
215       .build();
216     admin.snapshot(desc);
217     LOG.debug("Snapshot completed.");
218 
219     // make sure we have the snapshot
220     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
221       snapshot, TABLE_NAME);
222 
223     // make sure its a valid snapshot
224     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
225     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
226     LOG.debug("FS state after snapshot:");
227     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
228       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
229 
230     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
231       admin, fs);
232 
233     admin.deleteSnapshot(snapshot);
234     snapshots = admin.listSnapshots();
235     SnapshotTestingUtils.assertNoSnapshots(admin);
236   }
237 
238   @Test (timeout=300000)
239   public void testSnapshotFailsOnNonExistantTable() throws Exception {
240     Admin admin = UTIL.getHBaseAdmin();
241     // make sure we don't fail on listing snapshots
242     SnapshotTestingUtils.assertNoSnapshots(admin);
243     String tableName = "_not_a_table";
244 
245     // make sure the table doesn't exist
246     boolean fail = false;
247     do {
248     try {
249       admin.getTableDescriptor(TableName.valueOf(tableName));
250       fail = true;
251           LOG.error("Table:" + tableName + " already exists, checking a new name");
252       tableName = tableName+"!";
253     } catch (TableNotFoundException e) {
254       fail = false;
255       }
256     } while (fail);
257 
258     // snapshot the non-existant table
259     try {
260       admin.snapshot("fail", TableName.valueOf(tableName));
261       fail("Snapshot succeeded even though there is not table.");
262     } catch (SnapshotCreationException e) {
263       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
264     }
265   }
266 
267   @Test (timeout=300000)
268   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
269     // test with an empty table with one region
270 
271     Admin admin = UTIL.getHBaseAdmin();
272     // make sure we don't fail on listing snapshots
273     SnapshotTestingUtils.assertNoSnapshots(admin);
274 
275     LOG.debug("FS state before disable:");
276     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
277       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
278     admin.disableTable(TABLE_NAME);
279 
280     LOG.debug("FS state before snapshot:");
281     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
282       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
283 
284     // take a snapshot of the disabled table
285     byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
286     admin.snapshot(snapshot, TABLE_NAME);
287     LOG.debug("Snapshot completed.");
288 
289     // make sure we have the snapshot
290     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
291       snapshot, TABLE_NAME);
292 
293     // make sure its a valid snapshot
294     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
295     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
296     LOG.debug("FS state after snapshot:");
297     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
298       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
299 
300     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
301     List<byte[]> nonEmptyCfs = Lists.newArrayList();
302     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
303       rootDir, admin, fs);
304 
305     admin.deleteSnapshot(snapshot);
306     snapshots = admin.listSnapshots();
307     SnapshotTestingUtils.assertNoSnapshots(admin);
308   }
309 }