1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
35 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
36 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
37 import org.apache.hadoop.hbase.testclassification.LargeTests;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.Threads;
40 import org.junit.After;
41 import org.junit.AfterClass;
42 import org.junit.Assert;
43 import org.junit.Assume;
44 import org.junit.Before;
45 import org.junit.BeforeClass;
46 import org.junit.Ignore;
47 import org.junit.Rule;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50 import org.junit.rules.TestName;
51 import org.junit.rules.Timeout;
52
53
54
55
56 @Category(LargeTests.class)
57 public class TestSnapshotCloneIndependence {
58 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
59
60
61 private static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
62
63 @Rule
64 public Timeout globalTimeout = Timeout.seconds(60);
65
66 @Rule
67 public TestName testName = new TestName();
68 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
69
70 protected static final int NUM_RS = 2;
71 private static final String TEST_FAM_STR = "fam";
72 protected static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
73 private static final int CLEANER_INTERVAL = 10;
74
75 private FileSystem fs;
76 private Path rootDir;
77 private Admin admin;
78 private TableName originalTableName;
79 private Table originalTable;
80 private TableName cloneTableName;
81 private int countOriginalTable;
82 String snapshotNameAsString;
83 byte[] snapshotName;
84
85
86
87
88
89 @BeforeClass
90 public static void setupCluster() throws Exception {
91 setupConf(UTIL.getConfiguration());
92 UTIL.startMiniCluster(NUM_RS);
93 }
94
95 protected static void setupConf(Configuration conf) {
96
97 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
98
99 conf.setInt("hbase.regionsever.info.port", -1);
100
101 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
102
103
104 conf.setInt("hbase.hstore.compaction.min", 10);
105 conf.setInt("hbase.hstore.compactionThreshold", 10);
106
107 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
108 conf.setInt("hbase.regionserver.msginterval", 100);
109 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
110
111 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
112 ConstantSizeRegionSplitPolicy.class.getName());
113
114 conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
115 conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
116
117
118
119 conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
120 }
121
122 @Before
123 public void setup() throws Exception {
124 fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
125 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
126
127 admin = UTIL.getHBaseAdmin();
128 originalTableName = TableName.valueOf("test" + testName.getMethodName());
129 cloneTableName = TableName.valueOf("test-clone-" + originalTableName);
130 snapshotNameAsString = "snapshot_" + originalTableName;
131 snapshotName = Bytes.toBytes(snapshotNameAsString);
132
133 originalTable = createTable(originalTableName, TEST_FAM);
134 loadData(originalTable, TEST_FAM);
135 countOriginalTable = countRows(originalTable);
136 LOG.debug("Original table has: " + countOriginalTable + " rows");
137 }
138
139 @After
140 public void tearDown() throws Exception {
141 UTIL.deleteTable(originalTableName);
142 UTIL.deleteTable(cloneTableName);
143 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
144 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
145 }
146
147 @AfterClass
148 public static void cleanupTest() throws Exception {
149 try {
150 UTIL.shutdownMiniCluster();
151 } catch (Exception e) {
152 LOG.warn("failure shutting down cluster", e);
153 }
154 }
155
156
157
158
159
160 @Test (timeout=300000)
161 public void testOnlineSnapshotAppendIndependent() throws Exception {
162 createAndCloneSnapshot(true);
163 runTestSnapshotAppendIndependent();
164 }
165
166
167
168
169
170 @Test (timeout=300000)
171 public void testOfflineSnapshotAppendIndependent() throws Exception {
172 createAndCloneSnapshot(false);
173 runTestSnapshotAppendIndependent();
174 }
175
176
177
178
179
180 @Test (timeout=300000)
181 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
182 createAndCloneSnapshot(true);
183 runTestSnapshotMetadataChangesIndependent();
184 }
185
186
187
188
189
190 @Test (timeout=300000)
191 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
192 createAndCloneSnapshot(false);
193 runTestSnapshotMetadataChangesIndependent();
194 }
195
196
197
198
199
200 @Test (timeout=300000)
201 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
202 createAndCloneSnapshot(false);
203 runTestRegionOperationsIndependent();
204 }
205
206
207
208
209
210 @Test (timeout=300000)
211 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
212 createAndCloneSnapshot(true);
213 runTestRegionOperationsIndependent();
214 }
215
216 @Test (timeout=300000)
217 public void testOfflineSnapshotDeleteIndependent() throws Exception {
218
219
220 Assume.assumeTrue(!WINDOWS);
221
222 createAndCloneSnapshot(false);
223 runTestSnapshotDeleteIndependent();
224 }
225
226
227 @Ignore
228 public void testOnlineSnapshotDeleteIndependent() throws Exception {
229 createAndCloneSnapshot(true);
230 runTestSnapshotDeleteIndependent();
231 }
232
233 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
234 for (int i = 0; i < 200; i++) {
235 Threads.sleepWithoutInterrupt(500);
236 if (t.getAllRegionLocations().size() > originalCount) {
237 return;
238 }
239 }
240 throw new Exception("Split did not increase the number of regions");
241 }
242
243
244
245
246
247
248
249 private void createAndCloneSnapshot(boolean online) throws Exception {
250 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, TEST_FAM_STR,
251 snapshotNameAsString, rootDir, fs, online);
252
253
254 if (!online) {
255 admin.enableTable(originalTableName);
256 UTIL.waitTableAvailable(originalTableName);
257 }
258
259 admin.cloneSnapshot(snapshotName, cloneTableName);
260 UTIL.waitUntilAllRegionsAssigned(cloneTableName);
261 }
262
263
264
265
266 private void runTestSnapshotAppendIndependent() throws Exception {
267 try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
268 final int clonedTableRowCount = countRows(clonedTable);
269
270 Assert.assertEquals(
271 "The line counts of original and cloned tables do not match after clone. ",
272 countOriginalTable, clonedTableRowCount);
273
274
275 Put p = new Put(Bytes.toBytes("new-row-" + System.currentTimeMillis()));
276 p.addColumn(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
277 originalTable.put(p);
278
279
280 Assert.assertEquals("The row count of the original table was not modified by the put",
281 countOriginalTable + 1, countRows(originalTable));
282 Assert.assertEquals(
283 "The row count of the cloned table changed as a result of addition to the original",
284 clonedTableRowCount, countRows(clonedTable));
285
286 Put p2 = new Put(Bytes.toBytes("new-row-" + System.currentTimeMillis()));
287 p2.addColumn(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
288 clonedTable.put(p2);
289
290
291 Assert.assertEquals(
292 "The row count of the original table was modified by the put to the clone",
293 countOriginalTable + 1, countRows(originalTable));
294 Assert.assertEquals("The row count of the cloned table was not modified by the put",
295 clonedTableRowCount + 1, countRows(clonedTable));
296 }
297 }
298
299
300
301
302 private void runTestRegionOperationsIndependent() throws Exception {
303
304 ((ClusterConnection) UTIL.getConnection()).clearRegionCache();
305 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(originalTableName);
306
307 final int originalRegionCount = originalTableHRegions.size();
308 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
309 Assert.assertEquals(
310 "The number of regions in the cloned table is different than in the original table.",
311 originalRegionCount, cloneTableRegionCount);
312
313
314 admin.splitRegion(originalTableHRegions.get(0).getRegionName());
315 waitOnSplit(
316 (HTable)originalTable, originalRegionCount);
317
318
319 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
320 Assert.assertEquals(
321 "The number of regions in the cloned table changed though none of its regions were split.",
322 cloneTableRegionCount, cloneTableRegionCount2);
323 }
324
325
326
327
328 private void runTestSnapshotMetadataChangesIndependent() throws Exception {
329
330 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
331 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
332
333 admin.disableTable(originalTableName);
334 admin.addColumn(originalTableName, hcd);
335
336
337 admin.enableTable(originalTableName);
338 UTIL.waitTableAvailable(originalTableName);
339
340
341
342
343 HTableDescriptor originalTableDescriptor = originalTable.getTableDescriptor();
344 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
345
346 Assert.assertTrue("The original family was not found. There is something wrong. ",
347 originalTableDescriptor.hasFamily(TEST_FAM));
348 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
349 clonedTableDescriptor.hasFamily(TEST_FAM));
350
351 Assert.assertTrue("The new family was not found. ",
352 originalTableDescriptor.hasFamily(TEST_FAM_2));
353 Assert.assertTrue("The new family was not found. ",
354 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
355 }
356
357
358
359
360 private void runTestSnapshotDeleteIndependent() throws Exception {
361
362 admin.majorCompact(originalTableName);
363
364
365 admin.deleteSnapshot(snapshotName);
366
367
368 do {
369 Thread.sleep(5000);
370 } while (!admin.listSnapshots(snapshotNameAsString).isEmpty());
371
372 try (Table original = UTIL.getConnection().getTable(originalTableName)) {
373 try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
374
375 final int origTableRowCount = countRows(original);
376 final int clonedTableRowCount = countRows(clonedTable);
377 Assert.assertEquals(origTableRowCount, clonedTableRowCount);
378 }
379 }
380 }
381
382 protected Table createTable(final TableName table, byte[] family) throws Exception {
383 return UTIL.createTable(table, family);
384 }
385
386 protected void loadData(final Table table, byte[]... families) throws Exception {
387 UTIL.loadTable(table, families);
388 }
389
390 protected int countRows(final Table table, final byte[]... families) throws Exception {
391 return UTIL.countRows(table, families);
392 }
393 }