1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.concurrent.CountDownLatch;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.testclassification.LargeTests;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.TableNotFoundException;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Table;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
48 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.junit.After;
53 import org.junit.AfterClass;
54 import org.junit.Before;
55 import org.junit.BeforeClass;
56 import org.junit.Test;
57 import org.junit.experimental.categories.Category;
58
59
60
61
62
63
64
65
66
67 @Category(LargeTests.class)
68 public class TestFlushSnapshotFromClient {
69 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
70
71 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
72 protected static final int NUM_RS = 2;
73 protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
74 protected static final TableName TABLE_NAME = TableName.valueOf("test");
75 protected final int DEFAULT_NUM_ROWS = 100;
76
77 @BeforeClass
78 public static void setupCluster() throws Exception {
79 setupConf(UTIL.getConfiguration());
80 UTIL.startMiniCluster(NUM_RS);
81 }
82
83 protected static void setupConf(Configuration conf) {
84
85 conf.setInt("hbase.regionsever.info.port", -1);
86
87 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
88
89
90 conf.setInt("hbase.hstore.compaction.min", 10);
91 conf.setInt("hbase.hstore.compactionThreshold", 10);
92
93 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
94
95 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
96 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
97 ConstantSizeRegionSplitPolicy.class.getName());
98 }
99
100 @Before
101 public void setup() throws Exception {
102 createTable();
103 }
104
105 protected void createTable() throws Exception {
106 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
107 }
108
109 @After
110 public void tearDown() throws Exception {
111 UTIL.deleteTable(TABLE_NAME);
112
113 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
114 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
115 }
116
117 @AfterClass
118 public static void cleanupTest() throws Exception {
119 try {
120 UTIL.shutdownMiniCluster();
121 } catch (Exception e) {
122 LOG.warn("failure shutting down cluster", e);
123 }
124 }
125
126
127
128
129
130 @Test (timeout=300000)
131 public void testFlushTableSnapshot() throws Exception {
132 Admin admin = UTIL.getHBaseAdmin();
133
134 SnapshotTestingUtils.assertNoSnapshots(admin);
135
136
137 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
138
139 LOG.debug("FS state before snapshot:");
140 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
141 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
142
143
144 String snapshotString = "offlineTableSnapshot";
145 byte[] snapshot = Bytes.toBytes(snapshotString);
146 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
147 LOG.debug("Snapshot completed.");
148
149
150 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
151 snapshot, TABLE_NAME);
152
153
154 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
155 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
156 LOG.debug("FS state after snapshot:");
157 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
158 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
159
160 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
161 admin, fs);
162 }
163
164
165
166
167
168 @Test(timeout=30000)
169 public void testSkipFlushTableSnapshot() throws Exception {
170 Admin admin = UTIL.getHBaseAdmin();
171
172 SnapshotTestingUtils.assertNoSnapshots(admin);
173
174
175 try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
176 UTIL.loadTable(table, TEST_FAM);
177 }
178
179 LOG.debug("FS state before snapshot:");
180 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
181 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
182
183
184 String snapshotString = "skipFlushTableSnapshot";
185 byte[] snapshot = Bytes.toBytes(snapshotString);
186 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
187 LOG.debug("Snapshot completed.");
188
189
190 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
191 snapshot, TABLE_NAME);
192
193
194 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
195 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
196 LOG.debug("FS state after snapshot:");
197 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
198 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
199
200 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
201 admin, fs);
202
203 admin.deleteSnapshot(snapshot);
204 snapshots = admin.listSnapshots();
205 SnapshotTestingUtils.assertNoSnapshots(admin);
206 }
207
208
209
210
211
212
213 @Test (timeout=300000)
214 public void testFlushTableSnapshotWithProcedure() throws Exception {
215 Admin admin = UTIL.getHBaseAdmin();
216
217 SnapshotTestingUtils.assertNoSnapshots(admin);
218
219
220 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
221
222 LOG.debug("FS state before snapshot:");
223 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
224 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
225
226
227 String snapshotString = "offlineTableSnapshot";
228 byte[] snapshot = Bytes.toBytes(snapshotString);
229 Map<String, String> props = new HashMap<String, String>();
230 props.put("table", TABLE_NAME.getNameAsString());
231 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
232 snapshotString, props);
233
234
235 LOG.debug("Snapshot completed.");
236
237
238 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
239 snapshot, TABLE_NAME);
240
241
242 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
243 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
244 LOG.debug("FS state after snapshot:");
245 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
246 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
247
248 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
249 admin, fs);
250 }
251
252 @Test (timeout=300000)
253 public void testSnapshotFailsOnNonExistantTable() throws Exception {
254 Admin admin = UTIL.getHBaseAdmin();
255
256 SnapshotTestingUtils.assertNoSnapshots(admin);
257 TableName tableName = TableName.valueOf("_not_a_table");
258
259
260 boolean fail = false;
261 do {
262 try {
263 admin.getTableDescriptor(tableName);
264 fail = true;
265 LOG.error("Table:" + tableName + " already exists, checking a new name");
266 tableName = TableName.valueOf(tableName+"!");
267 } catch (TableNotFoundException e) {
268 fail = false;
269 }
270 } while (fail);
271
272
273 try {
274 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
275 fail("Snapshot succeeded even though there is not table.");
276 } catch (SnapshotCreationException e) {
277 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
278 }
279 }
280
281 @Test(timeout = 300000)
282 public void testAsyncFlushSnapshot() throws Exception {
283 Admin admin = UTIL.getHBaseAdmin();
284 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
285 .setTable(TABLE_NAME.getNameAsString())
286 .setType(SnapshotDescription.Type.FLUSH)
287 .build();
288
289
290 admin.takeSnapshotAsync(snapshot);
291
292
293 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
294 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
295 LOG.info(" === Async Snapshot Completed ===");
296 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
297 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
298
299 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
300 }
301
302 @Test (timeout=300000)
303 public void testSnapshotStateAfterMerge() throws Exception {
304 int numRows = DEFAULT_NUM_ROWS;
305 Admin admin = UTIL.getHBaseAdmin();
306
307 SnapshotTestingUtils.assertNoSnapshots(admin);
308
309 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
310
311
312 String snapshotBeforeMergeName = "snapshotBeforeMerge";
313 admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
314
315
316 TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
317 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
318 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
319
320
321 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
322 Collections.sort(regions, new Comparator<HRegionInfo>() {
323 public int compare(HRegionInfo r1, HRegionInfo r2) {
324 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
325 }
326 });
327
328 int numRegions = admin.getTableRegions(TABLE_NAME).size();
329 int numRegionsAfterMerge = numRegions - 2;
330 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
331 regions.get(2).getEncodedNameAsBytes(), true);
332 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
333 regions.get(6).getEncodedNameAsBytes(), true);
334
335
336 waitRegionsAfterMerge(numRegionsAfterMerge);
337 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
338
339
340 TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
341 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
342 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
343
344 verifyRowCount(UTIL, TABLE_NAME, numRows);
345 verifyRowCount(UTIL, cloneBeforeMergeName, numRows);
346 verifyRowCount(UTIL, cloneAfterMergeName, numRows);
347
348
349 UTIL.deleteTable(cloneAfterMergeName);
350 UTIL.deleteTable(cloneBeforeMergeName);
351 }
352
353 @Test (timeout=300000)
354 public void testTakeSnapshotAfterMerge() throws Exception {
355 int numRows = DEFAULT_NUM_ROWS;
356 Admin admin = UTIL.getHBaseAdmin();
357
358 SnapshotTestingUtils.assertNoSnapshots(admin);
359
360 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
361
362
363 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
364 Collections.sort(regions, new Comparator<HRegionInfo>() {
365 public int compare(HRegionInfo r1, HRegionInfo r2) {
366 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
367 }
368 });
369
370 int numRegions = admin.getTableRegions(TABLE_NAME).size();
371 int numRegionsAfterMerge = numRegions - 2;
372 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
373 regions.get(2).getEncodedNameAsBytes(), true);
374 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
375 regions.get(6).getEncodedNameAsBytes(), true);
376
377 waitRegionsAfterMerge(numRegionsAfterMerge);
378 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
379
380
381 String snapshotName = "snapshotAfterMerge";
382 SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
383 SnapshotDescription.Type.FLUSH, 3);
384
385
386 TableName cloneName = TableName.valueOf("cloneMerge");
387 admin.cloneSnapshot(snapshotName, cloneName);
388 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
389
390 verifyRowCount(UTIL, TABLE_NAME, numRows);
391 verifyRowCount(UTIL, cloneName, numRows);
392
393
394 UTIL.deleteTable(cloneName);
395 }
396
397
398
399
400 @Test (timeout=300000)
401 public void testFlushCreateListDestroy() throws Exception {
402 LOG.debug("------- Starting Snapshot test -------------");
403 Admin admin = UTIL.getHBaseAdmin();
404
405 SnapshotTestingUtils.assertNoSnapshots(admin);
406
407 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
408
409 String snapshotName = "flushSnapshotCreateListDestroy";
410 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
411 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
412 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
413 snapshotName, rootDir, fs, true);
414 }
415
416
417
418
419
420
421 @Test(timeout=300000)
422 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
423 final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2");
424
425 int ssNum = 20;
426 Admin admin = UTIL.getHBaseAdmin();
427
428 SnapshotTestingUtils.assertNoSnapshots(admin);
429
430 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
431
432 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
433 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
434
435 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
436
437 class SSRunnable implements Runnable {
438 SnapshotDescription ss;
439 SSRunnable(SnapshotDescription ss) {
440 this.ss = ss;
441 }
442
443 @Override
444 public void run() {
445 try {
446 Admin admin = UTIL.getHBaseAdmin();
447 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
448 admin.takeSnapshotAsync(ss);
449 } catch (Exception e) {
450 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
451 ss)
452 + ". This is ok, we expect some", e);
453 }
454 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
455 toBeSubmitted.countDown();
456 }
457 };
458
459
460 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
461 for (int i = 0; i < ssNum; i++) {
462 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
463 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
464 builder.setName("ss"+i);
465 builder.setType(SnapshotDescription.Type.FLUSH);
466 descs[i] = builder.build();
467 }
468
469
470 for (int i=0 ; i < ssNum; i++) {
471 new Thread(new SSRunnable(descs[i])).start();
472 }
473
474
475 toBeSubmitted.await();
476
477
478 while (true) {
479 int doneCount = 0;
480 for (SnapshotDescription ss : descs) {
481 try {
482 if (admin.isSnapshotFinished(ss)) {
483 doneCount++;
484 }
485 } catch (Exception e) {
486 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
487 doneCount++;
488 }
489 }
490 if (doneCount == descs.length) {
491 break;
492 }
493 Thread.sleep(100);
494 }
495
496
497 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
498
499 List<SnapshotDescription> taken = admin.listSnapshots();
500 int takenSize = taken.size();
501 LOG.info("Taken " + takenSize + " snapshots: " + taken);
502 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
503 " issued many requests", takenSize < ssNum && takenSize > 0);
504
505
506 int t1SnapshotsCount = 0;
507 int t2SnapshotsCount = 0;
508 for (SnapshotDescription ss : taken) {
509 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
510 t1SnapshotsCount++;
511 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
512 t2SnapshotsCount++;
513 }
514 }
515 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
516 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
517
518 UTIL.deleteTable(TABLE2_NAME);
519 }
520
521 private void logFSTree(Path root) throws IOException {
522 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
523 }
524
525 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
526 throws IOException, InterruptedException {
527 Admin admin = UTIL.getHBaseAdmin();
528
529 long startTime = System.currentTimeMillis();
530 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
531
532
533 if ((System.currentTimeMillis() - startTime) > 15000)
534 break;
535 Thread.sleep(100);
536 }
537 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
538 }
539
540
541 protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
542 long expectedRows) throws IOException {
543 SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
544 }
545
546 protected int countRows(final Table table, final byte[]... families) throws IOException {
547 return UTIL.countRows(table, families);
548 }
549 }