1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collections;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Set;
35 import java.util.TreeSet;
36
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
43 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.testclassification.MediumTests;
47 import org.junit.After;
48 import org.junit.AfterClass;
49 import org.junit.Before;
50 import org.junit.BeforeClass;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53
54
55
56
57
58 @Category(MediumTests.class)
59 public class TestBackupSystemTable {
60
61 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
62 protected static Configuration conf = UTIL.getConfiguration();
63 protected static MiniHBaseCluster cluster;
64 protected static Connection conn;
65 protected BackupSystemTable table;
66
67 @BeforeClass
68 public static void setUp() throws Exception {
69 conf.setBoolean(HConstants.BACKUP_ENABLE_KEY, true);
70 cluster = UTIL.startMiniCluster();
71 conn = UTIL.getConnection();
72 waitForSystemTable();
73 }
74
75 static void waitForSystemTable() throws Exception
76 {
77 try(Admin admin = UTIL.getHBaseAdmin();) {
78 while (!admin.tableExists(BackupSystemTable.getTableName())
79 || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
80 Thread.sleep(1000);
81 }
82 }
83 }
84
85 @Before
86 public void before() throws IOException {
87 table = new BackupSystemTable(conn);
88 }
89
90 @After
91 public void after() {
92 if (table != null) {
93 table.close();
94 }
95
96 }
97
98 @Test
99 public void testUpdateReadDeleteBackupStatus() throws IOException {
100 BackupInfo ctx = createBackupContext();
101 table.updateBackupInfo(ctx);
102 BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
103 assertTrue(compare(ctx, readCtx));
104
105 readCtx = table.readBackupInfo("fake");
106 assertNull(readCtx);
107
108 table.deleteBackupInfo(ctx.getBackupId());
109 readCtx = table.readBackupInfo(ctx.getBackupId());
110 assertNull(readCtx);
111 cleanBackupTable();
112 }
113
114 @Test
115 public void testWriteReadBackupStartCode() throws IOException {
116 Long code = 100L;
117 table.writeBackupStartCode(code, "root");
118 String readCode = table.readBackupStartCode("root");
119 assertEquals(code, new Long(Long.parseLong(readCode)));
120 cleanBackupTable();
121 }
122
123 private void cleanBackupTable() throws IOException {
124 Admin admin = UTIL.getHBaseAdmin();
125 admin.disableTable(TableName.BACKUP_TABLE_NAME);
126 admin.truncateTable(TableName.BACKUP_TABLE_NAME, true);
127 if (admin.isTableDisabled(TableName.BACKUP_TABLE_NAME)) {
128 admin.enableTable(TableName.BACKUP_TABLE_NAME);
129 }
130 }
131
132 @Test
133 public void testBackupHistory() throws IOException {
134 int n = 10;
135 List<BackupInfo> list = createBackupContextList(n);
136
137
138 for (BackupInfo bc : list) {
139
140 bc.setState(BackupState.COMPLETE);
141 table.updateBackupInfo(bc);
142 }
143
144
145 Collections.reverse(list);
146 ArrayList<BackupInfo> history = table.getBackupHistory();
147 assertTrue(history.size() == n);
148
149 for (int i = 0; i < n; i++) {
150 BackupInfo ctx = list.get(i);
151 BackupInfo data = history.get(i);
152 assertTrue(compare(ctx, data));
153 }
154
155 cleanBackupTable();
156
157 }
158
159 @Test
160 public void testBackupDelete() throws IOException {
161
162 try (BackupSystemTable table = new BackupSystemTable(conn)) {
163
164 int n = 10;
165 List<BackupInfo> list = createBackupContextList(n);
166
167
168 for (BackupInfo bc : list) {
169
170 bc.setState(BackupState.COMPLETE);
171 table.updateBackupInfo(bc);
172 }
173
174
175 for (BackupInfo bc : list) {
176 assertNotNull(table.readBackupInfo(bc.getBackupId()));
177 }
178
179
180 for (BackupInfo bc : list) {
181 table.deleteBackupInfo(bc.getBackupId());
182 }
183
184
185 for (BackupInfo bc : list) {
186 assertNull(table.readBackupInfo(bc.getBackupId()));
187 }
188
189 cleanBackupTable();
190 }
191
192 }
193
194
195
196 @Test
197 public void testRegionServerLastLogRollResults() throws IOException {
198 String[] servers = new String[] { "server1", "server2", "server3" };
199 Long[] timestamps = new Long[] { 100L, 102L, 107L };
200
201 for (int i = 0; i < servers.length; i++) {
202 table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
203 }
204
205 HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
206 assertTrue(servers.length == result.size());
207 Set<String> keys = result.keySet();
208 String[] keysAsArray = new String[keys.size()];
209 keys.toArray(keysAsArray);
210 Arrays.sort(keysAsArray);
211
212 for (int i = 0; i < keysAsArray.length; i++) {
213 assertEquals(keysAsArray[i], servers[i]);
214 Long ts1 = timestamps[i];
215 Long ts2 = result.get(keysAsArray[i]);
216 assertEquals(ts1, ts2);
217 }
218
219 cleanBackupTable();
220 }
221
222 @Test
223 public void testIncrementalBackupTableSet() throws IOException {
224 TreeSet<TableName> tables1 = new TreeSet<>();
225
226 tables1.add(TableName.valueOf("t1"));
227 tables1.add(TableName.valueOf("t2"));
228 tables1.add(TableName.valueOf("t3"));
229
230 TreeSet<TableName> tables2 = new TreeSet<>();
231
232 tables2.add(TableName.valueOf("t3"));
233 tables2.add(TableName.valueOf("t4"));
234 tables2.add(TableName.valueOf("t5"));
235
236 table.addIncrementalBackupTableSet(tables1, "root");
237 BackupSystemTable table = new BackupSystemTable(conn);
238 TreeSet<TableName> res1 = (TreeSet<TableName>)
239 table.getIncrementalBackupTableSet("root");
240 assertTrue(tables1.size() == res1.size());
241 Iterator<TableName> desc1 = tables1.descendingIterator();
242 Iterator<TableName> desc2 = res1.descendingIterator();
243 while (desc1.hasNext()) {
244 assertEquals(desc1.next(), desc2.next());
245 }
246
247 table.addIncrementalBackupTableSet(tables2, "root");
248 TreeSet<TableName> res2 = (TreeSet<TableName>)
249 table.getIncrementalBackupTableSet("root");
250 assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
251
252 tables1.addAll(tables2);
253
254 desc1 = tables1.descendingIterator();
255 desc2 = res2.descendingIterator();
256
257 while (desc1.hasNext()) {
258 assertEquals(desc1.next(), desc2.next());
259 }
260 cleanBackupTable();
261
262 }
263
264 @Test
265 public void testRegionServerLogTimestampMap() throws IOException {
266 TreeSet<TableName> tables = new TreeSet<>();
267
268 tables.add(TableName.valueOf("t1"));
269 tables.add(TableName.valueOf("t2"));
270 tables.add(TableName.valueOf("t3"));
271
272 HashMap<String, Long> rsTimestampMap = new HashMap<String, Long>();
273
274 rsTimestampMap.put("rs1", 100L);
275 rsTimestampMap.put("rs2", 101L);
276 rsTimestampMap.put("rs3", 103L);
277
278 table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
279
280 HashMap<TableName, HashMap<String, Long>> result = table.readLogTimestampMap("root");
281
282 assertTrue(tables.size() == result.size());
283
284 for (TableName t : tables) {
285 HashMap<String, Long> rstm = result.get(t);
286 assertNotNull(rstm);
287 assertEquals(rstm.get("rs1"), new Long(100L));
288 assertEquals(rstm.get("rs2"), new Long(101L));
289 assertEquals(rstm.get("rs3"), new Long(103L));
290 }
291
292 Set<TableName> tables1 = new TreeSet<>();
293
294 tables1.add(TableName.valueOf("t3"));
295 tables1.add(TableName.valueOf("t4"));
296 tables1.add(TableName.valueOf("t5"));
297
298 HashMap<String, Long> rsTimestampMap1 = new HashMap<String, Long>();
299
300 rsTimestampMap1.put("rs1", 200L);
301 rsTimestampMap1.put("rs2", 201L);
302 rsTimestampMap1.put("rs3", 203L);
303
304 table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
305
306 result = table.readLogTimestampMap("root");
307
308 assertTrue(5 == result.size());
309
310 for (TableName t : tables) {
311 HashMap<String, Long> rstm = result.get(t);
312 assertNotNull(rstm);
313 if (t.equals(TableName.valueOf("t3")) == false) {
314 assertEquals(rstm.get("rs1"), new Long(100L));
315 assertEquals(rstm.get("rs2"), new Long(101L));
316 assertEquals(rstm.get("rs3"), new Long(103L));
317 } else {
318 assertEquals(rstm.get("rs1"), new Long(200L));
319 assertEquals(rstm.get("rs2"), new Long(201L));
320 assertEquals(rstm.get("rs3"), new Long(203L));
321 }
322 }
323
324 for (TableName t : tables1) {
325 HashMap<String, Long> rstm = result.get(t);
326 assertNotNull(rstm);
327 assertEquals(rstm.get("rs1"), new Long(200L));
328 assertEquals(rstm.get("rs2"), new Long(201L));
329 assertEquals(rstm.get("rs3"), new Long(203L));
330 }
331
332 cleanBackupTable();
333
334 }
335
336 @Test
337 public void testAddWALFiles() throws IOException {
338 List<String> files =
339 Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
340 "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
341 "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
342 String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
343
344 table.addWALFiles(files, "backup", "root");
345
346 assertTrue(table.isWALFileDeletable(files.get(0)));
347 assertTrue(table.isWALFileDeletable(files.get(1)));
348 assertTrue(table.isWALFileDeletable(files.get(2)));
349 assertFalse(table.isWALFileDeletable(newFile));
350
351 cleanBackupTable();
352 }
353
354
355
356
357
358
359 @Test
360 public void testBackupSetAddNotExists() throws IOException {
361 try (BackupSystemTable table = new BackupSystemTable(conn)) {
362
363 String[] tables = new String[] { "table1", "table2", "table3" };
364 String setName = "name";
365 table.addToBackupSet(setName, tables);
366 List<TableName> tnames = table.describeBackupSet(setName);
367 assertTrue(tnames != null);
368 assertTrue(tnames.size() == tables.length);
369 for (int i = 0; i < tnames.size(); i++) {
370 assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
371 }
372 cleanBackupTable();
373 }
374
375 }
376
377 @Test
378 public void testBackupSetAddExists() throws IOException {
379 try (BackupSystemTable table = new BackupSystemTable(conn)) {
380
381 String[] tables = new String[] { "table1", "table2", "table3" };
382 String setName = "name";
383 table.addToBackupSet(setName, tables);
384 String[] addTables = new String[] { "table4", "table5", "table6" };
385 table.addToBackupSet(setName, addTables);
386
387 List<TableName> tnames = table.describeBackupSet(setName);
388 assertTrue(tnames != null);
389 assertTrue(tnames.size() == tables.length + addTables.length);
390 for (int i = 0; i < tnames.size(); i++) {
391 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
392 }
393 cleanBackupTable();
394 }
395 }
396
397 @Test
398 public void testBackupSetAddExistsIntersects() throws IOException {
399 try (BackupSystemTable table = new BackupSystemTable(conn)) {
400
401 String[] tables = new String[] { "table1", "table2", "table3" };
402 String setName = "name";
403 table.addToBackupSet(setName, tables);
404 String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
405 table.addToBackupSet(setName, addTables);
406
407 List<TableName> tnames = table.describeBackupSet(setName);
408 assertTrue(tnames != null);
409 assertTrue(tnames.size()== tables.length + addTables.length - 1);
410 for (int i = 0; i < tnames.size(); i++) {
411 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
412 }
413 cleanBackupTable();
414 }
415 }
416
417 @Test
418 public void testBackupSetRemoveSomeNotExists() throws IOException {
419 try (BackupSystemTable table = new BackupSystemTable(conn)) {
420
421 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
422 String setName = "name";
423 table.addToBackupSet(setName, tables);
424 String[] removeTables = new String[] { "table4", "table5", "table6" };
425 table.removeFromBackupSet(setName, removeTables);
426
427 List<TableName> tnames = table.describeBackupSet(setName);
428 assertTrue(tnames != null);
429 assertTrue(tnames.size() == tables.length - 1);
430 for (int i = 0; i < tnames.size(); i++) {
431 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
432 }
433 cleanBackupTable();
434 }
435 }
436
437 @Test
438 public void testBackupSetRemove() throws IOException {
439 try (BackupSystemTable table = new BackupSystemTable(conn)) {
440
441 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
442 String setName = "name";
443 table.addToBackupSet(setName, tables);
444 String[] removeTables = new String[] { "table4", "table3" };
445 table.removeFromBackupSet(setName, removeTables);
446
447 List<TableName> tnames = table.describeBackupSet(setName);
448 assertTrue(tnames != null);
449 assertTrue(tnames.size() == tables.length - 2);
450 for (int i = 0; i < tnames.size(); i++) {
451 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
452 }
453 cleanBackupTable();
454 }
455 }
456
457 @Test
458 public void testBackupSetDelete() throws IOException {
459 try (BackupSystemTable table = new BackupSystemTable(conn)) {
460
461 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
462 String setName = "name";
463 table.addToBackupSet(setName, tables);
464 table.deleteBackupSet(setName);
465
466 List<TableName> tnames = table.describeBackupSet(setName);
467 assertTrue(tnames == null);
468 cleanBackupTable();
469 }
470 }
471
472 @Test
473 public void testBackupSetList() throws IOException {
474 try (BackupSystemTable table = new BackupSystemTable(conn)) {
475
476 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
477 String setName1 = "name1";
478 String setName2 = "name2";
479 table.addToBackupSet(setName1, tables);
480 table.addToBackupSet(setName2, tables);
481
482 List<String> list = table.listBackupSets();
483
484 assertTrue(list.size() == 2);
485 assertTrue(list.get(0).equals(setName1));
486 assertTrue(list.get(1).equals(setName2));
487
488 cleanBackupTable();
489 }
490 }
491
492
493 private boolean compare(BackupInfo one, BackupInfo two) {
494 return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
495 && one.getTargetRootDir().equals(two.getTargetRootDir())
496 && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
497 }
498
499 private BackupInfo createBackupContext() {
500
501 BackupInfo ctxt =
502 new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
503 new TableName[] {
504 TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
505 "/hbase/backup");
506 ctxt.setStartTs(System.currentTimeMillis());
507 ctxt.setEndTs(System.currentTimeMillis() + 1);
508 return ctxt;
509 }
510
511 private List<BackupInfo> createBackupContextList(int size) {
512 List<BackupInfo> list = new ArrayList<BackupInfo>();
513 for (int i = 0; i < size; i++) {
514 list.add(createBackupContext());
515 try {
516 Thread.sleep(10);
517 } catch (InterruptedException e) {
518 e.printStackTrace();
519 }
520 }
521 return list;
522 }
523
524 @AfterClass
525 public static void tearDown() throws IOException {
526 if (cluster != null) cluster.shutdown();
527 }
528 }