View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.backup;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collections;
31  import java.util.HashMap;
32  import java.util.Iterator;
33  import java.util.List;
34  import java.util.Set;
35  import java.util.TreeSet;
36  
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.MiniHBaseCluster;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
43  import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.testclassification.MediumTests;
47  import org.junit.After;
48  import org.junit.AfterClass;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  import org.junit.experimental.categories.Category;
53  
54  /**
55   * Test cases for hbase:backup API
56   *
57   */
58  @Category(MediumTests.class)
59  public class TestBackupSystemTable {
60  
61    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
62    protected static Configuration conf = UTIL.getConfiguration();
63    protected static MiniHBaseCluster cluster;
64    protected static Connection conn;
65    protected BackupSystemTable table;
66  
67    @BeforeClass
68    public static void setUp() throws Exception {
69      conf.setBoolean(HConstants.BACKUP_ENABLE_KEY, true);
70      cluster = UTIL.startMiniCluster(); 
71      conn = UTIL.getConnection();
72      waitForSystemTable();
73    }
74    
75    static void waitForSystemTable() throws Exception
76    {
77      try(Admin admin = UTIL.getHBaseAdmin();) {
78        while (!admin.tableExists(BackupSystemTable.getTableName()) 
79            || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
80          Thread.sleep(1000);
81        }      
82      }
83    }
84    
85    @Before
86    public void before() throws IOException {
87      table = new BackupSystemTable(conn);
88    }
89  
90    @After
91    public void after() {
92      if (table != null) {
93        table.close();
94      }
95  
96    }
97  
98    @Test
99    public void testUpdateReadDeleteBackupStatus() throws IOException {
100     BackupInfo ctx = createBackupContext();
101     table.updateBackupInfo(ctx);
102     BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
103     assertTrue(compare(ctx, readCtx));
104     // try fake backup id
105     readCtx = table.readBackupInfo("fake");
106     assertNull(readCtx);
107     // delete backup context
108     table.deleteBackupInfo(ctx.getBackupId());
109     readCtx = table.readBackupInfo(ctx.getBackupId());
110     assertNull(readCtx);
111     cleanBackupTable();
112   }
113 
114   @Test
115   public void testWriteReadBackupStartCode() throws IOException {
116     Long code = 100L;
117     table.writeBackupStartCode(code, "root");
118     String readCode = table.readBackupStartCode("root");
119     assertEquals(code, new Long(Long.parseLong(readCode)));
120     cleanBackupTable();
121   }
122 
123   private void cleanBackupTable() throws IOException {
124     Admin admin = UTIL.getHBaseAdmin();
125     admin.disableTable(TableName.BACKUP_TABLE_NAME);
126     admin.truncateTable(TableName.BACKUP_TABLE_NAME, true);
127     if (admin.isTableDisabled(TableName.BACKUP_TABLE_NAME)) {
128       admin.enableTable(TableName.BACKUP_TABLE_NAME);
129     }
130   }
131 
132   @Test
133   public void testBackupHistory() throws IOException {
134     int n = 10;
135     List<BackupInfo> list = createBackupContextList(n);
136 
137     // Load data
138     for (BackupInfo bc : list) {
139       // Make sure we set right status
140       bc.setState(BackupState.COMPLETE);
141       table.updateBackupInfo(bc);
142     }
143 
144     // Reverse list for comparison
145     Collections.reverse(list);
146     ArrayList<BackupInfo> history = table.getBackupHistory();
147     assertTrue(history.size() == n);
148 
149     for (int i = 0; i < n; i++) {
150       BackupInfo ctx = list.get(i);
151       BackupInfo data = history.get(i);
152       assertTrue(compare(ctx, data));
153     }
154 
155     cleanBackupTable();
156 
157   }
158 
159   @Test
160   public void testBackupDelete() throws IOException {
161     
162     try (BackupSystemTable table = new BackupSystemTable(conn)) {
163 
164       int n = 10;
165       List<BackupInfo> list = createBackupContextList(n);
166 
167       // Load data
168       for (BackupInfo bc : list) {
169         // Make sure we set right status
170         bc.setState(BackupState.COMPLETE);
171         table.updateBackupInfo(bc);
172       }
173 
174       // Verify exists
175       for (BackupInfo bc : list) {
176         assertNotNull(table.readBackupInfo(bc.getBackupId()));
177       }
178 
179       // Delete all
180       for (BackupInfo bc : list) {
181         table.deleteBackupInfo(bc.getBackupId());
182       }
183 
184       // Verify do not exists
185       for (BackupInfo bc : list) {
186         assertNull(table.readBackupInfo(bc.getBackupId()));
187       }
188 
189       cleanBackupTable();
190     }
191 
192   }
193 
194   
195   
196   @Test
197   public void testRegionServerLastLogRollResults() throws IOException {
198     String[] servers = new String[] { "server1", "server2", "server3" };
199     Long[] timestamps = new Long[] { 100L, 102L, 107L };
200 
201     for (int i = 0; i < servers.length; i++) {
202       table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
203     }
204 
205     HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
206     assertTrue(servers.length == result.size());
207     Set<String> keys = result.keySet();
208     String[] keysAsArray = new String[keys.size()];
209     keys.toArray(keysAsArray);
210     Arrays.sort(keysAsArray);
211 
212     for (int i = 0; i < keysAsArray.length; i++) {
213       assertEquals(keysAsArray[i], servers[i]);
214       Long ts1 = timestamps[i];
215       Long ts2 = result.get(keysAsArray[i]);
216       assertEquals(ts1, ts2);
217     }
218 
219     cleanBackupTable();
220   }
221 
222   @Test
223   public void testIncrementalBackupTableSet() throws IOException {
224     TreeSet<TableName> tables1 = new TreeSet<>();
225 
226     tables1.add(TableName.valueOf("t1"));
227     tables1.add(TableName.valueOf("t2"));
228     tables1.add(TableName.valueOf("t3"));
229 
230     TreeSet<TableName> tables2 = new TreeSet<>();
231 
232     tables2.add(TableName.valueOf("t3"));
233     tables2.add(TableName.valueOf("t4"));
234     tables2.add(TableName.valueOf("t5"));
235 
236     table.addIncrementalBackupTableSet(tables1, "root");
237     BackupSystemTable table = new BackupSystemTable(conn);
238     TreeSet<TableName> res1 = (TreeSet<TableName>) 
239         table.getIncrementalBackupTableSet("root");
240     assertTrue(tables1.size() == res1.size());
241     Iterator<TableName> desc1 = tables1.descendingIterator();
242     Iterator<TableName> desc2 = res1.descendingIterator();
243     while (desc1.hasNext()) {
244       assertEquals(desc1.next(), desc2.next());
245     }
246 
247     table.addIncrementalBackupTableSet(tables2, "root");
248     TreeSet<TableName> res2 = (TreeSet<TableName>) 
249         table.getIncrementalBackupTableSet("root");
250     assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
251 
252     tables1.addAll(tables2);
253 
254     desc1 = tables1.descendingIterator();
255     desc2 = res2.descendingIterator();
256 
257     while (desc1.hasNext()) {
258       assertEquals(desc1.next(), desc2.next());
259     }
260     cleanBackupTable();
261 
262   }
263 
264   @Test
265   public void testRegionServerLogTimestampMap() throws IOException {
266     TreeSet<TableName> tables = new TreeSet<>();
267 
268     tables.add(TableName.valueOf("t1"));
269     tables.add(TableName.valueOf("t2"));
270     tables.add(TableName.valueOf("t3"));
271 
272     HashMap<String, Long> rsTimestampMap = new HashMap<String, Long>();
273 
274     rsTimestampMap.put("rs1", 100L);
275     rsTimestampMap.put("rs2", 101L);
276     rsTimestampMap.put("rs3", 103L);
277 
278     table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
279 
280     HashMap<TableName, HashMap<String, Long>> result = table.readLogTimestampMap("root");
281 
282     assertTrue(tables.size() == result.size());
283 
284     for (TableName t : tables) {
285       HashMap<String, Long> rstm = result.get(t);
286       assertNotNull(rstm);
287       assertEquals(rstm.get("rs1"), new Long(100L));
288       assertEquals(rstm.get("rs2"), new Long(101L));
289       assertEquals(rstm.get("rs3"), new Long(103L));
290     }
291 
292     Set<TableName> tables1 = new TreeSet<>();
293 
294     tables1.add(TableName.valueOf("t3"));
295     tables1.add(TableName.valueOf("t4"));
296     tables1.add(TableName.valueOf("t5"));
297 
298     HashMap<String, Long> rsTimestampMap1 = new HashMap<String, Long>();
299 
300     rsTimestampMap1.put("rs1", 200L);
301     rsTimestampMap1.put("rs2", 201L);
302     rsTimestampMap1.put("rs3", 203L);
303 
304     table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
305 
306     result = table.readLogTimestampMap("root");
307 
308     assertTrue(5 == result.size());
309 
310     for (TableName t : tables) {
311       HashMap<String, Long> rstm = result.get(t);
312       assertNotNull(rstm);
313       if (t.equals(TableName.valueOf("t3")) == false) {
314         assertEquals(rstm.get("rs1"), new Long(100L));
315         assertEquals(rstm.get("rs2"), new Long(101L));
316         assertEquals(rstm.get("rs3"), new Long(103L));
317       } else {
318         assertEquals(rstm.get("rs1"), new Long(200L));
319         assertEquals(rstm.get("rs2"), new Long(201L));
320         assertEquals(rstm.get("rs3"), new Long(203L));
321       }
322     }
323 
324     for (TableName t : tables1) {
325       HashMap<String, Long> rstm = result.get(t);
326       assertNotNull(rstm);
327       assertEquals(rstm.get("rs1"), new Long(200L));
328       assertEquals(rstm.get("rs2"), new Long(201L));
329       assertEquals(rstm.get("rs3"), new Long(203L));
330     }
331 
332     cleanBackupTable();
333 
334   }
335 
336   @Test
337   public void testAddWALFiles() throws IOException {
338     List<String> files =
339         Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
340           "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
341             "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
342     String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
343 
344     table.addWALFiles(files, "backup", "root");
345 
346     assertTrue(table.isWALFileDeletable(files.get(0)));
347     assertTrue(table.isWALFileDeletable(files.get(1)));
348     assertTrue(table.isWALFileDeletable(files.get(2)));
349     assertFalse(table.isWALFileDeletable(newFile));
350 
351     cleanBackupTable();
352   }
353 
354   
355   /**
356    * Backup set tests
357    */
358 
359   @Test
360   public void testBackupSetAddNotExists() throws IOException {
361     try (BackupSystemTable table = new BackupSystemTable(conn)) {
362 
363       String[] tables = new String[] { "table1", "table2", "table3" };
364       String setName = "name";
365       table.addToBackupSet(setName, tables);
366       List<TableName> tnames = table.describeBackupSet(setName);
367       assertTrue(tnames != null);
368       assertTrue(tnames.size() == tables.length);
369       for (int i = 0; i < tnames.size(); i++) {
370         assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
371       }
372       cleanBackupTable();
373     }
374 
375   }
376 
377   @Test
378   public void testBackupSetAddExists() throws IOException {
379     try (BackupSystemTable table = new BackupSystemTable(conn)) {
380 
381       String[] tables = new String[] { "table1", "table2", "table3" };
382       String setName = "name";
383       table.addToBackupSet(setName, tables);
384       String[] addTables = new String[] { "table4", "table5", "table6" };
385       table.addToBackupSet(setName, addTables);
386 
387       List<TableName> tnames = table.describeBackupSet(setName);
388       assertTrue(tnames != null);
389       assertTrue(tnames.size() == tables.length + addTables.length);
390       for (int i = 0; i < tnames.size(); i++) {
391         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
392       }
393       cleanBackupTable();
394     }
395   }
396 
397   @Test
398   public void testBackupSetAddExistsIntersects() throws IOException {
399     try (BackupSystemTable table = new BackupSystemTable(conn)) {
400 
401       String[] tables = new String[] { "table1", "table2", "table3" };
402       String setName = "name";
403       table.addToBackupSet(setName, tables);
404       String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
405       table.addToBackupSet(setName, addTables);
406 
407       List<TableName> tnames = table.describeBackupSet(setName);
408       assertTrue(tnames != null);
409       assertTrue(tnames.size()== tables.length + addTables.length - 1);
410       for (int i = 0; i < tnames.size(); i++) {
411         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
412       }
413       cleanBackupTable();
414     } 
415   }
416 
417   @Test
418   public void testBackupSetRemoveSomeNotExists() throws IOException {
419     try (BackupSystemTable table = new BackupSystemTable(conn)) {
420 
421       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
422       String setName = "name";
423       table.addToBackupSet(setName, tables);
424       String[] removeTables = new String[] { "table4", "table5", "table6" };
425       table.removeFromBackupSet(setName, removeTables);
426 
427       List<TableName> tnames = table.describeBackupSet(setName);
428       assertTrue(tnames != null);
429       assertTrue(tnames.size() == tables.length - 1);
430       for (int i = 0; i < tnames.size(); i++) {
431         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
432       }
433       cleanBackupTable();
434     }
435   }
436 
437   @Test
438   public void testBackupSetRemove() throws IOException {
439     try (BackupSystemTable table = new BackupSystemTable(conn)) {
440 
441       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
442       String setName = "name";
443       table.addToBackupSet(setName, tables);
444       String[] removeTables = new String[] { "table4", "table3" };
445       table.removeFromBackupSet(setName, removeTables);
446 
447       List<TableName> tnames = table.describeBackupSet(setName);
448       assertTrue(tnames != null);
449       assertTrue(tnames.size() == tables.length - 2);
450       for (int i = 0; i < tnames.size(); i++) {
451         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
452       }
453       cleanBackupTable();
454     }
455   }
456 
457   @Test
458   public void testBackupSetDelete() throws IOException {
459     try (BackupSystemTable table = new BackupSystemTable(conn)) {
460 
461       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
462       String setName = "name";
463       table.addToBackupSet(setName, tables);
464       table.deleteBackupSet(setName);
465 
466       List<TableName> tnames = table.describeBackupSet(setName);
467       assertTrue(tnames == null);
468       cleanBackupTable();
469     }
470   }
471 
472   @Test
473   public void testBackupSetList() throws IOException {
474     try (BackupSystemTable table = new BackupSystemTable(conn)) {
475 
476       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
477       String setName1 = "name1";
478       String setName2 = "name2";
479       table.addToBackupSet(setName1, tables);
480       table.addToBackupSet(setName2, tables);
481 
482       List<String> list = table.listBackupSets();
483 
484       assertTrue(list.size() == 2);
485       assertTrue(list.get(0).equals(setName1));
486       assertTrue(list.get(1).equals(setName2));
487 
488       cleanBackupTable();
489     }
490   }
491    
492 
493   private boolean compare(BackupInfo one, BackupInfo two) {
494     return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
495         && one.getTargetRootDir().equals(two.getTargetRootDir())
496         && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
497   }
498 
499   private BackupInfo createBackupContext() {
500 
501     BackupInfo ctxt =
502         new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
503           new TableName[] {
504               TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
505           "/hbase/backup");
506     ctxt.setStartTs(System.currentTimeMillis());
507     ctxt.setEndTs(System.currentTimeMillis() + 1);
508     return ctxt;
509   }
510 
511   private List<BackupInfo> createBackupContextList(int size) {
512     List<BackupInfo> list = new ArrayList<BackupInfo>();
513     for (int i = 0; i < size; i++) {
514       list.add(createBackupContext());
515       try {
516         Thread.sleep(10);
517       } catch (InterruptedException e) {
518         e.printStackTrace();
519       }
520     }
521     return list;
522   }
523 
524   @AfterClass
525   public static void tearDown() throws IOException {
526     if (cluster != null) cluster.shutdown();
527   }
528 }