1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup;
20
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.LocatedFileStatus;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.RemoteIterator;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
39 import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
40 import org.apache.hadoop.hbase.client.Connection;
41 import org.apache.hadoop.hbase.client.ConnectionFactory;
42 import org.apache.hadoop.hbase.client.HTable;
43 import org.apache.hadoop.hbase.client.Put;
44 import org.apache.hadoop.hbase.testclassification.LargeTests;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 import com.google.common.collect.Iterables;
52 import com.google.common.collect.Lists;
53
54 @Category(LargeTests.class)
55 public class TestBackupLogCleaner extends TestBackupBase {
56 private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
57
58
59
60 @Test
61 public void testBackupLogCleaner() throws Exception {
62
63
64 LOG.info("create full backup image for all tables");
65
66 List<TableName> tableSetFullList = Lists.newArrayList(table1, table2, table3, table4);
67
68 try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
69
70 assertFalse(systemTable.hasBackupSessions());
71
72 List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
73 List<String> swalFiles = convert(walFiles);
74 BackupLogCleaner cleaner = new BackupLogCleaner();
75 cleaner.setConf(TEST_UTIL.getConfiguration());
76
77 Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
78 int size = Iterables.size(deletable);
79
80
81 assertTrue(size == walFiles.size());
82
83 systemTable.addWALFiles(swalFiles, "backup", "root");
84 String backupIdFull = fullTableBackup(tableSetFullList);
85 assertTrue(checkSucceeded(backupIdFull));
86
87 deletable = cleaner.getDeletableFiles(walFiles);
88
89 size = Iterables.size(deletable);
90 assertTrue(size == walFiles.size());
91
92 List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
93 LOG.debug("WAL list after full backup");
94 convert(newWalFiles);
95
96
97
98 assertTrue(walFiles.size() < newWalFiles.size());
99 Connection conn = ConnectionFactory.createConnection(conf1);
100
101 HTable t1 = (HTable) conn.getTable(table1);
102 Put p1;
103 for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
104 p1 = new Put(Bytes.toBytes("row-t1" + i));
105 p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
106 t1.put(p1);
107 }
108
109 t1.close();
110
111 HTable t2 = (HTable) conn.getTable(table2);
112 Put p2;
113 for (int i = 0; i < 5; i++) {
114 p2 = new Put(Bytes.toBytes("row-t2" + i));
115 p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
116 t2.put(p2);
117 }
118
119 t2.close();
120
121
122
123 List<TableName> tableSetIncList = Lists.newArrayList(table1, table2, table3);
124 String backupIdIncMultiple = backupTables(BackupType.INCREMENTAL, tableSetIncList,
125 BACKUP_ROOT_DIR);
126 assertTrue(checkSucceeded(backupIdIncMultiple));
127 deletable = cleaner.getDeletableFiles(newWalFiles);
128
129 assertTrue(Iterables.size(deletable) == newWalFiles.size());
130
131 conn.close();
132 }
133 }
134
135 private List<String> convert(List<FileStatus> walFiles) {
136 List<String> result = new ArrayList<String>();
137 for (FileStatus fs : walFiles) {
138 LOG.debug("+++WAL: " + fs.getPath().toString());
139 result.add(fs.getPath().toString());
140 }
141 return result;
142 }
143
144 private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
145 Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
146 FileSystem fs = FileSystem.get(c);
147 RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
148 List<FileStatus> logFiles = new ArrayList<FileStatus>();
149 while (it.hasNext()) {
150 LocatedFileStatus lfs = it.next();
151 if (lfs.isFile() && !DefaultWALProvider.isMetaFile(lfs.getPath())) {
152 logFiles.add(lfs);
153 LOG.info(lfs);
154 }
155 }
156 return logFiles;
157 }
158
159 }