1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup;
20
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseConfiguration;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.NamespaceDescriptor;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
37 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
38 import org.apache.hadoop.hbase.client.Admin;
39 import org.apache.hadoop.hbase.client.BackupAdmin;
40 import org.apache.hadoop.hbase.client.Connection;
41 import org.apache.hadoop.hbase.client.ConnectionFactory;
42 import org.apache.hadoop.hbase.client.HBaseAdmin;
43 import org.apache.hadoop.hbase.client.HTable;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
46 import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
47 import org.apache.hadoop.hbase.security.UserProvider;
48 import org.apache.hadoop.hbase.security.access.SecureTestUtil;
49 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
52 import org.junit.AfterClass;
53 import org.junit.BeforeClass;
54
55
56
57
58
59
60
61 public class TestBackupBase {
62
63 private static final Log LOG = LogFactory.getLog(TestBackupBase.class);
64
65 protected static Configuration conf1;
66 protected static Configuration conf2;
67
68 protected static HBaseTestingUtility TEST_UTIL;
69 protected static HBaseTestingUtility TEST_UTIL2;
70 protected static TableName table1 = TableName.valueOf("table1");
71 protected static HTableDescriptor table1Desc;
72 protected static TableName table2 = TableName.valueOf("table2");
73 protected static TableName table3 = TableName.valueOf("table3");
74 protected static TableName table4 = TableName.valueOf("table4");
75
76 protected static TableName table1_restore = TableName.valueOf("ns1:table1_restore");
77 protected static TableName table2_restore = TableName.valueOf("ns2:table2_restore");
78 protected static TableName table3_restore = TableName.valueOf("ns3:table3_restore");
79 protected static TableName table4_restore = TableName.valueOf("ns4:table4_restore");
80
81 protected static final int NB_ROWS_IN_BATCH = 999;
82 protected static final byte[] qualName = Bytes.toBytes("q1");
83 protected static final byte[] famName = Bytes.toBytes("f");
84
85 protected static String BACKUP_ROOT_DIR = "/backupUT";
86 protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT";
87
88 protected static boolean secure = false;
89 protected static final String BACKUP_ZNODE = "/backup/hbase";
90 protected static final String BACKUP_SUCCEED_NODE = "complete";
91 protected static final String BACKUP_FAILED_NODE = "failed";
92
93
94
95
96 @BeforeClass
97 public static void setUpBeforeClass() throws Exception {
98 TEST_UTIL = new HBaseTestingUtility();
99 conf1 = TEST_UTIL.getConfiguration();
100 conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
101 conf1.setBoolean(HConstants.BACKUP_ENABLE_KEY, true);
102 if (secure) {
103
104 UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
105 HadoopSecurityEnabledUserProviderForTesting.class);
106
107 SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
108 }
109 String coproc = conf1.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
110 conf1.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, (coproc == null ? "" : coproc + ",") +
111 BackupObserver.class.getName());
112
113
114 TEST_UTIL.startMiniZKCluster();
115 MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
116
117 conf2 = HBaseConfiguration.create(conf1);
118 conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
119 TEST_UTIL2 = new HBaseTestingUtility(conf2);
120 TEST_UTIL2.setZkCluster(miniZK);
121 TEST_UTIL.startMiniCluster();
122 TEST_UTIL2.startMiniCluster();
123 conf1 = TEST_UTIL.getConfiguration();
124
125 TEST_UTIL.startMiniMapReduceCluster();
126 BACKUP_ROOT_DIR = TEST_UTIL.getConfiguration().get("fs.defaultFS") + "/backupUT";
127 LOG.info("ROOTDIR " + BACKUP_ROOT_DIR);
128 BACKUP_REMOTE_ROOT_DIR = TEST_UTIL2.getConfiguration().get("fs.defaultFS") + "/backupUT";
129 LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);
130 waitForSystemTable(TEST_UTIL);
131 createTables();
132 }
133
134 public static void waitForSystemTable(HBaseTestingUtility TEST_UTIL) throws Exception
135 {
136 try(Admin admin = TEST_UTIL.getHBaseAdmin();) {
137 while (!admin.tableExists(BackupSystemTable.getTableName())
138 || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
139 Thread.sleep(1000);
140 }
141 }
142 LOG.debug("backup table exists and available");
143
144 }
145
146
147
148
149 @AfterClass
150 public static void tearDownAfterClass() throws Exception {
151 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
152 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
153 TEST_UTIL2.shutdownMiniCluster();
154 TEST_UTIL.shutdownMiniCluster();
155 TEST_UTIL.shutdownMiniMapReduceCluster();
156 }
157
158 protected String backupTables(BackupType type, List<TableName> tables, String path)
159 throws IOException {
160 Connection conn = null;
161 HBaseAdmin admin = null;
162 BackupAdmin badmin = null;
163 String backupId;
164 try {
165 conn = ConnectionFactory.createConnection(conf1);
166 admin = (HBaseAdmin) conn.getAdmin();
167 BackupRequest request = new BackupRequest();
168 request.setBackupType(type).setTableList(tables).setTargetRootDir(path);
169 badmin = admin.getBackupAdmin();
170 backupId = badmin.backupTables(request);
171 } finally {
172 if(badmin != null){
173 badmin.close();
174 }
175 if (admin != null) {
176 admin.close();
177 }
178 if (conn != null) {
179 conn.close();
180 }
181 }
182 return backupId;
183 }
184
185 protected String fullTableBackup(List<TableName> tables) throws IOException {
186 return backupTables(BackupType.FULL, tables, BACKUP_ROOT_DIR);
187 }
188
189 protected String incrementalTableBackup(List<TableName> tables) throws IOException {
190 return backupTables(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
191 }
192
193 protected static void loadTable(HTable table) throws Exception {
194
195 Put p;
196 for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
197 p = new Put(Bytes.toBytes("row" + i));
198 p.addColumn(famName, qualName, Bytes.toBytes("val" + i));
199 table.put(p);
200 }
201 }
202
203 protected static void createTables() throws Exception {
204
205 long tid = System.currentTimeMillis();
206 table1 = TableName.valueOf("ns1:test-" + tid);
207 HBaseAdmin ha = TEST_UTIL.getHBaseAdmin();
208
209
210 NamespaceDescriptor desc1 = NamespaceDescriptor.create("ns1").build();
211 NamespaceDescriptor desc2 = NamespaceDescriptor.create("ns2").build();
212 NamespaceDescriptor desc3 = NamespaceDescriptor.create("ns3").build();
213 NamespaceDescriptor desc4 = NamespaceDescriptor.create("ns4").build();
214
215 ha.createNamespace(desc1);
216 ha.createNamespace(desc2);
217 ha.createNamespace(desc3);
218 ha.createNamespace(desc4);
219
220
221 HTableDescriptor desc = new HTableDescriptor(table1);
222 HColumnDescriptor fam = new HColumnDescriptor(famName);
223 desc.addFamily(fam);
224 ha.createTable(desc);
225 table1Desc = desc;
226 Connection conn = ConnectionFactory.createConnection(conf1);
227 HTable table = (HTable) conn.getTable(table1);
228 loadTable(table);
229 table.close();
230 table2 = TableName.valueOf("ns2:test-" + tid + 1);
231 desc = new HTableDescriptor(table2);
232 desc.addFamily(fam);
233 ha.createTable(desc);
234 table = (HTable) conn.getTable(table2);
235 loadTable(table);
236 table.close();
237 table3 = TableName.valueOf("ns3:test-" + tid + 2);
238 table = TEST_UTIL.createTable(table3, famName);
239 table.close();
240 table4 = TableName.valueOf("ns4:test-" + tid + 3);
241 table = TEST_UTIL.createTable(table4, famName);
242 table.close();
243 ha.close();
244 conn.close();
245 }
246
247 protected boolean checkSucceeded(String backupId) throws IOException {
248 BackupInfo status = getBackupContext(backupId);
249 if (status == null) return false;
250 return status.getState() == BackupState.COMPLETE;
251 }
252
253 protected boolean checkFailed(String backupId) throws IOException {
254 BackupInfo status = getBackupContext(backupId);
255 if (status == null) return false;
256 return status.getState() == BackupState.FAILED;
257 }
258
259 private BackupInfo getBackupContext(String backupId) throws IOException {
260 try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
261 BackupInfo status = table.readBackupInfo(backupId);
262 return status;
263 }
264 }
265
266
267 protected BackupAdmin getBackupAdmin() throws IOException {
268 return TEST_UTIL.getConnection().getAdmin().getBackupAdmin();
269 }
270
271
272
273
274
275 public RestoreRequest createRestoreRequest(
276 String backupRootDir,
277 String backupId, boolean check, TableName[] fromTables,
278 TableName[] toTables, boolean isOverwrite) {
279 RestoreRequest request = new RestoreRequest();
280 request.setBackupRootDir(backupRootDir).setBackupId(backupId).setCheck(check).
281 setFromTables(fromTables).setToTables(toTables).setOverwrite(isOverwrite);
282 return request;
283 }
284
285
286
287
288 protected List<TableName> toList(String... args){
289 List<TableName> ret = new ArrayList<>();
290 for(int i=0; i < args.length; i++){
291 ret.add(TableName.valueOf(args[i]));
292 }
293 return ret;
294 }
295
296 }