View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.backup;
20  
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HBaseConfiguration;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.NamespaceDescriptor;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
37  import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
38  import org.apache.hadoop.hbase.client.Admin;
39  import org.apache.hadoop.hbase.client.BackupAdmin;
40  import org.apache.hadoop.hbase.client.Connection;
41  import org.apache.hadoop.hbase.client.ConnectionFactory;
42  import org.apache.hadoop.hbase.client.HBaseAdmin;
43  import org.apache.hadoop.hbase.client.HTable;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
46  import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
47  import org.apache.hadoop.hbase.security.UserProvider;
48  import org.apache.hadoop.hbase.security.access.SecureTestUtil;
49  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
52  import org.junit.AfterClass;
53  import org.junit.BeforeClass;
54  
55  
56  /**
57   * This class is only a base for other integration-level backup tests. Do not add tests here.
58   * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other
59   * tests should have their own classes and extend this one
60   */
61  public class TestBackupBase {
62  
63    private static final Log LOG = LogFactory.getLog(TestBackupBase.class);
64  
65    protected static Configuration conf1;
66    protected static Configuration conf2;
67  
68    protected static HBaseTestingUtility TEST_UTIL;
69    protected static HBaseTestingUtility TEST_UTIL2;
70    protected static TableName table1 = TableName.valueOf("table1");
71    protected static HTableDescriptor table1Desc;
72    protected static TableName table2 = TableName.valueOf("table2");
73    protected static TableName table3 = TableName.valueOf("table3");
74    protected static TableName table4 = TableName.valueOf("table4");
75  
76    protected static TableName table1_restore = TableName.valueOf("ns1:table1_restore");
77    protected static TableName table2_restore = TableName.valueOf("ns2:table2_restore");
78    protected static TableName table3_restore = TableName.valueOf("ns3:table3_restore");
79    protected static TableName table4_restore = TableName.valueOf("ns4:table4_restore");
80  
81    protected static final int NB_ROWS_IN_BATCH = 999;
82    protected static final byte[] qualName = Bytes.toBytes("q1");
83    protected static final byte[] famName = Bytes.toBytes("f");
84  
85    protected static String BACKUP_ROOT_DIR = "/backupUT";
86    protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT";
87  
88    protected static boolean secure = false;
89    protected static final String BACKUP_ZNODE = "/backup/hbase";
90    protected static final String BACKUP_SUCCEED_NODE = "complete";
91    protected static final String BACKUP_FAILED_NODE = "failed";
92  
93    /**
94     * @throws java.lang.Exception
95     */
96    @BeforeClass
97    public static void setUpBeforeClass() throws Exception {
98      TEST_UTIL = new HBaseTestingUtility();
99      conf1 = TEST_UTIL.getConfiguration();
100     conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
101     conf1.setBoolean(HConstants.BACKUP_ENABLE_KEY, true);
102     if (secure) {
103       // set the always on security provider
104       UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
105           HadoopSecurityEnabledUserProviderForTesting.class);
106       // setup configuration
107       SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
108     }
109     String coproc = conf1.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
110     conf1.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, (coproc == null ? "" : coproc + ",") +
111         BackupObserver.class.getName());
112     // Set MultiWAL (with 2 default WAL files per RS)
113     //conf1.set(WAL_PROVIDER, "multiwal");
114     TEST_UTIL.startMiniZKCluster();
115     MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
116 
117     conf2 = HBaseConfiguration.create(conf1);
118     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
119     TEST_UTIL2 = new HBaseTestingUtility(conf2);
120     TEST_UTIL2.setZkCluster(miniZK);
121     TEST_UTIL.startMiniCluster();
122     TEST_UTIL2.startMiniCluster();
123     conf1 = TEST_UTIL.getConfiguration();
124 
125     TEST_UTIL.startMiniMapReduceCluster();
126     BACKUP_ROOT_DIR = TEST_UTIL.getConfiguration().get("fs.defaultFS") + "/backupUT";
127     LOG.info("ROOTDIR " + BACKUP_ROOT_DIR);
128     BACKUP_REMOTE_ROOT_DIR = TEST_UTIL2.getConfiguration().get("fs.defaultFS") + "/backupUT";
129     LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);
130     waitForSystemTable(TEST_UTIL);
131     createTables();
132   }
133   
134   public static void waitForSystemTable(HBaseTestingUtility TEST_UTIL) throws Exception
135   {
136     try(Admin admin = TEST_UTIL.getHBaseAdmin();) {
137       while (!admin.tableExists(BackupSystemTable.getTableName()) 
138           || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
139         Thread.sleep(1000);
140       }      
141     }
142     LOG.debug("backup table exists and available");
143 
144   }
145 
146   /**
147    * @throws java.lang.Exception
148    */
149   @AfterClass
150   public static void tearDownAfterClass() throws Exception {
151     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
152     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
153     TEST_UTIL2.shutdownMiniCluster();
154     TEST_UTIL.shutdownMiniCluster();
155     TEST_UTIL.shutdownMiniMapReduceCluster();
156   }
157 
158   protected String backupTables(BackupType type, List<TableName> tables, String path)
159       throws IOException {
160     Connection conn = null;
161     HBaseAdmin admin = null;
162     BackupAdmin badmin = null;
163     String backupId;
164     try {
165       conn = ConnectionFactory.createConnection(conf1);
166       admin = (HBaseAdmin) conn.getAdmin();
167       BackupRequest request = new BackupRequest();
168       request.setBackupType(type).setTableList(tables).setTargetRootDir(path);
169       badmin = admin.getBackupAdmin();
170       backupId = badmin.backupTables(request);
171     } finally {
172       if(badmin != null){
173         badmin.close();
174       }
175       if (admin != null) {
176         admin.close();
177       }
178       if (conn != null) {
179         conn.close();
180       }
181     }
182     return backupId;
183   }
184 
185   protected String fullTableBackup(List<TableName> tables) throws IOException {
186     return backupTables(BackupType.FULL, tables, BACKUP_ROOT_DIR);
187   }
188 
189   protected String incrementalTableBackup(List<TableName> tables) throws IOException {
190     return backupTables(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
191   }
192 
193   protected static void loadTable(HTable table) throws Exception {
194 
195     Put p; // 100 + 1 row to t1_syncup
196     for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
197       p = new Put(Bytes.toBytes("row" + i));
198       p.addColumn(famName, qualName, Bytes.toBytes("val" + i));
199       table.put(p);
200     }
201   }
202 
203   protected static void createTables() throws Exception {
204 
205     long tid = System.currentTimeMillis();
206     table1 = TableName.valueOf("ns1:test-" + tid);
207     HBaseAdmin ha = TEST_UTIL.getHBaseAdmin();
208 
209     // Create namespaces
210     NamespaceDescriptor desc1 = NamespaceDescriptor.create("ns1").build();
211     NamespaceDescriptor desc2 = NamespaceDescriptor.create("ns2").build();
212     NamespaceDescriptor desc3 = NamespaceDescriptor.create("ns3").build();
213     NamespaceDescriptor desc4 = NamespaceDescriptor.create("ns4").build();
214 
215     ha.createNamespace(desc1);
216     ha.createNamespace(desc2);
217     ha.createNamespace(desc3);
218     ha.createNamespace(desc4);
219 
220 
221     HTableDescriptor desc = new HTableDescriptor(table1);
222     HColumnDescriptor fam = new HColumnDescriptor(famName);
223     desc.addFamily(fam);
224     ha.createTable(desc);
225     table1Desc = desc;
226     Connection conn = ConnectionFactory.createConnection(conf1);
227     HTable table = (HTable) conn.getTable(table1);
228     loadTable(table);
229     table.close();
230     table2 = TableName.valueOf("ns2:test-" + tid + 1);
231     desc = new HTableDescriptor(table2);
232     desc.addFamily(fam);
233     ha.createTable(desc);
234     table = (HTable) conn.getTable(table2);
235     loadTable(table);
236     table.close();
237     table3 = TableName.valueOf("ns3:test-" + tid + 2);
238     table = TEST_UTIL.createTable(table3, famName);
239     table.close();
240     table4 = TableName.valueOf("ns4:test-" + tid + 3);
241     table = TEST_UTIL.createTable(table4, famName);
242     table.close();
243     ha.close();
244     conn.close();
245   }
246 
247   protected boolean checkSucceeded(String backupId) throws IOException {
248     BackupInfo status = getBackupContext(backupId);
249     if (status == null) return false;
250     return status.getState() == BackupState.COMPLETE;
251   }
252 
253   protected boolean checkFailed(String backupId) throws IOException {
254     BackupInfo status = getBackupContext(backupId);
255     if (status == null) return false;
256     return status.getState() == BackupState.FAILED;
257   }
258 
259   private BackupInfo getBackupContext(String backupId) throws IOException {
260     try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
261       BackupInfo status = table.readBackupInfo(backupId);
262       return status;
263     }
264   }
265 
266 
267   protected BackupAdmin getBackupAdmin() throws IOException {
268     return TEST_UTIL.getConnection().getAdmin().getBackupAdmin();
269   }
270 
271   /**
272    * Get restore request.
273    *
274    */
275   public  RestoreRequest createRestoreRequest(
276       String backupRootDir,
277       String backupId, boolean check, TableName[] fromTables,
278       TableName[] toTables, boolean isOverwrite) {
279     RestoreRequest request = new RestoreRequest();
280     request.setBackupRootDir(backupRootDir).setBackupId(backupId).setCheck(check).
281     setFromTables(fromTables).setToTables(toTables).setOverwrite(isOverwrite);
282     return request;
283 }
284 
285   /**
286    * Helper method
287    */
288   protected List<TableName> toList(String... args){
289     List<TableName> ret = new ArrayList<>();
290     for(int i=0; i < args.length; i++){
291       ret.add(TableName.valueOf(args[i]));
292     }
293     return ret;
294   }
295 
296 }