View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements. See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership. The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License. You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.backup;
20  
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileStatus;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.LocatedFileStatus;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.RemoteIterator;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
39  import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
40  import org.apache.hadoop.hbase.client.Connection;
41  import org.apache.hadoop.hbase.client.ConnectionFactory;
42  import org.apache.hadoop.hbase.client.HTable;
43  import org.apache.hadoop.hbase.client.Put;
44  import org.apache.hadoop.hbase.testclassification.LargeTests;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  import com.google.common.collect.Iterables;
52  import com.google.common.collect.Lists;
53  
54  @Category(LargeTests.class)
55  public class TestBackupLogCleaner extends TestBackupBase {
56    private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
57  
58    // implements all test cases in 1 test since incremental full backup/
59    // incremental backup has dependencies
60    @Test
61    public void testBackupLogCleaner() throws Exception {
62  
63      // #1 - create full backup for all tables
64      LOG.info("create full backup image for all tables");
65  
66      List<TableName> tableSetFullList = Lists.newArrayList(table1, table2, table3, table4);
67  
68      try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
69        // Verify that we have no backup sessions yet
70        assertFalse(systemTable.hasBackupSessions());
71  
72        List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
73        List<String> swalFiles = convert(walFiles);
74        BackupLogCleaner cleaner = new BackupLogCleaner();
75        cleaner.setConf(TEST_UTIL.getConfiguration());
76  
77        Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
78        int size = Iterables.size(deletable);
79  
80        // We can delete all files because we do not have yet recorded backup sessions
81        assertTrue(size == walFiles.size());
82  
83        systemTable.addWALFiles(swalFiles, "backup", "root");
84        String backupIdFull = fullTableBackup(tableSetFullList);
85        assertTrue(checkSucceeded(backupIdFull));
86        // Check one more time
87        deletable = cleaner.getDeletableFiles(walFiles);
88        // We can delete wal files because they were saved into hbase:backup table
89        size = Iterables.size(deletable);
90        assertTrue(size == walFiles.size());
91  
92        List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
93        LOG.debug("WAL list after full backup");
94        convert(newWalFiles);
95  
96        // New list of wal files is greater than the previous one,
97        // because new wal per RS have been opened after full backup
98        assertTrue(walFiles.size() < newWalFiles.size());
99        Connection conn = ConnectionFactory.createConnection(conf1);
100       // #2 - insert some data to table
101       HTable t1 = (HTable) conn.getTable(table1);
102       Put p1;
103       for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
104         p1 = new Put(Bytes.toBytes("row-t1" + i));
105         p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
106         t1.put(p1);
107       }
108 
109       t1.close();
110 
111       HTable t2 = (HTable) conn.getTable(table2);
112       Put p2;
113       for (int i = 0; i < 5; i++) {
114         p2 = new Put(Bytes.toBytes("row-t2" + i));
115         p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
116         t2.put(p2);
117       }
118 
119       t2.close();
120 
121       // #3 - incremental backup for multiple tables
122 
123       List<TableName> tableSetIncList = Lists.newArrayList(table1, table2, table3);
124       String backupIdIncMultiple = backupTables(BackupType.INCREMENTAL, tableSetIncList,
125         BACKUP_ROOT_DIR);
126       assertTrue(checkSucceeded(backupIdIncMultiple));
127       deletable = cleaner.getDeletableFiles(newWalFiles);
128 
129       assertTrue(Iterables.size(deletable) == newWalFiles.size());
130 
131       conn.close();
132     }
133   }
134 
135   private List<String> convert(List<FileStatus> walFiles) {
136     List<String> result = new ArrayList<String>();
137     for (FileStatus fs : walFiles) {
138       LOG.debug("+++WAL: " + fs.getPath().toString());
139       result.add(fs.getPath().toString());
140     }
141     return result;
142   }
143 
144   private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
145     Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
146     FileSystem fs = FileSystem.get(c);
147     RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
148     List<FileStatus> logFiles = new ArrayList<FileStatus>();
149     while (it.hasNext()) {
150       LocatedFileStatus lfs = it.next();
151       if (lfs.isFile() && !DefaultWALProvider.isMetaFile(lfs.getPath())) {
152         logFiles.add(lfs);
153         LOG.info(lfs);
154       }
155     }
156     return logFiles;
157   }
158 
159 }