View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mob;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.util.Random;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.client.*;
32  import org.apache.hadoop.hbase.testclassification.MediumTests;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.util.ToolRunner;
35  import org.junit.After;
36  import org.junit.AfterClass;
37  import org.junit.Before;
38  import org.junit.BeforeClass;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  @Category(MediumTests.class)
43  public class TestExpiredMobFileCleaner {
44  
45    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
46    private final static TableName tableName = TableName.valueOf("TestExpiredMobFileCleaner");
47    private final static String family = "family";
48    private final static byte[] row1 = Bytes.toBytes("row1");
49    private final static byte[] row2 = Bytes.toBytes("row2");
50    private final static byte[] qf = Bytes.toBytes("qf");
51  
52    private static BufferedMutator table;
53    private static Admin admin;
54  
55    @BeforeClass
56    public static void setUpBeforeClass() throws Exception {
57      TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
58      TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
59  
60      TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
61    }
62  
63    @AfterClass
64    public static void tearDownAfterClass() throws Exception {
65  
66    }
67  
68    @Before
69    public void setUp() throws Exception {
70      TEST_UTIL.startMiniCluster(1);
71    }
72  
73    @After
74    public void tearDown() throws Exception {
75      admin.disableTable(tableName);
76      admin.deleteTable(tableName);
77      admin.close();
78      TEST_UTIL.shutdownMiniCluster();
79      TEST_UTIL.getTestFileSystem().delete(TEST_UTIL.getDataTestDir(), true);
80    }
81  
82    private void init() throws Exception {
83      HTableDescriptor desc = new HTableDescriptor(tableName);
84      HColumnDescriptor hcd = new HColumnDescriptor(family);
85      hcd.setMobEnabled(true);
86      hcd.setMobThreshold(3L);
87      hcd.setMaxVersions(4);
88      desc.addFamily(hcd);
89  
90      admin = TEST_UTIL.getHBaseAdmin();
91      admin.createTable(desc);
92      table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
93              .getBufferedMutator(tableName);
94    }
95  
96    private void modifyColumnExpiryDays(int expireDays) throws Exception {
97      HColumnDescriptor hcd = new HColumnDescriptor(family);
98      hcd.setMobEnabled(true);
99      hcd.setMobThreshold(3L);
100     // change ttl as expire days to make some row expired
101     int timeToLive = expireDays * secondsOfDay();
102     hcd.setTimeToLive(timeToLive);
103 
104     admin.modifyColumn(tableName, hcd);
105   }
106 
107   private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts)
108       throws Exception {
109 
110     Put put = new Put(row, ts);
111     put.addColumn(Bytes.toBytes(family), qf, value);
112     table.mutate(put);
113 
114     table.flush();
115     admin.flush(tableName);
116   }
117 
118   /**
119    * Creates a 3 day old hfile and an 1 day old hfile then sets expiry to 2 days.
120    * Verifies that the 3 day old hfile is removed but the 1 day one is still present
121    * after the expiry based cleaner is run.
122    */
123   @Test
124   public void testCleaner() throws Exception {
125     init();
126 
127     Path mobDirPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
128 
129     byte[] dummyData = makeDummyData(600);
130     long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000; // 3 days before
131     putKVAndFlush(table, row1, dummyData, ts);
132     FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
133     //the first mob file
134     assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
135     String firstFile = firstFiles[0].getPath().getName();
136 
137     ts = System.currentTimeMillis() - 1 * secondsOfDay() * 1000; // 1 day before
138     putKVAndFlush(table, row2, dummyData, ts);
139     FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
140     //now there are 2 mob files
141     assertEquals("Before cleanup without delay 2", 2, secondFiles.length);
142     String f1 = secondFiles[0].getPath().getName();
143     String f2 = secondFiles[1].getPath().getName();
144     String secondFile = f1.equals(firstFile) ? f2 : f1;
145 
146     modifyColumnExpiryDays(2); // ttl = 2, make the first row expired
147 
148     //run the cleaner
149     String[] args = new String[2];
150     args[0] = tableName.getNameAsString();
151     args[1] = family;
152     ToolRunner.run(TEST_UTIL.getConfiguration(), new ExpiredMobFileCleaner(), args);
153 
154     FileStatus[] filesAfterClean = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
155     String lastFile = filesAfterClean[0].getPath().getName();
156     //the first mob fie is removed
157     assertEquals("After cleanup without delay 1", 1, filesAfterClean.length);
158     assertEquals("After cleanup without delay 2", secondFile, lastFile);
159   }
160 
161   private Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
162     Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), familyName);
163     return p;
164   }
165 
166   private int secondsOfDay() {
167     return 24 * 3600;
168   }
169 
170   private byte[] makeDummyData(int size) {
171     byte [] dummyData = new byte[size];
172     new Random().nextBytes(dummyData);
173     return dummyData;
174   }
175 }