1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mob;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.util.Random;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileStatus;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.HBaseTestingUtility;
28 import org.apache.hadoop.hbase.HColumnDescriptor;
29 import org.apache.hadoop.hbase.HTableDescriptor;
30 import org.apache.hadoop.hbase.TableName;
31 import org.apache.hadoop.hbase.client.*;
32 import org.apache.hadoop.hbase.testclassification.MediumTests;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.hadoop.util.ToolRunner;
35 import org.junit.After;
36 import org.junit.AfterClass;
37 import org.junit.Before;
38 import org.junit.BeforeClass;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 @Category(MediumTests.class)
43 public class TestExpiredMobFileCleaner {
44
45 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
46 private final static TableName tableName = TableName.valueOf("TestExpiredMobFileCleaner");
47 private final static String family = "family";
48 private final static byte[] row1 = Bytes.toBytes("row1");
49 private final static byte[] row2 = Bytes.toBytes("row2");
50 private final static byte[] qf = Bytes.toBytes("qf");
51
52 private static BufferedMutator table;
53 private static Admin admin;
54
55 @BeforeClass
56 public static void setUpBeforeClass() throws Exception {
57 TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
58 TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
59
60 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
61 }
62
63 @AfterClass
64 public static void tearDownAfterClass() throws Exception {
65
66 }
67
68 @Before
69 public void setUp() throws Exception {
70 TEST_UTIL.startMiniCluster(1);
71 }
72
73 @After
74 public void tearDown() throws Exception {
75 admin.disableTable(tableName);
76 admin.deleteTable(tableName);
77 admin.close();
78 TEST_UTIL.shutdownMiniCluster();
79 TEST_UTIL.getTestFileSystem().delete(TEST_UTIL.getDataTestDir(), true);
80 }
81
82 private void init() throws Exception {
83 HTableDescriptor desc = new HTableDescriptor(tableName);
84 HColumnDescriptor hcd = new HColumnDescriptor(family);
85 hcd.setMobEnabled(true);
86 hcd.setMobThreshold(3L);
87 hcd.setMaxVersions(4);
88 desc.addFamily(hcd);
89
90 admin = TEST_UTIL.getHBaseAdmin();
91 admin.createTable(desc);
92 table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
93 .getBufferedMutator(tableName);
94 }
95
96 private void modifyColumnExpiryDays(int expireDays) throws Exception {
97 HColumnDescriptor hcd = new HColumnDescriptor(family);
98 hcd.setMobEnabled(true);
99 hcd.setMobThreshold(3L);
100
101 int timeToLive = expireDays * secondsOfDay();
102 hcd.setTimeToLive(timeToLive);
103
104 admin.modifyColumn(tableName, hcd);
105 }
106
107 private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts)
108 throws Exception {
109
110 Put put = new Put(row, ts);
111 put.addColumn(Bytes.toBytes(family), qf, value);
112 table.mutate(put);
113
114 table.flush();
115 admin.flush(tableName);
116 }
117
118
119
120
121
122
123 @Test
124 public void testCleaner() throws Exception {
125 init();
126
127 Path mobDirPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
128
129 byte[] dummyData = makeDummyData(600);
130 long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000;
131 putKVAndFlush(table, row1, dummyData, ts);
132 FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
133
134 assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
135 String firstFile = firstFiles[0].getPath().getName();
136
137 ts = System.currentTimeMillis() - 1 * secondsOfDay() * 1000;
138 putKVAndFlush(table, row2, dummyData, ts);
139 FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
140
141 assertEquals("Before cleanup without delay 2", 2, secondFiles.length);
142 String f1 = secondFiles[0].getPath().getName();
143 String f2 = secondFiles[1].getPath().getName();
144 String secondFile = f1.equals(firstFile) ? f2 : f1;
145
146 modifyColumnExpiryDays(2);
147
148
149 String[] args = new String[2];
150 args[0] = tableName.getNameAsString();
151 args[1] = family;
152 ToolRunner.run(TEST_UTIL.getConfiguration(), new ExpiredMobFileCleaner(), args);
153
154 FileStatus[] filesAfterClean = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
155 String lastFile = filesAfterClean[0].getPath().getName();
156
157 assertEquals("After cleanup without delay 1", 1, filesAfterClean.length);
158 assertEquals("After cleanup without delay 2", secondFile, lastFile);
159 }
160
161 private Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
162 Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), familyName);
163 return p;
164 }
165
166 private int secondsOfDay() {
167 return 24 * 3600;
168 }
169
170 private byte[] makeDummyData(int size) {
171 byte [] dummyData = new byte[size];
172 new Random().nextBytes(dummyData);
173 return dummyData;
174 }
175 }