1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.TableName;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.testclassification.LargeTests;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Table;
37 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44
45 @Category(LargeTests.class)
46 public class TestCompactionState {
47 final static Log LOG = LogFactory.getLog(TestCompactionState.class);
48 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49 private final static Random random = new Random();
50
51 @BeforeClass
52 public static void setUpBeforeClass() throws Exception {
53 TEST_UTIL.startMiniCluster();
54 }
55
56 @AfterClass
57 public static void tearDownAfterClass() throws Exception {
58 TEST_UTIL.shutdownMiniCluster();
59 }
60
61 @Test(timeout=600000)
62 public void testMajorCompaction() throws IOException, InterruptedException {
63 compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
64 }
65
66 @Test(timeout=600000)
67 public void testMinorCompaction() throws IOException, InterruptedException {
68 compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
69 }
70
71 @Test(timeout=600000)
72 public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
73 compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
74 }
75
76 @Test(timeout=600000)
77 public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
78 compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
79 }
80
81 @Test
82 public void testInvalidColumnFamily() throws IOException, InterruptedException {
83 TableName table = TableName.valueOf("testInvalidColumnFamily");
84 byte [] family = Bytes.toBytes("family");
85 byte [] fakecf = Bytes.toBytes("fakecf");
86 boolean caughtMinorCompact = false;
87 boolean caughtMajorCompact = false;
88 Table ht = null;
89 try {
90 ht = TEST_UTIL.createTable(table, family);
91 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
92 try {
93 admin.compact(table, fakecf);
94 } catch (IOException ioe) {
95 caughtMinorCompact = true;
96 }
97 try {
98 admin.majorCompact(table, fakecf);
99 } catch (IOException ioe) {
100 caughtMajorCompact = true;
101 }
102 } finally {
103 if (ht != null) {
104 TEST_UTIL.deleteTable(table);
105 }
106 assertTrue(caughtMinorCompact);
107 assertTrue(caughtMajorCompact);
108 }
109 }
110
111
112
113
114
115
116
117
118
119
120
121
122 private void compaction(final String tableName, final int flushes,
123 final CompactionState expectedState, boolean singleFamily)
124 throws IOException, InterruptedException {
125
126 TableName table = TableName.valueOf(tableName);
127 byte [] family = Bytes.toBytes("family");
128 byte [][] families =
129 {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
130 Table ht = null;
131 try {
132 ht = TEST_UTIL.createTable(table, families);
133 loadData(ht, families, 3000, flushes);
134 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
135 List<Region> regions = rs.getOnlineRegions(table);
136 int countBefore = countStoreFilesInFamilies(regions, families);
137 int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
138 assertTrue(countBefore > 0);
139 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
140 if (expectedState == CompactionState.MINOR) {
141 if (singleFamily) {
142 admin.compact(table.getName(), family);
143 } else {
144 admin.compact(table.getName());
145 }
146 } else {
147 if (singleFamily) {
148 admin.majorCompact(table.getName(), family);
149 } else {
150 admin.majorCompact(table.getName());
151 }
152 }
153 long curt = System.currentTimeMillis();
154 long waitTime = 5000;
155 long endt = curt + waitTime;
156 CompactionState state = admin.getCompactionState(table.getName());
157 while (state == CompactionState.NONE && curt < endt) {
158 Thread.sleep(10);
159 state = admin.getCompactionState(table.getName());
160 curt = System.currentTimeMillis();
161 }
162
163
164 if (expectedState != state) {
165 for (Region region: regions) {
166 state = region.getCompactionState();
167 assertEquals(CompactionState.NONE, state);
168 }
169 } else {
170
171 state = admin.getCompactionState(table.getName());
172 while (state != CompactionState.NONE && curt < endt) {
173 Thread.sleep(10);
174 state = admin.getCompactionState(table.getName());
175 }
176
177 assertEquals(CompactionState.NONE, state);
178 }
179 int countAfter = countStoreFilesInFamilies(regions, families);
180 int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
181 assertTrue(countAfter < countBefore);
182 if (!singleFamily) {
183 if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
184 else assertTrue(families.length < countAfter);
185 } else {
186 int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
187
188 assertTrue(singleFamDiff == (countBefore - countAfter));
189 if (expectedState == CompactionState.MAJOR) {
190 assertTrue(1 == countAfterSingleFamily);
191 } else {
192 assertTrue(1 < countAfterSingleFamily);
193 }
194 }
195 } finally {
196 if (ht != null) {
197 TEST_UTIL.deleteTable(table);
198 }
199 }
200 }
201
202 private static int countStoreFilesInFamily(
203 List<Region> regions, final byte[] family) {
204 return countStoreFilesInFamilies(regions, new byte[][]{family});
205 }
206
207 private static int countStoreFilesInFamilies(List<Region> regions, final byte[][] families) {
208 int count = 0;
209 for (Region region: regions) {
210 count += region.getStoreFileList(families).size();
211 }
212 return count;
213 }
214
215 private static void loadData(final Table ht, final byte[][] families,
216 final int rows, final int flushes) throws IOException {
217 List<Put> puts = new ArrayList<Put>(rows);
218 byte[] qualifier = Bytes.toBytes("val");
219 for (int i = 0; i < flushes; i++) {
220 for (int k = 0; k < rows; k++) {
221 byte[] row = Bytes.toBytes(random.nextLong());
222 Put p = new Put(row);
223 for (int j = 0; j < families.length; ++j) {
224 p.add(families[ j ], qualifier, row);
225 }
226 puts.add(p);
227 }
228 ht.put(puts);
229 TEST_UTIL.flush();
230 puts.clear();
231 }
232 }
233 }