1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import com.google.common.collect.ImmutableList;
21 import com.google.common.collect.Lists;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
30 import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy;
31 import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
32 import org.apache.hadoop.hbase.testclassification.SmallTests;
33 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
34 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
35 import org.junit.Assert;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39 @Category(SmallTests.class)
40 public class TestDateTieredCompactionPolicy extends TestCompactionPolicy {
41 ArrayList<StoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes)
42 throws IOException {
43 ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
44 EnvironmentEdgeManager.injectEdge(timeMachine);
45
46 timeMachine.setValue(1);
47 ArrayList<Long> ageInDisk = new ArrayList<Long>();
48 for (int i = 0; i < sizes.length; i++) {
49 ageInDisk.add(0L);
50 }
51
52 ArrayList<StoreFile> ret = Lists.newArrayList();
53 for (int i = 0; i < sizes.length; i++) {
54 MockStoreFile msf =
55 new MockStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i);
56 msf.setTimeRangeTracker(new TimeRangeTracker(minTimestamps[i], maxTimestamps[i]));
57 ret.add(msf);
58 }
59 return ret;
60 }
61
62 @Override
63 protected void config() {
64 super.config();
65
66
67 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
68 "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
69 conf.setLong(CompactionConfiguration.MAX_AGE_MILLIS_KEY, 100);
70 conf.setLong(CompactionConfiguration.INCOMING_WINDOW_MIN_KEY, 3);
71 conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, 6);
72 conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 4);
73 conf.setBoolean(CompactionConfiguration.SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, false);
74
75
76 this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
77 this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
78 this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
79
80 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
81 conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
82 }
83
84 void compactEquals(long now, ArrayList<StoreFile> candidates, long[] expectedFileSizes,
85 long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException {
86 ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
87 EnvironmentEdgeManager.injectEdge(timeMachine);
88 timeMachine.setValue(now);
89 DateTieredCompactionRequest request;
90 if (isMajor) {
91 for (StoreFile file : candidates) {
92 ((MockStoreFile)file).setIsMajor(true);
93 }
94 Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
95 .shouldPerformMajorCompaction(candidates));
96 request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
97 .getCompactionPolicy()).selectMajorCompaction(candidates);
98 } else {
99 Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
100 .needsCompaction(candidates, ImmutableList.<StoreFile> of()));
101 request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
102 .getCompactionPolicy()).selectMinorCompaction(candidates, false, false);
103 }
104 List<StoreFile> actual = Lists.newArrayList(request.getFiles());
105 Assert.assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual)));
106 Assert.assertEquals(Arrays.toString(expectedBoundaries),
107 Arrays.toString(request.getBoundaries().toArray()));
108 }
109
110
111
112
113
114 @Test
115 public void incomingWindow() throws IOException {
116 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
117 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
118 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
119
120 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
121 new long[] { Long.MIN_VALUE, 12 }, false, true);
122 }
123
124
125
126
127
128 @Test
129 public void NotIncomingWindow() throws IOException {
130 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
131 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
132 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
133
134 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23,
135 24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
136 }
137
138
139
140
141
142 @Test
143 public void OnUpperBoundOfIncomingWindow() throws IOException {
144 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
145 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
146 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
147
148 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
149 new long[] { Long.MIN_VALUE, 12 }, false, true);
150 }
151
152
153
154
155
156 @Test
157 public void NewerThanIncomingWindow() throws IOException {
158 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
159 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
160 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
161
162 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
163 new long[] { Long.MIN_VALUE, 12}, false, true);
164 }
165
166
167
168
169
170 @Test
171 public void NoT2() throws IOException {
172 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
173 long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
174 long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
175
176 compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
177 new long[] { Long.MIN_VALUE, 96}, false, true);
178 }
179
180 @Test
181 public void T1() throws IOException {
182 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
183 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
184 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
185
186 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
187 new long[] { Long.MIN_VALUE, 120 }, false, true);
188 }
189
190
191
192
193
194 @Test
195 public void RatioT0() throws IOException {
196 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
197 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
198 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
199
200 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
201 new long[] { Long.MIN_VALUE }, false, true);
202 }
203
204
205
206
207
208 @Test
209 public void RatioT2() throws IOException {
210 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
211 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
212 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
213
214 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
215 new long[] { Long.MIN_VALUE }, false, true);
216 }
217
218
219
220
221
222 @Test
223 public void RatioT0Next() throws IOException {
224 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
225 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
226 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
227
228 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
229 new long[] { Long.MIN_VALUE }, false, true);
230 }
231
232
233
234
235
236 @Test
237 public void olderThanMaxAge() throws IOException {
238 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
239 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
240 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
241
242 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33,
243 30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
244 }
245
246
247
248
249
250 @Test
251 public void outOfOrder() throws IOException {
252 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
253 long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
254 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
255
256 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34,
257 22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
258 }
259
260
261
262
263
264 @Test
265 public void negativeEpochtime() throws IOException {
266 long[] minTimestamps =
267 new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
268 long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
269 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
270
271 compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
272 new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
273 new long[] { Long.MIN_VALUE, -24 }, false, true);
274 }
275
276
277
278
279
280 @Test
281 public void majorCompation() throws IOException {
282 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
283 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
284 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
285
286 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,41, 42,
287 33, 30, 31, 2, 1 }, new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
288 }
289
290
291
292
293
294
295 @Test
296 public void checkMinMaxTimestampSameBoundary() throws IOException {
297 long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
298 long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
299 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
300
301 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
302 new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
303 new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
304 }
305
306
307
308
309
310 @Test
311 public void negativeForMajor() throws IOException {
312 long[] minTimestamps =
313 new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
314 long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
315 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
316
317 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,
318 41, 42, 33, 30, 31, 2, 1 },
319 new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
320 }
321
322
323
324
325
326 @Test
327 public void maxValuesForMajor() throws IOException {
328 conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, Long.MAX_VALUE / 2);
329 conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 2);
330 store.storeEngine.getCompactionPolicy().setConf(conf);
331 long[] minTimestamps =
332 new long[] { Long.MIN_VALUE, -100 };
333 long[] maxTimestamps = new long[] { -8, Long.MAX_VALUE };
334 long[] sizes = new long[] { 0, 1 };
335
336 compactEquals(Long.MAX_VALUE, sfCreate(minTimestamps, maxTimestamps, sizes),
337 new long[] { 0, 1 },
338 new long[] { Long.MIN_VALUE, -4611686018427387903L, 0, 4611686018427387903L,
339 9223372036854775806L }, true, true);
340 }
341 }