View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import com.google.common.collect.ImmutableList;
21  import com.google.common.collect.Lists;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.List;
27  
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
30  import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy;
31  import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
32  import org.apache.hadoop.hbase.testclassification.SmallTests;
33  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
34  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
35  import org.junit.Assert;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  @Category(SmallTests.class)
40  public class TestDateTieredCompactionPolicy extends TestCompactionPolicy {
41    ArrayList<StoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes)
42        throws IOException {
43      ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
44      EnvironmentEdgeManager.injectEdge(timeMachine);
45      // Has to be  > 0 and < now.
46      timeMachine.setValue(1);
47      ArrayList<Long> ageInDisk = new ArrayList<Long>();
48      for (int i = 0; i < sizes.length; i++) {
49        ageInDisk.add(0L);
50      }
51  
52      ArrayList<StoreFile> ret = Lists.newArrayList();
53      for (int i = 0; i < sizes.length; i++) {
54        MockStoreFile msf =
55            new MockStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i);
56        msf.setTimeRangeTracker(new TimeRangeTracker(minTimestamps[i], maxTimestamps[i]));
57        ret.add(msf);
58      }
59      return ret;
60    }
61  
62    @Override
63    protected void config() {
64      super.config();
65  
66      // Set up policy
67      conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
68        "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
69      conf.setLong(CompactionConfiguration.MAX_AGE_MILLIS_KEY, 100);
70      conf.setLong(CompactionConfiguration.INCOMING_WINDOW_MIN_KEY, 3);
71      conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, 6);
72      conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 4);
73      conf.setBoolean(CompactionConfiguration.SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, false);
74  
75      // Special settings for compaction policy per window
76      this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
77      this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
78      this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
79  
80      conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
81      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
82    }
83  
84    void compactEquals(long now, ArrayList<StoreFile> candidates, long[] expectedFileSizes,
85        long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException {
86      ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
87      EnvironmentEdgeManager.injectEdge(timeMachine);
88      timeMachine.setValue(now);
89      DateTieredCompactionRequest request;
90      if (isMajor) {
91        for (StoreFile file : candidates) {
92          ((MockStoreFile)file).setIsMajor(true);
93        }
94        Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
95          .shouldPerformMajorCompaction(candidates));
96        request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
97            .getCompactionPolicy()).selectMajorCompaction(candidates);
98      } else {
99        Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
100           .needsCompaction(candidates, ImmutableList.<StoreFile> of()));
101       request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
102           .getCompactionPolicy()).selectMinorCompaction(candidates, false, false);
103     }
104     List<StoreFile> actual = Lists.newArrayList(request.getFiles());
105     Assert.assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual)));
106     Assert.assertEquals(Arrays.toString(expectedBoundaries),
107     Arrays.toString(request.getBoundaries().toArray()));
108   }
109 
110   /**
111    * Test for incoming window
112    * @throws IOException with error
113    */
114   @Test
115   public void incomingWindow() throws IOException {
116     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
117     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
118     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
119 
120     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
121       new long[] { Long.MIN_VALUE, 12 }, false, true);
122   }
123 
124   /**
125    * Not enough files in incoming window
126    * @throws IOException with error
127    */
128   @Test
129   public void NotIncomingWindow() throws IOException {
130     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
131     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
132     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
133 
134     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23,
135         24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
136   }
137 
138   /**
139    * Test for file on the upper bound of incoming window
140    * @throws IOException with error
141    */
142   @Test
143   public void OnUpperBoundOfIncomingWindow() throws IOException {
144     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
145     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
146     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
147 
148     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
149       new long[] { Long.MIN_VALUE, 12 }, false, true);
150   }
151 
152   /**
153    * Test for file newer than incoming window
154    * @throws IOException with error
155    */
156   @Test
157   public void NewerThanIncomingWindow() throws IOException {
158     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
159     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
160     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
161 
162     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
163       new long[] { Long.MIN_VALUE, 12}, false, true);
164   }
165 
166   /**
167    * If there is no T1 window, we don't build T2
168    * @throws IOException with error
169    */
170   @Test
171   public void NoT2() throws IOException {
172     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
173     long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
174     long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
175 
176     compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
177       new long[] { Long.MIN_VALUE, 96}, false, true);
178   }
179 
180   @Test
181   public void T1() throws IOException {
182     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
183     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
184     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
185 
186     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
187       new long[] { Long.MIN_VALUE, 120 }, false, true);
188   }
189 
190   /**
191    * Apply exploring logic on non-incoming window
192    * @throws IOException with error
193    */
194   @Test
195   public void RatioT0() throws IOException {
196     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
197     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
198     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
199 
200     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
201       new long[] { Long.MIN_VALUE }, false, true);
202   }
203 
204   /**
205    * Also apply ratio-based logic on t2 window
206    * @throws IOException with error
207    */
208   @Test
209   public void RatioT2() throws IOException {
210     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
211     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
212     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
213 
214     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
215       new long[] { Long.MIN_VALUE }, false, true);
216   }
217 
218   /**
219    * The next compaction call after testTieredCompactionRatioT0 is compacted
220    * @throws IOException with error
221    */
222   @Test
223   public void RatioT0Next() throws IOException {
224     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
225     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
226     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
227 
228     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
229       new long[] { Long.MIN_VALUE }, false, true);
230   }
231 
232   /**
233    * Older than now(161) - maxAge(100)
234    * @throws IOException with error
235    */
236   @Test
237   public void olderThanMaxAge() throws IOException {
238     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
239     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
240     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
241 
242     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33,
243         30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
244   }
245 
246   /**
247    * Out-of-order data
248    * @throws IOException with error
249    */
250   @Test
251   public void outOfOrder() throws IOException {
252     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
253     long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
254     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
255 
256     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34,
257         22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
258   }
259 
260   /**
261    * Negative epoch time
262    * @throws IOException with error
263    */
264   @Test
265   public void negativeEpochtime() throws IOException {
266     long[] minTimestamps =
267         new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
268     long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
269     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
270 
271     compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
272       new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
273       new long[] { Long.MIN_VALUE, -24 }, false, true);
274   }
275 
276   /**
277    * Major compaction
278    * @throws IOException with error
279    */
280   @Test
281   public void majorCompation() throws IOException {
282     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
283     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
284     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
285 
286     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,41, 42,
287       33, 30, 31, 2, 1 }, new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
288   }
289 
290   /**
291    * Major Compaction to check min max timestamp falling in the same window and also to check
292    * boundary condition in which case binary sort gives insertion point as length of the array
293    * @throws IOException
294    */
295   @Test
296   public void checkMinMaxTimestampSameBoundary() throws IOException {
297     long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
298     long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
299     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
300 
301     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
302       new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
303       new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
304   }
305 
306   /**
307    * Major compaction with negative numbers
308    * @throws IOException with error
309    */
310   @Test
311   public void negativeForMajor() throws IOException {
312     long[] minTimestamps =
313         new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
314     long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
315     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
316 
317     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,
318         41, 42, 33, 30, 31, 2, 1 },
319       new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
320   }
321 
322   /**
323    * Major compaction with maximum values
324    * @throws IOException with error
325    */
326   @Test
327   public void maxValuesForMajor() throws IOException {
328     conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, Long.MAX_VALUE / 2);
329     conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 2);
330     store.storeEngine.getCompactionPolicy().setConf(conf);
331     long[] minTimestamps =
332         new long[] { Long.MIN_VALUE, -100 };
333     long[] maxTimestamps = new long[] { -8, Long.MAX_VALUE };
334     long[] sizes = new long[] { 0, 1 };
335 
336     compactEquals(Long.MAX_VALUE, sfCreate(minTimestamps, maxTimestamps, sizes),
337       new long[] { 0, 1 },
338       new long[] { Long.MIN_VALUE, -4611686018427387903L, 0, 4611686018427387903L,
339       9223372036854775806L }, true, true);
340   }
341 }