View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import com.google.common.collect.Lists;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
40  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
41  import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
42  import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
43  import org.apache.hadoop.hbase.testclassification.SmallTests;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  import org.junit.After;
47  import org.junit.Assert;
48  import org.junit.Before;
49  import org.junit.experimental.categories.Category;
50  
51  @Category(SmallTests.class)
52  public class TestCompactionPolicy {
53    private final static Log LOG = LogFactory.getLog(TestDefaultCompactSelection.class);
54    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    protected Configuration conf;
57    protected HStore store;
58    private static final String DIR = TEST_UTIL.getDataTestDir(
59      TestDefaultCompactSelection.class.getSimpleName()).toString();
60    protected static Path TEST_FILE;
61    protected static final int minFiles = 3;
62    protected static final int maxFiles = 5;
63  
64    protected static final long minSize = 10;
65    protected static final long maxSize = 2100;
66  
67    private FSHLog hlog;
68    private HRegion region;
69  
70    @Before
71    public void setUp() throws Exception {
72      config();
73      initialize();
74    }
75  
76    /**
77     * setup config values necessary for store
78     */
79    protected void config() {
80      this.conf = TEST_UTIL.getConfiguration();
81      this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
82      this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, minFiles);
83      this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, maxFiles);
84      this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, minSize);
85      this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, maxSize);
86      this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F);
87    }
88  
89    /**
90     * Setting up a Store
91     * @throws IOException with error
92     */
93    protected void initialize() throws IOException {
94      Path basedir = new Path(DIR);
95      String logName = "logs";
96      Path logdir = new Path(DIR, logName);
97      HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
98      FileSystem fs = FileSystem.get(conf);
99  
100     fs.delete(logdir, true);
101 
102     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table")));
103     htd.addFamily(hcd);
104     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
105 
106     hlog = new FSHLog(fs, basedir, logName, conf);
107     region = HRegion.createHRegion(info, basedir, conf, htd, hlog);
108     region.close();
109     Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
110     region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
111 
112     store = new HStore(region, hcd, conf);
113 
114     TEST_FILE = region.getRegionFileSystem().createTempName();
115     fs.createNewFile(TEST_FILE);
116   }
117 
118   @After
119   public void tearDown() throws IOException {
120     IOException ex = null;
121     try {
122       region.close();
123     } catch (IOException e) {
124       LOG.warn("Caught Exception", e);
125       ex = e;
126     }
127     try {
128       hlog.close();
129     } catch (IOException e) {
130       LOG.warn("Caught Exception", e);
131       ex = e;
132     }
133     if (ex != null) {
134       throw ex;
135     }
136   }
137 
138   ArrayList<Long> toArrayList(long... numbers) {
139     ArrayList<Long> result = new ArrayList<Long>();
140     for (long i : numbers) {
141       result.add(i);
142     }
143     return result;
144   }
145 
146   List<StoreFile> sfCreate(long... sizes) throws IOException {
147     ArrayList<Long> ageInDisk = new ArrayList<Long>();
148     for (int i = 0; i < sizes.length; i++) {
149       ageInDisk.add(0L);
150     }
151     return sfCreate(toArrayList(sizes), ageInDisk);
152   }
153 
154   List<StoreFile> sfCreate(ArrayList<Long> sizes, ArrayList<Long> ageInDisk) throws IOException {
155     return sfCreate(false, sizes, ageInDisk);
156   }
157 
158   List<StoreFile> sfCreate(boolean isReference, long... sizes) throws IOException {
159     ArrayList<Long> ageInDisk = new ArrayList<Long>(sizes.length);
160     for (int i = 0; i < sizes.length; i++) {
161       ageInDisk.add(0L);
162     }
163     return sfCreate(isReference, toArrayList(sizes), ageInDisk);
164   }
165 
166   List<StoreFile> sfCreate(boolean isReference, ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
167       throws IOException {
168     List<StoreFile> ret = Lists.newArrayList();
169     for (int i = 0; i < sizes.size(); i++) {
170       ret.add(new MockStoreFile(TEST_UTIL, TEST_FILE, sizes.get(i), ageInDisk.get(i), isReference,
171           i));
172     }
173     return ret;
174   }
175 
176   long[] getSizes(List<StoreFile> sfList) {
177     long[] aNums = new long[sfList.size()];
178     for (int i = 0; i < sfList.size(); ++i) {
179       aNums[i] = sfList.get(i).getReader().length();
180     }
181     return aNums;
182   }
183 
184   void compactEquals(List<StoreFile> candidates, long... expected) throws IOException {
185     compactEquals(candidates, false, false, expected);
186   }
187 
188   void compactEquals(List<StoreFile> candidates, boolean forcemajor, long... expected)
189       throws IOException {
190     compactEquals(candidates, forcemajor, false, expected);
191   }
192 
193   void compactEquals(List<StoreFile> candidates, boolean forcemajor, boolean isOffPeak,
194       long... expected) throws IOException {
195     store.forceMajor = forcemajor;
196     // Test Default compactions
197     CompactionRequest result =
198         ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy()).selectCompaction(
199           candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
200     List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
201     if (isOffPeak && !forcemajor) {
202       Assert.assertTrue(result.isOffPeak());
203     }
204     Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
205     store.forceMajor = false;
206   }
207 }