View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  
29  /**
30   * Compaction configuration for a particular instance of HStore.
31   * Takes into account both global settings and ones set on the column family/store.
32   * Control knobs for default compaction algorithm:
33   * <p/>
34   * maxCompactSize - upper bound on file size to be included in minor compactions
35   * minCompactSize - lower bound below which compaction is selected without ratio test
36   * minFilesToCompact - lower bound on number of files in any minor compaction
37   * maxFilesToCompact - upper bound on number of files in any minor compaction
38   * compactionRatio - Ratio used for compaction
39   * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
40   * <p/>
41   * Set parameter as "hbase.hstore.compaction.<attribute>"
42   */
43  
44  @InterfaceAudience.Private
45  public class CompactionConfiguration {
46  
47    static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
48  
49    public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
50    public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
51      "hbase.hstore.compaction.ratio.offpeak";
52    public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
53    public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
54      "hbase.hstore.compaction.min.size";
55    public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
56    public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
57      "hbase.hstore.compaction.max.size";
58    public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
59    public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
60    public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
61        "hbase.hstore.min.locality.to.skip.major.compact";
62  
63    /*
64     * The epoch time length for the windows we no longer compact
65     */
66    public static final String MAX_AGE_MILLIS_KEY =
67      "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
68    public static final String BASE_WINDOW_MILLIS_KEY =
69      "hbase.hstore.compaction.date.tiered.base.window.millis";
70    public static final String WINDOWS_PER_TIER_KEY =
71      "hbase.hstore.compaction.date.tiered.windows.per.tier";
72    public static final String INCOMING_WINDOW_MIN_KEY =
73      "hbase.hstore.compaction.date.tiered.incoming.window.min";
74    public static final String COMPACTION_POLICY_CLASS_FOR_TIERED_WINDOWS_KEY =
75      "hbase.hstore.compaction.date.tiered.window.policy.class";
76    public static final String SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
77      "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
78  
79    private static final Class<? extends RatioBasedCompactionPolicy>
80      DEFAULT_TIER_COMPACTION_POLICY_CLASS = ExploringCompactionPolicy.class;
81  
82    Configuration conf;
83    StoreConfigInformation storeConfigInfo;
84  
85    private final double offPeakCompactionRatio;
86    /** Since all these properties can change online, they are volatile **/
87    private final long maxCompactSize;
88    private final long minCompactSize;
89    /** This one can be update **/
90    private int minFilesToCompact;
91    private final int maxFilesToCompact;
92    private final double compactionRatio;
93    private final long throttlePoint;
94    private final long majorCompactionPeriod;
95    private final float majorCompactionJitter;
96    private final float minLocalityToForceCompact;
97    private final long maxStoreFileAgeMillis;
98    private final long baseWindowMillis;
99    private final int windowsPerTier;
100   private final int incomingWindowMin;
101   private final String compactionPolicyForTieredWindow;
102   private final boolean singleOutputForMinorCompaction;
103 
104   CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
105     this.conf = conf;
106     this.storeConfigInfo = storeConfigInfo;
107 
108     maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
109     minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
110         storeConfigInfo.getMemstoreFlushSize());
111     minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
112           /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
113     maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
114     compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
115     offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
116 
117     throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
118           2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());
119     majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7);
120     // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run
121     majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F);
122     minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
123 
124     maxStoreFileAgeMillis = conf.getLong(MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
125     baseWindowMillis = conf.getLong(BASE_WINDOW_MILLIS_KEY, 3600000 * 6);
126     windowsPerTier = conf.getInt(WINDOWS_PER_TIER_KEY, 4);
127     incomingWindowMin = conf.getInt(INCOMING_WINDOW_MIN_KEY, 6);
128     compactionPolicyForTieredWindow = conf.get(COMPACTION_POLICY_CLASS_FOR_TIERED_WINDOWS_KEY,
129         DEFAULT_TIER_COMPACTION_POLICY_CLASS.getName());
130     singleOutputForMinorCompaction = conf.getBoolean(SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
131       true);
132 
133     LOG.info(this);
134   }
135 
136   @Override
137   public String toString() {
138     return String.format(
139       "size [%d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
140           + " major period %d, major jitter %f, min locality to compact %f;"
141           + " tiered compaction: max_age %d, base window in milliseconds %d, windows per tier %d,"
142           + "incoming window min %d",      
143       minCompactSize,
144       maxCompactSize,
145       minFilesToCompact,
146       maxFilesToCompact,
147       compactionRatio,
148       offPeakCompactionRatio,
149       throttlePoint,
150       majorCompactionPeriod,
151       majorCompactionJitter,
152       minLocalityToForceCompact,
153       maxStoreFileAgeMillis,
154       baseWindowMillis,
155       windowsPerTier,
156       incomingWindowMin);
157   }
158 
159   /**
160    * @return lower bound below which compaction is selected without ratio test
161    */
162   public long getMinCompactSize() {
163     return minCompactSize;
164   }
165 
166   /**
167    * @return upper bound on file size to be included in minor compactions
168    */
169   public long getMaxCompactSize() {
170     return maxCompactSize;
171   }
172 
173   /**
174    * @return upper bound on number of files to be included in minor compactions
175    */
176   public int getMinFilesToCompact() {
177     return minFilesToCompact;
178   }
179 
180   /**
181    * Set upper bound on number of files to be included in minor compactions
182    * @param threshold value to set to
183    */
184   public void setMinFilesToCompact(int threshold) {
185     minFilesToCompact = threshold;
186   }
187 
188   /**
189    * @return upper bound on number of files to be included in minor compactions
190    */
191   public int getMaxFilesToCompact() {
192     return maxFilesToCompact;
193   }
194 
195   /**
196    * @return Ratio used for compaction
197    */
198   public double getCompactionRatio() {
199     return compactionRatio;
200   }
201 
202   /**
203    * @return Off peak Ratio used for compaction
204    */
205   public double getCompactionRatioOffPeak() {
206     return offPeakCompactionRatio;
207   }
208 
209   /**
210    * @return ThrottlePoint used for classifying small and large compactions
211    */
212   public long getThrottlePoint() {
213     return throttlePoint;
214   }
215 
216   /**
217    * @return Major compaction period from compaction.
218    *   Major compactions are selected periodically according to this parameter plus jitter
219    */
220   public long getMajorCompactionPeriod() {
221     return majorCompactionPeriod;
222   }
223 
224   /**
225    * @return Major the jitter fraction, the fraction within which the major compaction
226    *    period is randomly chosen from the majorCompactionPeriod in each store.
227    */
228   public float getMajorCompactionJitter() {
229     return majorCompactionJitter;
230   }
231 
232   /**
233    * @return Block locality ratio, the ratio at which we will include old regions with a single
234    *   store file for major compaction.  Used to improve block locality for regions that
235    *   haven't had writes in a while but are still being read.
236    */
237   public float getMinLocalityToForceCompact() {
238     return minLocalityToForceCompact;
239   }
240 
241   public long getMaxStoreFileAgeMillis() {
242     return maxStoreFileAgeMillis;
243   }
244 
245   public long getBaseWindowMillis() {
246     return baseWindowMillis;
247   }
248 
249   public int getWindowsPerTier() {
250     return windowsPerTier;
251   }
252 
253   public int getIncomingWindowMin() {
254     return incomingWindowMin;
255   }
256 
257   public String getCompactionPolicyForTieredWindow() {
258     return compactionPolicyForTieredWindow;
259   }
260 
261   public boolean useSingleOutputForMinorCompaction() {
262     return singleOutputForMinorCompaction;
263   }
264 }