View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.util.Collection;
21  import java.util.HashSet;
22  import java.util.Set;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  
29  /**
30   * A {@link FlushPolicy} that only flushes store larger a given threshold. If no store is large
31   * enough, then all stores will be flushed.
32   */
33  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
34  public class FlushLargeStoresPolicy extends FlushPolicy {
35  
36    private static final Log LOG = LogFactory.getLog(FlushLargeStoresPolicy.class);
37  
38    public static final String HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND =
39        "hbase.hregion.percolumnfamilyflush.size.lower.bound";
40  
41    public static final String HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN =
42        "hbase.hregion.percolumnfamilyflush.size.lower.bound.min";
43  
44    private static final long DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN =
45        1024 * 1024 * 16L;
46  
47    private long flushSizeLowerBound = -1;
48  
49    @Override
50    protected void configureForRegion(HRegion region) {
51      super.configureForRegion(region);
52      int familyNumber = region.getTableDesc().getFamilies().size();
53      if (familyNumber <= 1) {
54        // No need to parse and set flush size lower bound if only one family
55        // Family number might also be zero in some of our unit test case
56        return;
57      }
58      // For multiple families, lower bound is the "average flush size" by default
59      // unless setting in configuration is larger.
60      long flushSizeLowerBound = region.getMemstoreFlushSize() / familyNumber;
61      long minimumLowerBound =
62          getConf().getLong(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
63            DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN);
64      if (minimumLowerBound > flushSizeLowerBound) {
65        flushSizeLowerBound = minimumLowerBound;
66      }
67      // use the setting in table description if any
68      String flushedSizeLowerBoundString =
69          region.getTableDesc().getValue(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND);
70      if (flushedSizeLowerBoundString == null) {
71        if (LOG.isDebugEnabled()) {
72          LOG.debug("No " + HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND
73              + " set in description of table " + region.getTableDesc().getTableName()
74              + ", use config (" + flushSizeLowerBound + ") instead");
75        }
76      } else {
77        try {
78          flushSizeLowerBound = Long.parseLong(flushedSizeLowerBoundString);
79        } catch (NumberFormatException nfe) {
80          // fall back for fault setting
81          LOG.warn("Number format exception when parsing "
82              + HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND + " for table "
83              + region.getTableDesc().getTableName() + ":" + flushedSizeLowerBoundString + ". " + nfe
84              + ", use config (" + flushSizeLowerBound + ") instead");
85  
86        }
87      }
88      this.flushSizeLowerBound = flushSizeLowerBound;
89    }
90  
91    private boolean shouldFlush(Store store) {
92      if (store.getMemStoreSize() > this.flushSizeLowerBound) {
93        if (LOG.isDebugEnabled()) {
94          LOG.debug("Column Family: " + store.getColumnFamilyName() + " of region " + region
95              + " will be flushed because of memstoreSize(" + store.getMemStoreSize()
96              + ") is larger than lower bound(" + this.flushSizeLowerBound + ")");
97        }
98        return true;
99      }
100     return region.shouldFlushStore(store);
101   }
102 
103   @Override
104   public Collection<Store> selectStoresToFlush() {
105     // no need to select stores if only one family
106     if (region.getTableDesc().getFamilies().size() == 1) {
107       return region.stores.values();
108     }
109     // start selection
110     Collection<Store> stores = region.stores.values();
111     Set<Store> specificStoresToFlush = new HashSet<Store>();
112     for (Store store : stores) {
113       if (shouldFlush(store)) {
114         specificStoresToFlush.add(store);
115       }
116     }
117     // Didn't find any CFs which were above the threshold for selection.
118     if (specificStoresToFlush.isEmpty()) {
119       if (LOG.isDebugEnabled()) {
120         LOG.debug("Since none of the CFs were above the size, flushing all.");
121       }
122       return stores;
123     } else {
124       return specificStoresToFlush;
125     }
126   }
127 
128 }