View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.Iterator;
27  import java.util.List;
28  
29  import com.google.common.collect.ImmutableCollection;
30  import com.google.common.collect.ImmutableList;
31  import com.google.common.collect.Lists;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.KeyValue.KVComparator;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
41  
42  /**
43   * Default implementation of StoreFileManager. Not thread-safe.
44   */
45  @InterfaceAudience.Private
46  class DefaultStoreFileManager implements StoreFileManager {
47    static final Log LOG = LogFactory.getLog(DefaultStoreFileManager.class);
48  
49    private final KVComparator kvComparator;
50    private final CompactionConfiguration comConf;
51    private final int blockingFileCount;
52    private final Comparator<StoreFile> storeFileComparator;
53    /**
54     * List of store files inside this store. This is an immutable list that
55     * is atomically replaced when its contents change.
56     */
57    private volatile ImmutableList<StoreFile> storefiles = null;
58  
59    public DefaultStoreFileManager(KVComparator kvComparator,
60        Comparator<StoreFile> storeFileComparator, Configuration conf,
61        CompactionConfiguration comConf) {
62      this.kvComparator = kvComparator;
63      this.storeFileComparator = storeFileComparator;
64      this.comConf = comConf;
65      this.blockingFileCount =
66          conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
67    }
68  
69    @Override
70    public void loadFiles(List<StoreFile> storeFiles) {
71      sortAndSetStoreFiles(storeFiles);
72    }
73  
74    @Override
75    public final Collection<StoreFile> getStorefiles() {
76      // TODO: I can return a null list of StoreFiles? That'll mess up clients. St.Ack 20151111
77      return storefiles;
78    }
79  
80    @Override
81    public void insertNewFiles(Collection<StoreFile> sfs) throws IOException {
82      ArrayList<StoreFile> newFiles = new ArrayList<StoreFile>(storefiles);
83      newFiles.addAll(sfs);
84      sortAndSetStoreFiles(newFiles);
85    }
86  
87    @Override
88    public ImmutableCollection<StoreFile> clearFiles() {
89      ImmutableList<StoreFile> result = storefiles;
90      storefiles = ImmutableList.of();
91      return result;
92    }
93  
94    @Override
95    public final int getStorefileCount() {
96      return storefiles.size();
97    }
98  
99    @Override
100   public void addCompactionResults(
101     Collection<StoreFile> compactedFiles, Collection<StoreFile> results) {
102     ArrayList<StoreFile> newStoreFiles = Lists.newArrayList(storefiles);
103     newStoreFiles.removeAll(compactedFiles);
104     if (!results.isEmpty()) {
105       newStoreFiles.addAll(results);
106     }
107     sortAndSetStoreFiles(newStoreFiles);
108   }
109 
110   @Override
111   public final Iterator<StoreFile> getCandidateFilesForRowKeyBefore(final KeyValue targetKey) {
112     return new ArrayList<StoreFile>(Lists.reverse(this.storefiles)).iterator();
113   }
114 
115   @Override
116   public Iterator<StoreFile> updateCandidateFilesForRowKeyBefore(
117       Iterator<StoreFile> candidateFiles, final KeyValue targetKey, final Cell candidate) {
118     // Default store has nothing useful to do here.
119     // TODO: move this comment when implementing Level:
120     // Level store can trim the list by range, removing all the files which cannot have
121     // any useful candidates less than "candidate".
122     return candidateFiles;
123   }
124 
125   @Override
126   public final byte[] getSplitPoint() throws IOException {
127     if (this.storefiles.isEmpty()) {
128       return null;
129     }
130     return StoreUtils.getLargestFile(this.storefiles).getFileSplitPoint(this.kvComparator);
131   }
132 
133   @Override
134   public final Collection<StoreFile> getFilesForScanOrGet(boolean isGet,
135       byte[] startRow, byte[] stopRow) {
136     // We cannot provide any useful input and already have the files sorted by seqNum.
137     return getStorefiles();
138   }
139 
140   @Override
141   public int getStoreCompactionPriority() {
142     int priority = blockingFileCount - storefiles.size();
143     return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority;
144   }
145 
146   @Override
147   public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
148     Collection<StoreFile> expiredStoreFiles = null;
149     ImmutableList<StoreFile> files = storefiles;
150     // 1) We can never get rid of the last file which has the maximum seqid.
151     // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
152     for (int i = 0; i < files.size() - 1; ++i) {
153       StoreFile sf = files.get(i);
154       long fileTs = sf.getReader().getMaxTimestamp();
155       if (fileTs < maxTs && !filesCompacting.contains(sf)) {
156         LOG.info("Found an expired store file: " + sf.getPath()
157             + " whose maxTimeStamp is " + fileTs + ", which is below " + maxTs);
158         if (expiredStoreFiles == null) {
159           expiredStoreFiles = new ArrayList<StoreFile>();
160         }
161         expiredStoreFiles.add(sf);
162       }
163     }
164     return expiredStoreFiles;
165   }
166 
167   private void sortAndSetStoreFiles(List<StoreFile> storeFiles) {
168     Collections.sort(storeFiles, storeFileComparator);
169     storefiles = ImmutableList.copyOf(storeFiles);
170   }
171 
172   @Override
173   public double getCompactionPressure() {
174     int storefileCount = getStorefileCount();
175     int minFilesToCompact = comConf.getMinFilesToCompact();
176     if (storefileCount <= minFilesToCompact) {
177       return 0.0;
178     }
179     return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact);
180   }
181 
182   @Override
183   public Comparator<StoreFile> getStoreFileComparator() {
184     return storeFileComparator;
185   }
186 }
187