View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.hbase.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configurable;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FSDataInputStream;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.KVComparator;
31  import org.apache.hadoop.hbase.fs.HFileSystem;
32  import org.apache.hadoop.hbase.io.compress.Compression;
33  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
34  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
35  
36  /**
37   * Common functionality needed by all versions of {@link HFile} readers.
38   */
39  @InterfaceAudience.Private
40  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
41  public abstract class AbstractHFileReader
42      implements HFile.Reader, Configurable {
43    /** Stream to read from. Does checksum verifications in file system */
44    protected FSDataInputStream istream; // UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD
45  
46    /** The file system stream of the underlying {@link HFile} that
47     * does not do checksum verification in the file system */
48    protected FSDataInputStream istreamNoFsChecksum;  // UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD
49  
50    /** Data block index reader keeping the root data index in memory */
51    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
52  
53    /** Meta block index reader -- always single level */
54    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
55  
56    protected final FixedFileTrailer trailer;
57  
58    /** Filled when we read in the trailer. */
59    protected final Compression.Algorithm compressAlgo;
60  
61    private boolean isPrimaryReplicaReader;
62  
63    /**
64     * What kind of data block encoding should be used while reading, writing,
65     * and handling cache.
66     */
67    protected HFileDataBlockEncoder dataBlockEncoder =
68        NoOpDataBlockEncoder.INSTANCE;
69  
70    /** Last key in the file. Filled in when we read in the file info */
71    protected byte [] lastKey = null;
72  
73    /** Average key length read from file info */
74    protected int avgKeyLen = -1;
75  
76    /** Average value length read from file info */
77    protected int avgValueLen = -1;
78  
79    /** Key comparator */
80    protected KVComparator comparator = new KVComparator();
81  
82    /** Size of this file. */
83    protected final long fileSize;
84  
85    /** Block cache configuration. */
86    protected final CacheConfig cacheConf;
87  
88    /** Path of file */
89    protected final Path path;
90  
91    /** File name to be used for block names */
92    protected final String name;
93  
94    protected FileInfo fileInfo;
95  
96    /** The filesystem used for accesing data */
97    protected HFileSystem hfs;
98  
99    protected Configuration conf;
100 
101   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
102   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
103       final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs,
104       final Configuration conf) {
105     this.trailer = trailer;
106     this.compressAlgo = trailer.getCompressionCodec();
107     this.cacheConf = cacheConf;
108     this.fileSize = fileSize;
109     this.path = path;
110     this.name = path.getName();
111     this.hfs = hfs; // URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD
112     this.conf = conf;
113   }
114 
115   @SuppressWarnings("serial")
116   public static class BlockIndexNotLoadedException
117       extends IllegalStateException {
118     public BlockIndexNotLoadedException() {
119       // Add a message in case anyone relies on it as opposed to class name.
120       super("Block index not loaded");
121     }
122   }
123 
124   protected String toStringFirstKey() {
125     return KeyValue.keyToString(getFirstKey());
126   }
127 
128   protected String toStringLastKey() {
129     return KeyValue.keyToString(getLastKey());
130   }
131 
132   public abstract boolean isFileInfoLoaded();
133 
134   @Override
135   public String toString() {
136     return "reader=" + path.toString() +
137         (!isFileInfoLoaded()? "":
138           ", compression=" + compressAlgo.getName() +
139           ", cacheConf=" + cacheConf +
140           ", firstKey=" + toStringFirstKey() +
141           ", lastKey=" + toStringLastKey()) +
142           ", avgKeyLen=" + avgKeyLen +
143           ", avgValueLen=" + avgValueLen +
144           ", entries=" + trailer.getEntryCount() +
145           ", length=" + fileSize;
146   }
147 
148   @Override
149   public long length() {
150     return fileSize;
151   }
152 
153   /**
154    * Create a Scanner on this file. No seeks or reads are done on creation. Call
155    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
156    * nothing to clean up in a Scanner. Letting go of your references to the
157    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
158    * compactions.
159    *
160    * @param cacheBlocks True if we should cache blocks read in by this scanner.
161    * @param pread Use positional read rather than seek+read if true (pread is
162    *          better for random reads, seek+read is better scanning).
163    * @return Scanner on this file.
164    */
165   @Override
166   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
167     return getScanner(cacheBlocks, pread, false);
168   }
169 
170   /**
171    * @return the first key in the file. May be null if file has no entries. Note
172    *         that this is not the first row key, but rather the byte form of the
173    *         first KeyValue.
174    */
175   @Override
176   public byte [] getFirstKey() {
177     if (dataBlockIndexReader == null) {
178       throw new BlockIndexNotLoadedException();
179     }
180     return dataBlockIndexReader.isEmpty() ? null
181         : dataBlockIndexReader.getRootBlockKey(0);
182   }
183 
184   /**
185    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
186    * patch goes in to eliminate {@link KeyValue} here.
187    *
188    * @return the first row key, or null if the file is empty.
189    */
190   @Override
191   public byte[] getFirstRowKey() {
192     byte[] firstKey = getFirstKey();
193     if (firstKey == null)
194       return null;
195     return KeyValue.createKeyValueFromKey(firstKey).getRow();
196   }
197 
198   /**
199    * TODO left from {@link HFile} version 1: move this to StoreFile after
200    * Ryan's patch goes in to eliminate {@link KeyValue} here.
201    *
202    * @return the last row key, or null if the file is empty.
203    */
204   @Override
205   public byte[] getLastRowKey() {
206     byte[] lastKey = getLastKey();
207     if (lastKey == null)
208       return null;
209     return KeyValue.createKeyValueFromKey(lastKey).getRow();
210   }
211 
212   /** @return number of KV entries in this HFile */
213   @Override
214   public long getEntries() {
215     return trailer.getEntryCount();
216   }
217 
218   /** @return comparator */
219   @Override
220   public KVComparator getComparator() {
221     return comparator;
222   }
223 
224   /** @return compression algorithm */
225   @Override
226   public Compression.Algorithm getCompressionAlgorithm() {
227     return compressAlgo;
228   }
229 
230   /**
231    * @return the total heap size of data and meta block indexes in bytes. Does
232    *         not take into account non-root blocks of a multilevel data index.
233    */
234   public long indexSize() {
235     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
236         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
237             : 0);
238   }
239 
240   @Override
241   public String getName() {
242     return name;
243   }
244 
245   @Override
246   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
247     return dataBlockIndexReader;
248   }
249 
250   @Override
251   public FixedFileTrailer getTrailer() {
252     return trailer;
253   }
254 
255   @Override
256   public boolean isPrimaryReplicaReader() {
257     return isPrimaryReplicaReader;
258   }
259 
260   @Override
261   public void setPrimaryReplicaReader(boolean isPrimaryReplicaReader) {
262     this.isPrimaryReplicaReader = isPrimaryReplicaReader;
263   }
264 
265   public FileInfo loadFileInfo() throws IOException {
266     return fileInfo;
267   }
268 
269   /**
270    * An exception thrown when an operation requiring a scanner to be seeked
271    * is invoked on a scanner that is not seeked.
272    */
273   @SuppressWarnings("serial")
274   public static class NotSeekedException extends IllegalStateException {
275     public NotSeekedException() {
276       super("Not seeked to a key/value");
277     }
278   }
279 
280   protected static abstract class Scanner implements HFileScanner {
281     protected ByteBuffer blockBuffer;
282 
283     protected boolean cacheBlocks;
284     protected final boolean pread;
285     protected final boolean isCompaction;
286 
287     protected int currKeyLen;
288     protected int currValueLen;
289     protected int currMemstoreTSLen;
290     protected long currMemstoreTS;
291 
292     protected int blockFetches;
293 
294     protected final HFile.Reader reader;
295 
296     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
297         final boolean pread, final boolean isCompaction) {
298       this.reader = reader;
299       this.cacheBlocks = cacheBlocks;
300       this.pread = pread;
301       this.isCompaction = isCompaction;
302     }
303 
304     @Override
305     public boolean isSeeked(){
306       return blockBuffer != null;
307     }
308 
309     @Override
310     public String toString() {
311       return "HFileScanner for reader " + String.valueOf(getReader());
312     }
313 
314     protected void assertSeeked() {
315       if (!isSeeked())
316         throw new NotSeekedException();
317     }
318 
319     @Override
320     public int seekTo(byte[] key) throws IOException {
321       return seekTo(key, 0, key.length);
322     }
323     
324     @Override
325     public boolean seekBefore(byte[] key) throws IOException {
326       return seekBefore(key, 0, key.length);
327     }
328     
329     @Override
330     public int reseekTo(byte[] key) throws IOException {
331       return reseekTo(key, 0, key.length);
332     }
333 
334     @Override
335     public HFile.Reader getReader() {
336       return reader;
337     }
338 
339     @Override
340     public void close() {
341       if (!pread) {
342         // For seek + pread stream socket should be closed when the scanner is closed. HBASE-9393
343         reader.unbufferStream();
344       }
345     }
346   }
347 
348   /** For testing */
349   abstract HFileBlock.FSReader getUncachedBlockReader();
350 
351   public Path getPath() {
352     return path;
353   }
354 
355   @Override
356   public DataBlockEncoding getDataBlockEncoding() {
357     return dataBlockEncoder.getDataBlockEncoding();
358   }
359 
360   public abstract int getMajorVersion();
361 
362   @Override
363   public Configuration getConf() {
364     return conf;
365   }
366 
367   @Override
368   public void setConf(Configuration conf) {
369     this.conf = conf;
370   }
371 }