View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.Closeable;
23  import java.io.DataInput;
24  import java.io.DataInputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.io.SequenceInputStream;
28  import java.net.InetSocketAddress;
29  import java.nio.ByteBuffer;
30  import java.util.ArrayList;
31  import java.util.Collection;
32  import java.util.Comparator;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Set;
36  import java.util.SortedMap;
37  import java.util.TreeMap;
38  
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.hbase.classification.InterfaceAudience;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.fs.FSDataInputStream;
44  import org.apache.hadoop.fs.FSDataOutputStream;
45  import org.apache.hadoop.fs.FileStatus;
46  import org.apache.hadoop.fs.FileSystem;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.fs.PathFilter;
49  import org.apache.hadoop.hbase.Cell;
50  import org.apache.hadoop.hbase.HConstants;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.KeyValue.KVComparator;
53  import org.apache.hadoop.hbase.fs.HFileSystem;
54  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
55  import org.apache.hadoop.hbase.io.MetricsIO;
56  import org.apache.hadoop.hbase.io.MetricsIOWrapperImpl;
57  import org.apache.hadoop.hbase.io.compress.Compression;
58  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
61  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
62  import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
63  import org.apache.hadoop.hbase.util.BloomFilterWriter;
64  import org.apache.hadoop.hbase.util.ByteStringer;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.apache.hadoop.hbase.util.ChecksumType;
67  import org.apache.hadoop.hbase.util.Counter;
68  import org.apache.hadoop.hbase.util.FSUtils;
69  import org.apache.hadoop.io.Writable;
70  
71  import com.google.common.base.Preconditions;
72  
73  /**
74   * File format for hbase.
75   * A file of sorted key/value pairs. Both keys and values are byte arrays.
76   * <p>
77   * The memory footprint of a HFile includes the following (below is taken from the
78   * <a
79   * href=https://issues.apache.org/jira/browse/HADOOP-3315>TFile</a> documentation
80   * but applies also to HFile):
81   * <ul>
82   * <li>Some constant overhead of reading or writing a compressed block.
83   * <ul>
84   * <li>Each compressed block requires one compression/decompression codec for
85   * I/O.
86   * <li>Temporary space to buffer the key.
87   * <li>Temporary space to buffer the value.
88   * </ul>
89   * <li>HFile index, which is proportional to the total number of Data Blocks.
90   * The total amount of memory needed to hold the index can be estimated as
91   * (56+AvgKeySize)*NumBlocks.
92   * </ul>
93   * Suggestions on performance optimization.
94   * <ul>
95   * <li>Minimum block size. We recommend a setting of minimum block size between
96   * 8KB to 1MB for general usage. Larger block size is preferred if files are
97   * primarily for sequential access. However, it would lead to inefficient random
98   * access (because there are more data to decompress). Smaller blocks are good
99   * for random access, but require more memory to hold the block index, and may
100  * be slower to create (because we must flush the compressor stream at the
101  * conclusion of each data block, which leads to an FS I/O flush). Further, due
102  * to the internal caching in Compression codec, the smallest possible block
103  * size would be around 20KB-30KB.
104  * <li>The current implementation does not offer true multi-threading for
105  * reading. The implementation uses FSDataInputStream seek()+read(), which is
106  * shown to be much faster than positioned-read call in single thread mode.
107  * However, it also means that if multiple threads attempt to access the same
108  * HFile (using multiple scanners) simultaneously, the actual I/O is carried out
109  * sequentially even if they access different DFS blocks (Reexamine! pread seems
110  * to be 10% faster than seek+read in my testing -- stack).
111  * <li>Compression codec. Use "none" if the data is not very compressable (by
112  * compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
113  * as the starting point for experimenting. "gz" overs slightly better
114  * compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
115  * decompress, comparing to "lzo".
116  * </ul>
117  *
118  * For more on the background behind HFile, see <a
119  * href=https://issues.apache.org/jira/browse/HBASE-61>HBASE-61</a>.
120  * <p>
121  * File is made of data blocks followed by meta data blocks (if any), a fileinfo
122  * block, data block index, meta data block index, and a fixed size trailer
123  * which records the offsets at which file changes content type.
124  * <pre>&lt;data blocks>&lt;meta blocks>&lt;fileinfo>&lt;data index>&lt;meta index>&lt;trailer></pre>
125  * Each block has a bit of magic at its start.  Block are comprised of
126  * key/values.  In data blocks, they are both byte arrays.  Metadata blocks are
127  * a String key and a byte array value.  An empty file looks like this:
128  * <pre>&lt;fileinfo>&lt;trailer></pre>.  That is, there are not data nor meta
129  * blocks present.
130  * <p>
131  * TODO: Do scanners need to be able to take a start and end row?
132  * TODO: Should BlockIndex know the name of its file?  Should it have a Path
133  * that points at its file say for the case where an index lives apart from
134  * an HFile instance?
135  */
136 @InterfaceAudience.Private
137 public class HFile {
138   static final Log LOG = LogFactory.getLog(HFile.class);
139 
140   /**
141    * Maximum length of key in HFile.
142    */
143   public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
144 
145   /**
146    * Default compression: none.
147    */
148   public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
149     Compression.Algorithm.NONE;
150 
151   /** Minimum supported HFile format version */
152   public static final int MIN_FORMAT_VERSION = 2;
153 
154   /** Maximum supported HFile format version
155    */
156   public static final int MAX_FORMAT_VERSION = 3;
157 
158   /**
159    * Minimum HFile format version with support for persisting cell tags
160    */
161   public static final int MIN_FORMAT_VERSION_WITH_TAGS = 3;
162 
163   /** Default compression name: none. */
164   public final static String DEFAULT_COMPRESSION =
165     DEFAULT_COMPRESSION_ALGORITHM.getName();
166 
167   /** Meta data block name for bloom filter bits. */
168   public static final String BLOOM_FILTER_DATA_KEY = "BLOOM_FILTER_DATA";
169 
170   /**
171    * We assume that HFile path ends with
172    * ROOT_DIR/TABLE_NAME/REGION_NAME/CF_NAME/HFILE, so it has at least this
173    * many levels of nesting. This is needed for identifying table and CF name
174    * from an HFile path.
175    */
176   public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
177 
178   /**
179    * The number of bytes per checksum.
180    */
181   public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
182   // TODO: This define is done in three places.  Fix.
183   public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
184 
185   // For measuring number of checksum failures
186   static final Counter checksumFailures = new Counter();
187 
188   // for test purpose
189   public static final Counter dataBlockReadCnt = new Counter();
190 
191   /** Static instance for the metrics so that HFileReaders access the same instance */
192   static final MetricsIO metrics = new MetricsIO(new MetricsIOWrapperImpl());
193 
194   /**
195    * Number of checksum verification failures. It also
196    * clears the counter.
197    */
198   public static final long getAndResetChecksumFailuresCount() {
199     long count = checksumFailures.get();
200     checksumFailures.set(0);
201     return count;
202   }
203 
204   /**
205    * Number of checksum verification failures.
206    */
207   public static final long getChecksumFailuresCount() {
208     long count = checksumFailures.get();
209     return count;
210   }
211 
212   public static final void updateReadLatency(long latencyMillis, boolean pread) {
213     if (pread) {
214       metrics.updateFsPreadTime(latencyMillis);
215     } else {
216       metrics.updateFsReadTime(latencyMillis);
217     }
218   }
219 
220   public static final void updateWriteLatency(long latencyMillis) {
221     metrics.updateFsWriteTime(latencyMillis);
222   }
223 
224   /** API required to write an {@link HFile} */
225   public interface Writer extends Closeable {
226 
227     /** Add an element to the file info map. */
228     void appendFileInfo(byte[] key, byte[] value) throws IOException;
229 
230     void append(Cell cell) throws IOException;
231 
232     /** @return the path to this {@link HFile} */
233     Path getPath();
234 
235     /**
236      * Adds an inline block writer such as a multi-level block index writer or
237      * a compound Bloom filter writer.
238      */
239     void addInlineBlockWriter(InlineBlockWriter bloomWriter);
240 
241     // The below three methods take Writables.  We'd like to undo Writables but undoing the below would be pretty
242     // painful.  Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need
243     // to map between Message and Writable or byte [] and current Writable serialization.  This would be a bit of work
244     // to little gain.  Thats my thinking at moment.  St.Ack 20121129
245 
246     void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
247 
248     /**
249      * Store general Bloom filter in the file. This does not deal with Bloom filter
250      * internals but is necessary, since Bloom filters are stored differently
251      * in HFile version 1 and version 2.
252      */
253     void addGeneralBloomFilter(BloomFilterWriter bfw);
254 
255     /**
256      * Store delete family Bloom filter in the file, which is only supported in
257      * HFile V2.
258      */
259     void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
260 
261     /**
262      * Return the file context for the HFile this writer belongs to
263      */
264     HFileContext getFileContext();
265   }
266 
267   /**
268    * This variety of ways to construct writers is used throughout the code, and
269    * we want to be able to swap writer implementations.
270    */
271   public static abstract class WriterFactory {
272     protected final Configuration conf;
273     protected final CacheConfig cacheConf;
274     protected FileSystem fs;
275     protected Path path;
276     protected FSDataOutputStream ostream;
277     protected KVComparator comparator = KeyValue.COMPARATOR;
278     protected InetSocketAddress[] favoredNodes;
279     private HFileContext fileContext;
280 
281     WriterFactory(Configuration conf, CacheConfig cacheConf) {
282       this.conf = conf;
283       this.cacheConf = cacheConf;
284     }
285 
286     public WriterFactory withPath(FileSystem fs, Path path) {
287       Preconditions.checkNotNull(fs);
288       Preconditions.checkNotNull(path);
289       this.fs = fs;
290       this.path = path;
291       return this;
292     }
293 
294     public WriterFactory withOutputStream(FSDataOutputStream ostream) {
295       Preconditions.checkNotNull(ostream);
296       this.ostream = ostream;
297       return this;
298     }
299 
300     public WriterFactory withComparator(KVComparator comparator) {
301       Preconditions.checkNotNull(comparator);
302       this.comparator = comparator;
303       return this;
304     }
305 
306     public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
307       // Deliberately not checking for null here.
308       this.favoredNodes = favoredNodes;
309       return this;
310     }
311 
312     public WriterFactory withFileContext(HFileContext fileContext) {
313       this.fileContext = fileContext;
314       return this;
315     }
316 
317     public Writer create() throws IOException {
318       if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
319         throw new AssertionError("Please specify exactly one of " +
320             "filesystem/path or path");
321       }
322       if (path != null) {
323         ostream = AbstractHFileWriter.createOutputStream(conf, fs, path, favoredNodes);
324       }
325       return createWriter(fs, path, ostream,
326                    comparator, fileContext);
327     }
328 
329     protected abstract Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
330         KVComparator comparator, HFileContext fileContext) throws IOException;
331   }
332 
333   /** The configuration key for HFile version to use for new files */
334   public static final String FORMAT_VERSION_KEY = "hfile.format.version";
335 
336   public static int getFormatVersion(Configuration conf) {
337     int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
338     checkFormatVersion(version);
339     return version;
340   }
341 
342   /**
343    * Returns the factory to be used to create {@link HFile} writers.
344    * Disables block cache access for all writers created through the
345    * returned factory.
346    */
347   public static final WriterFactory getWriterFactoryNoCache(Configuration
348        conf) {
349     Configuration tempConf = new Configuration(conf);
350     tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
351     return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
352   }
353 
354   /**
355    * Returns the factory to be used to create {@link HFile} writers
356    */
357   public static final WriterFactory getWriterFactory(Configuration conf,
358       CacheConfig cacheConf) {
359     int version = getFormatVersion(conf);
360     switch (version) {
361     case 2:
362       return new HFileWriterV2.WriterFactoryV2(conf, cacheConf);
363     case 3:
364       return new HFileWriterV3.WriterFactoryV3(conf, cacheConf);
365     default:
366       throw new IllegalArgumentException("Cannot create writer for HFile " +
367           "format version " + version);
368     }
369   }
370 
371   /**
372    * An abstraction used by the block index.
373    * Implementations will check cache for any asked-for block and return cached block if found.
374    * Otherwise, after reading from fs, will try and put block into cache before returning.
375    */
376   public interface CachingBlockReader {
377     /**
378      * Read in a file block.
379      * @param offset offset to read.
380      * @param onDiskBlockSize size of the block
381      * @param cacheBlock
382      * @param pread
383      * @param isCompaction is this block being read as part of a compaction
384      * @param expectedBlockType the block type we are expecting to read with this read operation,
385      *  or null to read whatever block type is available and avoid checking (that might reduce
386      *  caching efficiency of encoded data blocks)
387      * @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks
388      *  to be in, or null to not perform this check and return the block irrespective of the
389      *  encoding. This check only applies to data blocks and can be set to null when the caller is
390      *  expecting to read a non-data block and has set expectedBlockType accordingly.
391      * @return Block wrapped in a ByteBuffer.
392      * @throws IOException
393      */
394     HFileBlock readBlock(long offset, long onDiskBlockSize,
395         boolean cacheBlock, final boolean pread, final boolean isCompaction,
396         final boolean updateCacheMetrics, BlockType expectedBlockType,
397         DataBlockEncoding expectedDataBlockEncoding)
398         throws IOException;
399   }
400 
401   /** An interface used by clients to open and iterate an {@link HFile}. */
402   public interface Reader extends Closeable, CachingBlockReader {
403     /**
404      * Returns this reader's "name". Usually the last component of the path.
405      * Needs to be constant as the file is being moved to support caching on
406      * write.
407      */
408     String getName();
409 
410     KVComparator getComparator();
411 
412     HFileScanner getScanner(boolean cacheBlocks, final boolean pread, final boolean isCompaction);
413 
414     ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock) throws IOException;
415 
416     Map<byte[], byte[]> loadFileInfo() throws IOException;
417 
418     byte[] getLastKey();
419 
420     byte[] midkey() throws IOException;
421 
422     long length();
423 
424     long getEntries();
425 
426     byte[] getFirstKey();
427 
428     long indexSize();
429 
430     byte[] getFirstRowKey();
431 
432     byte[] getLastRowKey();
433 
434     FixedFileTrailer getTrailer();
435 
436     HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
437 
438     HFileScanner getScanner(boolean cacheBlocks, boolean pread);
439 
440     Compression.Algorithm getCompressionAlgorithm();
441 
442     /**
443      * Retrieves general Bloom filter metadata as appropriate for each
444      * {@link HFile} version.
445      * Knows nothing about how that metadata is structured.
446      */
447     DataInput getGeneralBloomFilterMetadata() throws IOException;
448 
449     /**
450      * Retrieves delete family Bloom filter metadata as appropriate for each
451      * {@link HFile}  version.
452      * Knows nothing about how that metadata is structured.
453      */
454     DataInput getDeleteBloomFilterMetadata() throws IOException;
455 
456     Path getPath();
457 
458     /** Close method with optional evictOnClose */
459     void close(boolean evictOnClose) throws IOException;
460 
461     DataBlockEncoding getDataBlockEncoding();
462 
463     boolean hasMVCCInfo();
464 
465     /**
466      * Return the file context of the HFile this reader belongs to
467      */
468     HFileContext getFileContext();
469     
470     boolean isPrimaryReplicaReader();
471 
472     void setPrimaryReplicaReader(boolean isPrimaryReplicaReader);
473 
474     /**
475      * To close the stream's socket. Note: This can be concurrently called from multiple threads and
476      * implementation should take care of thread safety.
477      */
478     void unbufferStream();
479   }
480 
481   /**
482    * Method returns the reader given the specified arguments.
483    * TODO This is a bad abstraction.  See HBASE-6635.
484    *
485    * @param path hfile's path
486    * @param fsdis stream of path's file
487    * @param size max size of the trailer.
488    * @param cacheConf Cache configuation values, cannot be null.
489    * @param hfs
490    * @return an appropriate instance of HFileReader
491    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
492    */
493   private static Reader openReader(Path path, FSDataInputStreamWrapper fsdis,
494       long size, CacheConfig cacheConf, HFileSystem hfs, Configuration conf) throws IOException {
495     FixedFileTrailer trailer = null;
496     try {
497       boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
498       assert !isHBaseChecksum; // Initially we must read with FS checksum.
499       trailer = FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
500       switch (trailer.getMajorVersion()) {
501       case 2:
502         return new HFileReaderV2(path, trailer, fsdis, size, cacheConf, hfs, conf);
503       case 3 :
504         return new HFileReaderV3(path, trailer, fsdis, size, cacheConf, hfs, conf);
505       default:
506         throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
507       }
508     } catch (Throwable t) {
509       try {
510         fsdis.close();
511       } catch (Throwable t2) {
512         LOG.warn("Error closing fsdis FSDataInputStreamWrapper", t2);
513       }
514       throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, t);
515     } finally {
516       fsdis.unbuffer();
517     }
518   }
519 
520   /**
521    * The sockets and the file descriptors held by the method parameter
522    * {@code FSDataInputStreamWrapper} passed will be freed after its usage so caller needs to ensure
523    * that no other threads have access to the same passed reference.
524    * @param fs A file system
525    * @param path Path to HFile
526    * @param fsdis a stream of path's file
527    * @param size max size of the trailer.
528    * @param cacheConf Cache configuration for hfile's contents
529    * @param conf Configuration
530    * @return A version specific Hfile Reader
531    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
532    */
533   public static Reader createReader(FileSystem fs, Path path,
534       FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
535       throws IOException {
536     HFileSystem hfs = null;
537 
538     // If the fs is not an instance of HFileSystem, then create an
539     // instance of HFileSystem that wraps over the specified fs.
540     // In this case, we will not be able to avoid checksumming inside
541     // the filesystem.
542     if (!(fs instanceof HFileSystem)) {
543       hfs = new HFileSystem(fs);
544     } else {
545       hfs = (HFileSystem)fs;
546     }
547     return openReader(path, fsdis, size, cacheConf, hfs, conf);
548   }
549 
550   /**
551   * Creates reader with cache configuration disabled
552   * @param fs filesystem
553   * @param path Path to file to read
554   * @return an active Reader instance
555   * @throws IOException Will throw a CorruptHFileException
556   * (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
557   */
558  public static Reader createReader(
559      FileSystem fs, Path path,  Configuration conf) throws IOException {
560      return createReader(fs, path, CacheConfig.DISABLED, conf);
561  }
562 
563   /**
564    *
565    * @param fs filesystem
566    * @param path Path to file to read
567    * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
568    * @return an active Reader instance
569    * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
570    */
571   public static Reader createReader(
572       FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
573     Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
574     FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
575     return openReader(path, stream, fs.getFileStatus(path).getLen(),
576       cacheConf, stream.getHfs(), conf);
577   }
578 
579   /**
580    * This factory method is used only by unit tests. <br/>
581    * The sockets and the file descriptors held by the method parameter
582    * {@code FSDataInputStreamWrapper} passed will be freed after its usage so caller needs to ensure
583    * that no other threads have access to the same passed reference.
584    */
585   static Reader createReaderFromStream(Path path,
586       FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
587       throws IOException {
588     FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
589     return openReader(path, wrapper, size, cacheConf, null, conf);
590   }
591 
592   /**
593    * Returns true if the specified file has a valid HFile Trailer.
594    * @param fs filesystem
595    * @param path Path to file to verify
596    * @return true if the file has a valid HFile Trailer, otherwise false
597    * @throws IOException if failed to read from the underlying stream
598    */
599   public static boolean isHFileFormat(final FileSystem fs, final Path path) throws IOException {
600     return isHFileFormat(fs, fs.getFileStatus(path));
601   }
602 
603   /**
604    * Returns true if the specified file has a valid HFile Trailer.
605    * @param fs filesystem
606    * @param fileStatus the file to verify
607    * @return true if the file has a valid HFile Trailer, otherwise false
608    * @throws IOException if failed to read from the underlying stream
609    */
610   public static boolean isHFileFormat(final FileSystem fs, final FileStatus fileStatus)
611       throws IOException {
612     final Path path = fileStatus.getPath();
613     final long size = fileStatus.getLen();
614     FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path);
615     try {
616       boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
617       assert !isHBaseChecksum; // Initially we must read with FS checksum.
618       FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
619       return true;
620     } catch (IllegalArgumentException e) {
621       return false;
622     } catch (IOException e) {
623       throw e;
624     } finally {
625       try {
626         fsdis.close();
627       } catch (Throwable t) {
628         LOG.warn("Error closing fsdis FSDataInputStreamWrapper: " + path, t);
629       }
630     }
631   }
632 
633   /**
634    * Metadata for this file. Conjured by the writer. Read in by the reader.
635    */
636   public static class FileInfo implements SortedMap<byte[], byte[]> {
637     static final String RESERVED_PREFIX = "hfile.";
638     static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
639     static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
640     static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
641     static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
642     static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
643     static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
644     static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
645     public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
646     private final SortedMap<byte [], byte []> map = new TreeMap<byte [], byte []>(Bytes.BYTES_COMPARATOR);
647 
648     public FileInfo() {
649       super();
650     }
651 
652     /**
653      * Append the given key/value pair to the file info, optionally checking the
654      * key prefix.
655      *
656      * @param k key to add
657      * @param v value to add
658      * @param checkPrefix whether to check that the provided key does not start
659      *          with the reserved prefix
660      * @return this file info object
661      * @throws IOException if the key or value is invalid
662      */
663     public FileInfo append(final byte[] k, final byte[] v,
664         final boolean checkPrefix) throws IOException {
665       if (k == null || v == null) {
666         throw new NullPointerException("Key nor value may be null");
667       }
668       if (checkPrefix && isReservedFileInfoKey(k)) {
669         throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX
670             + " are reserved");
671       }
672       put(k, v);
673       return this;
674     }
675 
676     @Override
677     public void clear() {
678       this.map.clear();
679     }
680 
681     @Override
682     public Comparator<? super byte[]> comparator() {
683       return map.comparator();
684     }
685 
686     @Override
687     public boolean containsKey(Object key) {
688       return map.containsKey(key);
689     }
690 
691     @Override
692     public boolean containsValue(Object value) {
693       return map.containsValue(value);
694     }
695 
696     @Override
697     public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() {
698       return map.entrySet();
699     }
700 
701     @Override
702     public boolean equals(Object o) {
703       return map.equals(o);
704     }
705 
706     @Override
707     public byte[] firstKey() {
708       return map.firstKey();
709     }
710 
711     @Override
712     public byte[] get(Object key) {
713       return map.get(key);
714     }
715 
716     @Override
717     public int hashCode() {
718       return map.hashCode();
719     }
720 
721     @Override
722     public SortedMap<byte[], byte[]> headMap(byte[] toKey) {
723       return this.map.headMap(toKey);
724     }
725 
726     @Override
727     public boolean isEmpty() {
728       return map.isEmpty();
729     }
730 
731     @Override
732     public Set<byte[]> keySet() {
733       return map.keySet();
734     }
735 
736     @Override
737     public byte[] lastKey() {
738       return map.lastKey();
739     }
740 
741     @Override
742     public byte[] put(byte[] key, byte[] value) {
743       return this.map.put(key, value);
744     }
745 
746     @Override
747     public void putAll(Map<? extends byte[], ? extends byte[]> m) {
748       this.map.putAll(m);
749     }
750 
751     @Override
752     public byte[] remove(Object key) {
753       return this.map.remove(key);
754     }
755 
756     @Override
757     public int size() {
758       return map.size();
759     }
760 
761     @Override
762     public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) {
763       return this.map.subMap(fromKey, toKey);
764     }
765 
766     @Override
767     public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) {
768       return this.map.tailMap(fromKey);
769     }
770 
771     @Override
772     public Collection<byte[]> values() {
773       return map.values();
774     }
775 
776     /**
777      * Write out this instance on the passed in <code>out</code> stream.
778      * We write it as a protobuf.
779      * @param out
780      * @throws IOException
781      * @see #read(DataInputStream)
782      */
783     void write(final DataOutputStream out) throws IOException {
784       HFileProtos.FileInfoProto.Builder builder = HFileProtos.FileInfoProto.newBuilder();
785       for (Map.Entry<byte [], byte[]> e: this.map.entrySet()) {
786         HBaseProtos.BytesBytesPair.Builder bbpBuilder = HBaseProtos.BytesBytesPair.newBuilder();
787         bbpBuilder.setFirst(ByteStringer.wrap(e.getKey()));
788         bbpBuilder.setSecond(ByteStringer.wrap(e.getValue()));
789         builder.addMapEntry(bbpBuilder.build());
790       }
791       out.write(ProtobufUtil.PB_MAGIC);
792       builder.build().writeDelimitedTo(out);
793     }
794 
795     /**
796      * Populate this instance with what we find on the passed in <code>in</code> stream.
797      * Can deserialize protobuf of old Writables format.
798      * @param in
799      * @throws IOException
800      * @see #write(DataOutputStream)
801      */
802     void read(final DataInputStream in) throws IOException {
803       // This code is tested over in TestHFileReaderV1 where we read an old hfile w/ this new code.
804       int pblen = ProtobufUtil.lengthOfPBMagic();
805       byte [] pbuf = new byte[pblen];
806       if (in.markSupported()) in.mark(pblen);
807       int read = in.read(pbuf);
808       if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen);
809       if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
810         parsePB(HFileProtos.FileInfoProto.parseDelimitedFrom(in));
811       } else {
812         if (in.markSupported()) {
813           in.reset();
814           parseWritable(in);
815         } else {
816           // We cannot use BufferedInputStream, it consumes more than we read from the underlying IS
817           ByteArrayInputStream bais = new ByteArrayInputStream(pbuf);
818           SequenceInputStream sis = new SequenceInputStream(bais, in); // Concatenate input streams
819           // TODO: Am I leaking anything here wrapping the passed in stream?  We are not calling close on the wrapped
820           // streams but they should be let go after we leave this context?  I see that we keep a reference to the
821           // passed in inputstream but since we no longer have a reference to this after we leave, we should be ok.
822           parseWritable(new DataInputStream(sis));
823         }
824       }
825     }
826 
827     /** Now parse the old Writable format.  It was a list of Map entries.  Each map entry was a key and a value of
828      * a byte [].  The old map format had a byte before each entry that held a code which was short for the key or
829      * value type.  We know it was a byte [] so in below we just read and dump it.
830      * @throws IOException
831      */
832     void parseWritable(final DataInputStream in) throws IOException {
833       // First clear the map.  Otherwise we will just accumulate entries every time this method is called.
834       this.map.clear();
835       // Read the number of entries in the map
836       int entries = in.readInt();
837       // Then read each key/value pair
838       for (int i = 0; i < entries; i++) {
839         byte [] key = Bytes.readByteArray(in);
840         // We used to read a byte that encoded the class type.  Read and ignore it because it is always byte [] in hfile
841         in.readByte();
842         byte [] value = Bytes.readByteArray(in);
843         this.map.put(key, value);
844       }
845     }
846 
847     /**
848      * Fill our map with content of the pb we read off disk
849      * @param fip protobuf message to read
850      */
851     void parsePB(final HFileProtos.FileInfoProto fip) {
852       this.map.clear();
853       for (BytesBytesPair pair: fip.getMapEntryList()) {
854         this.map.put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
855       }
856     }
857   }
858 
859   /** Return true if the given file info key is reserved for internal use. */
860   public static boolean isReservedFileInfoKey(byte[] key) {
861     return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
862   }
863 
864   /**
865    * Get names of supported compression algorithms. The names are acceptable by
866    * HFile.Writer.
867    *
868    * @return Array of strings, each represents a supported compression
869    *         algorithm. Currently, the following compression algorithms are
870    *         supported.
871    *         <ul>
872    *         <li>"none" - No compression.
873    *         <li>"gz" - GZIP compression.
874    *         </ul>
875    */
876   public static String[] getSupportedCompressionAlgorithms() {
877     return Compression.getSupportedAlgorithms();
878   }
879 
880   // Utility methods.
881   /*
882    * @param l Long to convert to an int.
883    * @return <code>l</code> cast as an int.
884    */
885   static int longToInt(final long l) {
886     // Expecting the size() of a block not exceeding 4GB. Assuming the
887     // size() will wrap to negative integer if it exceeds 2GB (From tfile).
888     return (int)(l & 0x00000000ffffffffL);
889   }
890 
891   /**
892    * Returns all HFiles belonging to the given region directory. Could return an
893    * empty list.
894    *
895    * @param fs  The file system reference.
896    * @param regionDir  The region directory to scan.
897    * @return The list of files found.
898    * @throws IOException When scanning the files fails.
899    */
900   static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
901       throws IOException {
902     List<Path> regionHFiles = new ArrayList<Path>();
903     PathFilter dirFilter = new FSUtils.DirFilter(fs);
904     FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
905     for(FileStatus dir : familyDirs) {
906       FileStatus[] files = fs.listStatus(dir.getPath());
907       for (FileStatus file : files) {
908         if (!file.isDirectory() &&
909             (!file.getPath().toString().contains(HConstants.HREGION_OLDLOGDIR_NAME)) &&
910             (!file.getPath().toString().contains(HConstants.RECOVERED_EDITS_DIR))) {
911           regionHFiles.add(file.getPath());
912         }
913       }
914     }
915     return regionHFiles;
916   }
917 
918   /**
919    * Checks the given {@link HFile} format version, and throws an exception if
920    * invalid. Note that if the version number comes from an input file and has
921    * not been verified, the caller needs to re-throw an {@link IOException} to
922    * indicate that this is not a software error, but corrupted input.
923    *
924    * @param version an HFile version
925    * @throws IllegalArgumentException if the version is invalid
926    */
927   public static void checkFormatVersion(int version)
928       throws IllegalArgumentException {
929     if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
930       throw new IllegalArgumentException("Invalid HFile version: " + version
931           + " (expected to be " + "between " + MIN_FORMAT_VERSION + " and "
932           + MAX_FORMAT_VERSION + ")");
933     }
934   }
935 
936   public static void main(String[] args) throws Exception {
937     // delegate to preserve old behavior
938     HFilePrettyPrinter.main(args);
939   }
940 }