View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Map;
27  import java.util.Random;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.fs.FSDataInputStream;
32  import org.apache.hadoop.fs.FSDataOutputStream;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HBaseTestCase;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.KeyValue.Type;
41  import org.apache.hadoop.hbase.Tag;
42  import org.apache.hadoop.hbase.io.compress.Compression;
43  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
44  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
45  import org.apache.hadoop.hbase.regionserver.StoreFile;
46  import org.apache.hadoop.hbase.testclassification.SmallTests;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.io.Writable;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * test hfile features.
54   * <p>
55   * Copied from
56   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
57   * Remove after tfile is committed and use the tfile version of this class
58   * instead.</p>
59   */
60  @Category(SmallTests.class)
61  public class TestHFile extends HBaseTestCase {
62    static final Log LOG = LogFactory.getLog(TestHFile.class);
63  
64    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65    private static String ROOT_DIR =
66      TEST_UTIL.getDataTestDir("TestHFile").toString();
67    private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
68    private final int minBlockSize = 512;
69    private static String localFormatter = "%010d";
70    private static CacheConfig cacheConf = null;
71    private Map<String, Long> startingMetrics;
72  
73    @Override
74    public void setUp() throws Exception {
75      super.setUp();
76    }
77  
78    @Override
79    public void tearDown() throws Exception {
80      super.tearDown();
81    }
82  
83  
84    /**
85     * Test empty HFile.
86     * Test all features work reasonably when hfile is empty of entries.
87     * @throws IOException
88     */
89    public void testEmptyHFile() throws IOException {
90      if (cacheConf == null) cacheConf = new CacheConfig(conf);
91      Path f = new Path(ROOT_DIR, getName());
92      HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
93      Writer w =
94          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
95      w.close();
96      Reader r = HFile.createReader(fs, f, cacheConf, conf);
97      r.loadFileInfo();
98      assertNull(r.getFirstKey());
99      assertNull(r.getLastKey());
100   }
101 
102   /**
103    * Create 0-length hfile and show that it fails
104    */
105   public void testCorrupt0LengthHFile() throws IOException {
106     if (cacheConf == null) cacheConf = new CacheConfig(conf);
107     Path f = new Path(ROOT_DIR, getName());
108     FSDataOutputStream fsos = fs.create(f);
109     fsos.close();
110 
111     try {
112       Reader r = HFile.createReader(fs, f, cacheConf, conf);
113     } catch (CorruptHFileException che) {
114       // Expected failure
115       return;
116     }
117     fail("Should have thrown exception");
118   }
119 
120   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
121     FileStatus fst = fs.getFileStatus(src);
122     long len = fst.getLen();
123     len = len / 2 ;
124 
125     // create a truncated hfile
126     FSDataOutputStream fdos = fs.create(dst);
127     byte[] buf = new byte[(int)len];
128     FSDataInputStream fdis = fs.open(src);
129     fdis.read(buf);
130     fdos.write(buf);
131     fdis.close();
132     fdos.close();
133   }
134 
135   /**
136    * Create a truncated hfile and verify that exception thrown.
137    */
138   public void testCorruptTruncatedHFile() throws IOException {
139     if (cacheConf == null) cacheConf = new CacheConfig(conf);
140     Path f = new Path(ROOT_DIR, getName());
141     HFileContext  context = new HFileContextBuilder().build();
142     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
143         .withFileContext(context).create();
144     writeSomeRecords(w, 0, 100, false);
145     w.close();
146 
147     Path trunc = new Path(f.getParent(), "trucated");
148     truncateFile(fs, w.getPath(), trunc);
149 
150     try {
151       Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
152     } catch (CorruptHFileException che) {
153       // Expected failure
154       return;
155     }
156     fail("Should have thrown exception");
157   }
158 
159   // write some records into the tfile
160   // write them twice
161   private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
162       throws IOException {
163     String value = "value";
164     KeyValue kv;
165     for (int i = start; i < (start + n); i++) {
166       String key = String.format(localFormatter, Integer.valueOf(i));
167       if (useTags) {
168         Tag t = new Tag((byte) 1, "myTag1");
169         Tag[] tags = new Tag[1];
170         tags[0] = t;
171         kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
172             HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
173         writer.append(kv);
174       } else {
175         kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
176             Bytes.toBytes(value + key));
177         writer.append(kv);
178       }
179     }
180     return (start + n);
181   }
182 
183   private void readAllRecords(HFileScanner scanner) throws IOException {
184     readAndCheckbytes(scanner, 0, 100);
185   }
186 
187   // read the records and check
188   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
189       throws IOException {
190     String value = "value";
191     int i = start;
192     for (; i < (start + n); i++) {
193       ByteBuffer key = scanner.getKey();
194       ByteBuffer val = scanner.getValue();
195       String keyStr = String.format(localFormatter, Integer.valueOf(i));
196       String valStr = value + keyStr;
197       KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
198           Bytes.toBytes("qual"), Bytes.toBytes(valStr));
199       byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
200           Bytes.toBytes(key).length).getKey();
201       assertTrue("bytes for keys do not match " + keyStr + " " +
202         Bytes.toString(Bytes.toBytes(key)),
203           Arrays.equals(kv.getKey(), keyBytes));
204       byte [] valBytes = Bytes.toBytes(val);
205       assertTrue("bytes for vals do not match " + valStr + " " +
206         Bytes.toString(valBytes),
207         Arrays.equals(Bytes.toBytes(valStr), valBytes));
208       if (!scanner.next()) {
209         break;
210       }
211     }
212     assertEquals(i, start + n - 1);
213     return (start + n);
214   }
215 
216   private byte[] getSomeKey(int rowId) {
217     KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(),
218         Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
219     return kv.getKey();
220   }
221 
222   private void writeRecords(Writer writer, boolean useTags) throws IOException {
223     writeSomeRecords(writer, 0, 100, useTags);
224     writer.close();
225   }
226 
227   private FSDataOutputStream createFSOutput(Path name) throws IOException {
228     //if (fs.exists(name)) fs.delete(name, true);
229     FSDataOutputStream fout = fs.create(name);
230     return fout;
231   }
232 
233   /**
234    * test none codecs
235    * @param useTags
236    */
237   void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
238     if (useTags) {
239       conf.setInt("hfile.format.version", 3);
240     }
241     if (cacheConf == null) cacheConf = new CacheConfig(conf);
242     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
243     FSDataOutputStream fout = createFSOutput(ncTFile);
244     HFileContext meta = new HFileContextBuilder()
245                         .withBlockSize(minBlockSize)
246                         .withCompression(AbstractHFileWriter.compressionByName(codec))
247                         .build();
248     Writer writer = HFile.getWriterFactory(conf, cacheConf)
249         .withOutputStream(fout)
250         .withFileContext(meta)
251         .withComparator(new KeyValue.KVComparator())
252         .create();
253     LOG.info(writer);
254     writeRecords(writer, useTags);
255     fout.close();
256     FSDataInputStream fin = fs.open(ncTFile);
257     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
258       fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
259     System.out.println(cacheConf.toString());
260     // Load up the index.
261     reader.loadFileInfo();
262     // Get a scanner that caches and that does not use pread.
263     HFileScanner scanner = reader.getScanner(true, false);
264     // Align scanner at start of the file.
265     scanner.seekTo();
266     readAllRecords(scanner);
267     int seekTo = scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50)));
268     System.out.println(seekTo);
269     assertTrue("location lookup failed",
270         scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))) == 0);
271     // read the key and see if it matches
272     ByteBuffer readKey = scanner.getKey();
273     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
274       Bytes.toBytes(readKey)));
275 
276     scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
277     ByteBuffer val1 = scanner.getValue();
278     scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
279     ByteBuffer val2 = scanner.getValue();
280     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
281 
282     reader.close();
283     fin.close();
284     fs.delete(ncTFile, true);
285   }
286 
287   public void testTFileFeatures() throws IOException {
288     testTFilefeaturesInternals(false);
289     testTFilefeaturesInternals(true);
290   }
291 
292   protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
293     basicWithSomeCodec("none", useTags);
294     basicWithSomeCodec("gz", useTags);
295   }
296 
297   private void writeNumMetablocks(Writer writer, int n) {
298     for (int i = 0; i < n; i++) {
299       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
300         private int val;
301         public Writable setVal(int val) { this.val = val; return this; }
302 
303         @Override
304         public void write(DataOutput out) throws IOException {
305           out.write(("something to test" + val).getBytes());
306         }
307 
308         @Override
309         public void readFields(DataInput in) throws IOException { }
310       }.setVal(i));
311     }
312   }
313 
314   private void someTestingWithMetaBlock(Writer writer) {
315     writeNumMetablocks(writer, 10);
316   }
317 
318   private void readNumMetablocks(Reader reader, int n) throws IOException {
319     for (int i = 0; i < n; i++) {
320       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
321       ByteBuffer expected =
322         ByteBuffer.wrap(("something to test" + i).getBytes());
323       assertEquals("failed to match metadata",
324         Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
325     }
326   }
327 
328   private void someReadingWithMetaBlock(Reader reader) throws IOException {
329     readNumMetablocks(reader, 10);
330   }
331 
332   private void metablocks(final String compress) throws Exception {
333     if (cacheConf == null) cacheConf = new CacheConfig(conf);
334     Path mFile = new Path(ROOT_DIR, "meta.hfile");
335     FSDataOutputStream fout = createFSOutput(mFile);
336     HFileContext meta = new HFileContextBuilder()
337                         .withCompression(AbstractHFileWriter.compressionByName(compress))
338                         .withBlockSize(minBlockSize).build();
339     Writer writer = HFile.getWriterFactory(conf, cacheConf)
340         .withOutputStream(fout)
341         .withFileContext(meta)
342         .create();
343     someTestingWithMetaBlock(writer);
344     writer.close();
345     fout.close();
346     FSDataInputStream fin = fs.open(mFile);
347     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
348         this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
349     reader.loadFileInfo();
350     // No data -- this should return false.
351     assertFalse(reader.getScanner(false, false).seekTo());
352     someReadingWithMetaBlock(reader);
353     fs.delete(mFile, true);
354     reader.close();
355     fin.close();
356   }
357 
358   // test meta blocks for tfiles
359   public void testMetaBlocks() throws Exception {
360     metablocks("none");
361     metablocks("gz");
362   }
363 
364   public void testNullMetaBlocks() throws Exception {
365     if (cacheConf == null) cacheConf = new CacheConfig(conf);
366     for (Compression.Algorithm compressAlgo :
367         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
368       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
369       FSDataOutputStream fout = createFSOutput(mFile);
370       HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
371                           .withBlockSize(minBlockSize).build();
372       Writer writer = HFile.getWriterFactory(conf, cacheConf)
373           .withOutputStream(fout)
374           .withFileContext(meta)
375           .create();
376       KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
377       writer.append(kv);
378       writer.close();
379       fout.close();
380       Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
381       reader.loadFileInfo();
382       assertNull(reader.getMetaBlock("non-existant", false));
383     }
384   }
385 
386   /**
387    * Make sure the ordinals for our compression algorithms do not change on us.
388    */
389   public void testCompressionOrdinance() {
390     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
391     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
392     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
393     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
394     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
395   }
396 
397   @Test
398   public void testReaderWithoutBlockCache() throws Exception {
399     Path path = writeStoreFile();
400     try {
401       readStoreFile(path);
402     } catch (Exception e) {
403       // fail test
404       assertTrue(false);
405     }
406   }
407 
408   private void readStoreFile(Path storeFilePath) throws Exception {
409     // Open the file reader with block cache disabled.
410     HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf);
411     long offset = 0;
412     while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
413       HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null);
414       offset += block.getOnDiskSizeWithHeader();
415     }
416   }
417 
418   private Path writeStoreFile() throws IOException {
419     Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
420     HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
421     StoreFile.Writer sfw =
422         new StoreFile.WriterBuilder(conf, cacheConf, fs).withOutputDir(storeFileParentDir)
423             .withFileContext(meta).build();
424 
425     final int rowLen = 32;
426     Random RNG = new Random();
427     for (int i = 0; i < 1000; ++i) {
428       byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
429       byte[] v = RandomKeyValueUtil.randomValue(RNG);
430       int cfLen = RNG.nextInt(k.length - rowLen + 1);
431       KeyValue kv =
432           new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
433               k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
434       sfw.append(kv);
435     }
436 
437     sfw.close();
438     return sfw.getPath();
439   }
440 
441   public static KeyValue.Type generateKeyType(Random rand) {
442     if (rand.nextBoolean()) {
443       // Let's make half of KVs puts.
444       return KeyValue.Type.Put;
445     } else {
446       KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
447       if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) {
448         throw new RuntimeException("Generated an invalid key type: " + keyType + ". "
449             + "Probably the layout of KeyValue.Type has changed.");
450       }
451       return keyType;
452     }
453   }
454 
455 }
456