1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Arrays;
26 import java.util.Map;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.fs.FSDataInputStream;
32 import org.apache.hadoop.fs.FSDataOutputStream;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HBaseTestCase;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.KeyValue.Type;
41 import org.apache.hadoop.hbase.Tag;
42 import org.apache.hadoop.hbase.io.compress.Compression;
43 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
44 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
45 import org.apache.hadoop.hbase.regionserver.StoreFile;
46 import org.apache.hadoop.hbase.testclassification.SmallTests;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.io.Writable;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52
53
54
55
56
57
58
59
60 @Category(SmallTests.class)
61 public class TestHFile extends HBaseTestCase {
62 static final Log LOG = LogFactory.getLog(TestHFile.class);
63
64 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private static String ROOT_DIR =
66 TEST_UTIL.getDataTestDir("TestHFile").toString();
67 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
68 private final int minBlockSize = 512;
69 private static String localFormatter = "%010d";
70 private static CacheConfig cacheConf = null;
71 private Map<String, Long> startingMetrics;
72
73 @Override
74 public void setUp() throws Exception {
75 super.setUp();
76 }
77
78 @Override
79 public void tearDown() throws Exception {
80 super.tearDown();
81 }
82
83
84
85
86
87
88
89 public void testEmptyHFile() throws IOException {
90 if (cacheConf == null) cacheConf = new CacheConfig(conf);
91 Path f = new Path(ROOT_DIR, getName());
92 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
93 Writer w =
94 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
95 w.close();
96 Reader r = HFile.createReader(fs, f, cacheConf, conf);
97 r.loadFileInfo();
98 assertNull(r.getFirstKey());
99 assertNull(r.getLastKey());
100 }
101
102
103
104
105 public void testCorrupt0LengthHFile() throws IOException {
106 if (cacheConf == null) cacheConf = new CacheConfig(conf);
107 Path f = new Path(ROOT_DIR, getName());
108 FSDataOutputStream fsos = fs.create(f);
109 fsos.close();
110
111 try {
112 Reader r = HFile.createReader(fs, f, cacheConf, conf);
113 } catch (CorruptHFileException che) {
114
115 return;
116 }
117 fail("Should have thrown exception");
118 }
119
120 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
121 FileStatus fst = fs.getFileStatus(src);
122 long len = fst.getLen();
123 len = len / 2 ;
124
125
126 FSDataOutputStream fdos = fs.create(dst);
127 byte[] buf = new byte[(int)len];
128 FSDataInputStream fdis = fs.open(src);
129 fdis.read(buf);
130 fdos.write(buf);
131 fdis.close();
132 fdos.close();
133 }
134
135
136
137
138 public void testCorruptTruncatedHFile() throws IOException {
139 if (cacheConf == null) cacheConf = new CacheConfig(conf);
140 Path f = new Path(ROOT_DIR, getName());
141 HFileContext context = new HFileContextBuilder().build();
142 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
143 .withFileContext(context).create();
144 writeSomeRecords(w, 0, 100, false);
145 w.close();
146
147 Path trunc = new Path(f.getParent(), "trucated");
148 truncateFile(fs, w.getPath(), trunc);
149
150 try {
151 Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
152 } catch (CorruptHFileException che) {
153
154 return;
155 }
156 fail("Should have thrown exception");
157 }
158
159
160
161 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
162 throws IOException {
163 String value = "value";
164 KeyValue kv;
165 for (int i = start; i < (start + n); i++) {
166 String key = String.format(localFormatter, Integer.valueOf(i));
167 if (useTags) {
168 Tag t = new Tag((byte) 1, "myTag1");
169 Tag[] tags = new Tag[1];
170 tags[0] = t;
171 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
172 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
173 writer.append(kv);
174 } else {
175 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
176 Bytes.toBytes(value + key));
177 writer.append(kv);
178 }
179 }
180 return (start + n);
181 }
182
183 private void readAllRecords(HFileScanner scanner) throws IOException {
184 readAndCheckbytes(scanner, 0, 100);
185 }
186
187
188 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
189 throws IOException {
190 String value = "value";
191 int i = start;
192 for (; i < (start + n); i++) {
193 ByteBuffer key = scanner.getKey();
194 ByteBuffer val = scanner.getValue();
195 String keyStr = String.format(localFormatter, Integer.valueOf(i));
196 String valStr = value + keyStr;
197 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
198 Bytes.toBytes("qual"), Bytes.toBytes(valStr));
199 byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
200 Bytes.toBytes(key).length).getKey();
201 assertTrue("bytes for keys do not match " + keyStr + " " +
202 Bytes.toString(Bytes.toBytes(key)),
203 Arrays.equals(kv.getKey(), keyBytes));
204 byte [] valBytes = Bytes.toBytes(val);
205 assertTrue("bytes for vals do not match " + valStr + " " +
206 Bytes.toString(valBytes),
207 Arrays.equals(Bytes.toBytes(valStr), valBytes));
208 if (!scanner.next()) {
209 break;
210 }
211 }
212 assertEquals(i, start + n - 1);
213 return (start + n);
214 }
215
216 private byte[] getSomeKey(int rowId) {
217 KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(),
218 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
219 return kv.getKey();
220 }
221
222 private void writeRecords(Writer writer, boolean useTags) throws IOException {
223 writeSomeRecords(writer, 0, 100, useTags);
224 writer.close();
225 }
226
227 private FSDataOutputStream createFSOutput(Path name) throws IOException {
228
229 FSDataOutputStream fout = fs.create(name);
230 return fout;
231 }
232
233
234
235
236
237 void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
238 if (useTags) {
239 conf.setInt("hfile.format.version", 3);
240 }
241 if (cacheConf == null) cacheConf = new CacheConfig(conf);
242 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
243 FSDataOutputStream fout = createFSOutput(ncTFile);
244 HFileContext meta = new HFileContextBuilder()
245 .withBlockSize(minBlockSize)
246 .withCompression(AbstractHFileWriter.compressionByName(codec))
247 .build();
248 Writer writer = HFile.getWriterFactory(conf, cacheConf)
249 .withOutputStream(fout)
250 .withFileContext(meta)
251 .withComparator(new KeyValue.KVComparator())
252 .create();
253 LOG.info(writer);
254 writeRecords(writer, useTags);
255 fout.close();
256 FSDataInputStream fin = fs.open(ncTFile);
257 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
258 fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
259 System.out.println(cacheConf.toString());
260
261 reader.loadFileInfo();
262
263 HFileScanner scanner = reader.getScanner(true, false);
264
265 scanner.seekTo();
266 readAllRecords(scanner);
267 int seekTo = scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50)));
268 System.out.println(seekTo);
269 assertTrue("location lookup failed",
270 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))) == 0);
271
272 ByteBuffer readKey = scanner.getKey();
273 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
274 Bytes.toBytes(readKey)));
275
276 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
277 ByteBuffer val1 = scanner.getValue();
278 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
279 ByteBuffer val2 = scanner.getValue();
280 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
281
282 reader.close();
283 fin.close();
284 fs.delete(ncTFile, true);
285 }
286
287 public void testTFileFeatures() throws IOException {
288 testTFilefeaturesInternals(false);
289 testTFilefeaturesInternals(true);
290 }
291
292 protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
293 basicWithSomeCodec("none", useTags);
294 basicWithSomeCodec("gz", useTags);
295 }
296
297 private void writeNumMetablocks(Writer writer, int n) {
298 for (int i = 0; i < n; i++) {
299 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
300 private int val;
301 public Writable setVal(int val) { this.val = val; return this; }
302
303 @Override
304 public void write(DataOutput out) throws IOException {
305 out.write(("something to test" + val).getBytes());
306 }
307
308 @Override
309 public void readFields(DataInput in) throws IOException { }
310 }.setVal(i));
311 }
312 }
313
314 private void someTestingWithMetaBlock(Writer writer) {
315 writeNumMetablocks(writer, 10);
316 }
317
318 private void readNumMetablocks(Reader reader, int n) throws IOException {
319 for (int i = 0; i < n; i++) {
320 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
321 ByteBuffer expected =
322 ByteBuffer.wrap(("something to test" + i).getBytes());
323 assertEquals("failed to match metadata",
324 Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
325 }
326 }
327
328 private void someReadingWithMetaBlock(Reader reader) throws IOException {
329 readNumMetablocks(reader, 10);
330 }
331
332 private void metablocks(final String compress) throws Exception {
333 if (cacheConf == null) cacheConf = new CacheConfig(conf);
334 Path mFile = new Path(ROOT_DIR, "meta.hfile");
335 FSDataOutputStream fout = createFSOutput(mFile);
336 HFileContext meta = new HFileContextBuilder()
337 .withCompression(AbstractHFileWriter.compressionByName(compress))
338 .withBlockSize(minBlockSize).build();
339 Writer writer = HFile.getWriterFactory(conf, cacheConf)
340 .withOutputStream(fout)
341 .withFileContext(meta)
342 .create();
343 someTestingWithMetaBlock(writer);
344 writer.close();
345 fout.close();
346 FSDataInputStream fin = fs.open(mFile);
347 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
348 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
349 reader.loadFileInfo();
350
351 assertFalse(reader.getScanner(false, false).seekTo());
352 someReadingWithMetaBlock(reader);
353 fs.delete(mFile, true);
354 reader.close();
355 fin.close();
356 }
357
358
359 public void testMetaBlocks() throws Exception {
360 metablocks("none");
361 metablocks("gz");
362 }
363
364 public void testNullMetaBlocks() throws Exception {
365 if (cacheConf == null) cacheConf = new CacheConfig(conf);
366 for (Compression.Algorithm compressAlgo :
367 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
368 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
369 FSDataOutputStream fout = createFSOutput(mFile);
370 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
371 .withBlockSize(minBlockSize).build();
372 Writer writer = HFile.getWriterFactory(conf, cacheConf)
373 .withOutputStream(fout)
374 .withFileContext(meta)
375 .create();
376 KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
377 writer.append(kv);
378 writer.close();
379 fout.close();
380 Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
381 reader.loadFileInfo();
382 assertNull(reader.getMetaBlock("non-existant", false));
383 }
384 }
385
386
387
388
389 public void testCompressionOrdinance() {
390 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
391 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
392 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
393 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
394 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
395 }
396
397 @Test
398 public void testReaderWithoutBlockCache() throws Exception {
399 Path path = writeStoreFile();
400 try {
401 readStoreFile(path);
402 } catch (Exception e) {
403
404 assertTrue(false);
405 }
406 }
407
408 private void readStoreFile(Path storeFilePath) throws Exception {
409
410 HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf);
411 long offset = 0;
412 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
413 HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null);
414 offset += block.getOnDiskSizeWithHeader();
415 }
416 }
417
418 private Path writeStoreFile() throws IOException {
419 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
420 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
421 StoreFile.Writer sfw =
422 new StoreFile.WriterBuilder(conf, cacheConf, fs).withOutputDir(storeFileParentDir)
423 .withFileContext(meta).build();
424
425 final int rowLen = 32;
426 Random RNG = new Random();
427 for (int i = 0; i < 1000; ++i) {
428 byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
429 byte[] v = RandomKeyValueUtil.randomValue(RNG);
430 int cfLen = RNG.nextInt(k.length - rowLen + 1);
431 KeyValue kv =
432 new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
433 k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
434 sfw.append(kv);
435 }
436
437 sfw.close();
438 return sfw.getPath();
439 }
440
441 public static KeyValue.Type generateKeyType(Random rand) {
442 if (rand.nextBoolean()) {
443
444 return KeyValue.Type.Put;
445 } else {
446 KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
447 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) {
448 throw new RuntimeException("Generated an invalid key type: " + keyType + ". "
449 + "Probably the layout of KeyValue.Type has changed.");
450 }
451 return keyType;
452 }
453 }
454
455 }
456