1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.ByteArrayInputStream;
28 import java.io.DataInputStream;
29 import java.io.IOException;
30 import java.nio.ByteBuffer;
31 import java.util.ArrayList;
32 import java.util.List;
33 import java.util.Random;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FSDataInputStream;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.KeyValue.KVComparator;
44 import org.apache.hadoop.hbase.testclassification.SmallTests;
45 import org.apache.hadoop.hbase.io.compress.Compression;
46 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
47 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.Writables;
50 import org.apache.hadoop.io.Text;
51 import org.apache.hadoop.io.WritableUtils;
52 import org.junit.Before;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56
57
58
59
60 @Category(SmallTests.class)
61 public class TestHFileWriterV2 {
62
63 private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class);
64
65 private static final HBaseTestingUtility TEST_UTIL =
66 new HBaseTestingUtility();
67
68 private Configuration conf;
69 private FileSystem fs;
70
71 @Before
72 public void setUp() throws IOException {
73 conf = TEST_UTIL.getConfiguration();
74 fs = FileSystem.get(conf);
75 }
76
77 @Test
78 public void testHFileFormatV2() throws IOException {
79 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV2");
80 final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
81 final int entryCount = 10000;
82 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false);
83 }
84
85 @Test
86 public void testMidKeyInHFile() throws IOException{
87 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
88 "testMidKeyInHFile");
89 Compression.Algorithm compressAlgo = Compression.Algorithm.NONE;
90 int entryCount = 50000;
91 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true);
92 }
93
94 private void writeDataAndReadFromHFile(Path hfilePath,
95 Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException {
96
97 HFileContext context = new HFileContextBuilder()
98 .withBlockSize(4096)
99 .withCompression(compressAlgo)
100 .build();
101 HFileWriterV2 writer = (HFileWriterV2)
102 new HFileWriterV2.WriterFactoryV2(conf, new CacheConfig(conf))
103 .withPath(fs, hfilePath)
104 .withFileContext(context)
105 .create();
106
107 Random rand = new Random(9713312);
108 List<KeyValue> keyValues = new ArrayList<KeyValue>(entryCount);
109
110 for (int i = 0; i < entryCount; ++i) {
111 byte[] keyBytes = randomOrderedKey(rand, i);
112
113
114 byte[] valueBytes = randomValue(rand);
115 KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes);
116 writer.append(keyValue);
117 keyValues.add(keyValue);
118 }
119
120
121
122 writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C."));
123 writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow"));
124 writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris"));
125
126 writer.close();
127
128
129 FSDataInputStream fsdis = fs.open(hfilePath);
130
131
132
133
134 long fileSize = fs.getFileStatus(hfilePath).getLen();
135 FixedFileTrailer trailer =
136 FixedFileTrailer.readFromStream(fsdis, fileSize);
137
138 assertEquals(2, trailer.getMajorVersion());
139 assertEquals(entryCount, trailer.getEntryCount());
140
141 HFileContext meta = new HFileContextBuilder()
142 .withHBaseCheckSum(true)
143 .withIncludesMvcc(false)
144 .withIncludesTags(false)
145 .withCompression(compressAlgo)
146 .build();
147
148 HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, meta);
149
150 KVComparator comparator = trailer.createComparator();
151 HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
152 new HFileBlockIndex.BlockIndexReader(comparator,
153 trailer.getNumDataIndexLevels());
154 HFileBlockIndex.BlockIndexReader metaBlockIndexReader =
155 new HFileBlockIndex.BlockIndexReader(
156 KeyValue.RAW_COMPARATOR, 1);
157
158 HFileBlock.BlockIterator blockIter = blockReader.blockRange(
159 trailer.getLoadOnOpenDataOffset(),
160 fileSize - trailer.getTrailerSize());
161
162
163 dataBlockIndexReader.readMultiLevelIndexRoot(
164 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
165 trailer.getDataIndexCount());
166
167 if (findMidKey) {
168 byte[] midkey = dataBlockIndexReader.midkey();
169 assertNotNull("Midkey should not be null", midkey);
170 }
171
172
173 metaBlockIndexReader.readRootIndex(
174 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX)
175 .getByteStream(), trailer.getMetaIndexCount());
176
177 FileInfo fileInfo = new FileInfo();
178 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
179 byte [] keyValueFormatVersion = fileInfo.get(
180 HFileWriterV2.KEY_VALUE_VERSION);
181 boolean includeMemstoreTS = keyValueFormatVersion != null &&
182 Bytes.toInt(keyValueFormatVersion) > 0;
183
184
185 int entriesRead = 0;
186 int blocksRead = 0;
187 long memstoreTS = 0;
188
189
190 fsdis.seek(0);
191 long curBlockPos = 0;
192 while (curBlockPos <= trailer.getLastDataBlockOffset()) {
193 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false, false);
194 assertEquals(BlockType.DATA, block.getBlockType());
195 if (meta.isCompressedOrEncrypted()) {
196 assertFalse(block.isUnpacked());
197 block = block.unpack(meta, blockReader);
198 }
199 ByteBuffer buf = block.getBufferWithoutHeader();
200 while (buf.hasRemaining()) {
201 int keyLen = buf.getInt();
202 int valueLen = buf.getInt();
203
204 byte[] key = new byte[keyLen];
205 buf.get(key);
206
207 byte[] value = new byte[valueLen];
208 buf.get(value);
209
210 if (includeMemstoreTS) {
211 ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(),
212 buf.arrayOffset() + buf.position(), buf.remaining());
213 DataInputStream data_input = new DataInputStream(byte_input);
214
215 memstoreTS = WritableUtils.readVLong(data_input);
216 buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS));
217 }
218
219
220 assertTrue(Bytes.compareTo(key, keyValues.get(entriesRead).getKey()) == 0);
221 assertTrue(Bytes.compareTo(value, keyValues.get(entriesRead).getValue()) == 0);
222
223 ++entriesRead;
224 }
225 ++blocksRead;
226 curBlockPos += block.getOnDiskSizeWithHeader();
227 }
228 LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead="
229 + blocksRead);
230 assertEquals(entryCount, entriesRead);
231
232
233
234
235
236 int metaCounter = 0;
237 while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) {
238 LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " +
239 trailer.getLoadOnOpenDataOffset());
240 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false, false)
241 .unpack(meta, blockReader);
242 assertEquals(BlockType.META, block.getBlockType());
243 Text t = new Text();
244 ByteBuffer buf = block.getBufferWithoutHeader();
245 if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) {
246 throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName());
247 }
248 Text expectedText =
249 (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text(
250 "Moscow") : new Text("Washington, D.C."));
251 assertEquals(expectedText, t);
252 LOG.info("Read meta block data: " + t);
253 ++metaCounter;
254 curBlockPos += block.getOnDiskSizeWithHeader();
255 }
256
257 fsdis.close();
258 }
259
260
261
262
263 private static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";
264 private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64;
265 private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128;
266
267
268
269
270
271
272
273
274
275
276
277 public static byte[] randomOrderedKey(Random rand, int i) {
278 StringBuilder k = new StringBuilder();
279
280
281 for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
282 if ((i & (1 << bitIndex)) == 0)
283 k.append("a");
284 else
285 k.append("b");
286 }
287
288
289 for (int j = 0; j < rand.nextInt(50); ++j)
290 k.append(randomReadableChar(rand));
291
292 byte[] keyBytes = k.toString().getBytes();
293 return keyBytes;
294 }
295
296 public static byte[] randomOrderedFixedLengthKey(Random rand, int i, int suffixLength) {
297 StringBuilder k = new StringBuilder();
298
299
300 for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
301 if ((i & (1 << bitIndex)) == 0)
302 k.append("a");
303 else
304 k.append("b");
305 }
306
307
308 for (int j = 0; j < suffixLength; ++j)
309 k.append(randomReadableChar(rand));
310
311 byte[] keyBytes = k.toString().getBytes();
312 return keyBytes;
313 }
314
315 public static byte[] randomValue(Random rand) {
316 StringBuilder v = new StringBuilder();
317 for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
318 v.append((char) (32 + rand.nextInt(95)));
319 }
320
321 byte[] valueBytes = v.toString().getBytes();
322 return valueBytes;
323 }
324
325 public static byte[] randomFixedLengthValue(Random rand, int valueLength) {
326 StringBuilder v = new StringBuilder();
327 for (int j = 0; j < valueLength; ++j) {
328 v.append((char) (32 + rand.nextInt(95)));
329 }
330
331 byte[] valueBytes = v.toString().getBytes();
332 return valueBytes;
333 }
334
335 public static final char randomReadableChar(Random rand) {
336 int i = rand.nextInt(26 * 2 + 10 + 1);
337 if (i < 26)
338 return (char) ('A' + i);
339 i -= 26;
340
341 if (i < 26)
342 return (char) ('a' + i);
343 i -= 26;
344
345 if (i < 10)
346 return (char) ('0' + i);
347 i -= 10;
348
349 assert i == 0;
350 return '_';
351 }
352
353 public static byte[] randomRowOrQualifier(Random rand) {
354 StringBuilder field = new StringBuilder();
355 int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
356 + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
357 - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
358 for (int i = 0; i < fieldLen; ++i)
359 field.append(randomReadableChar(rand));
360 return field.toString().getBytes();
361 }
362
363 public static KeyValue randomKeyValue(Random rand) {
364 return new KeyValue(randomRowOrQualifier(rand),
365 COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand),
366 randomValue(rand));
367 }
368
369
370 }
371