1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.List;
23 import java.util.Random;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileStatus;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.CellUtil;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.client.ConnectionFactory;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HBaseAdmin;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.ResultScanner;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.client.Table;
44 import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
45 import org.apache.hadoop.hbase.io.hfile.TestHFile;
46 import org.apache.hadoop.hbase.mob.MobConstants;
47 import org.apache.hadoop.hbase.mob.MobUtils;
48 import org.apache.hadoop.hbase.testclassification.MediumTests;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
52 import org.junit.AfterClass;
53 import org.junit.Assert;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56 import org.junit.experimental.categories.Category;
57
58 @Category(MediumTests.class)
59 public class TestMobStoreScanner {
60
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private final static byte [] row1 = Bytes.toBytes("row1");
63 private final static byte [] row2 = Bytes.toBytes("row2");
64 private final static byte [] family = Bytes.toBytes("family");
65 private final static byte [] qf1 = Bytes.toBytes("qualifier1");
66 private final static byte [] qf2 = Bytes.toBytes("qualifier2");
67 protected final byte[] qf3 = Bytes.toBytes("qualifier3");
68 private static Table table;
69 private static HBaseAdmin admin;
70 private static HColumnDescriptor hcd;
71 private static HTableDescriptor desc;
72 private static Random random = new Random();
73 private static long defaultThreshold = 10;
74 private FileSystem fs;
75 private Configuration conf;
76
77 @BeforeClass
78 public static void setUpBeforeClass() throws Exception {
79 TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
80 TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
81 TEST_UTIL.getConfiguration().setInt("hbase.client.keyvalue.maxsize", 100*1024*1024);
82
83 TEST_UTIL.startMiniCluster(1);
84 }
85
86 @AfterClass
87 public static void tearDownAfterClass() throws Exception {
88 TEST_UTIL.shutdownMiniCluster();
89 }
90
91 public void setUp(long threshold, TableName tn) throws Exception {
92 conf = TEST_UTIL.getConfiguration();
93 fs = FileSystem.get(conf);
94 desc = new HTableDescriptor(tn);
95 hcd = new HColumnDescriptor(family);
96 hcd.setMobEnabled(true);
97 hcd.setMobThreshold(threshold);
98 hcd.setMaxVersions(4);
99 desc.addFamily(hcd);
100 admin = TEST_UTIL.getHBaseAdmin();
101 admin.createTable(desc);
102 table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
103 .getTable(tn);
104 }
105
106
107
108
109
110
111
112 private static byte[] generateMobValue(int size) {
113 byte[] mobVal = new byte[size];
114 random.nextBytes(mobVal);
115 return mobVal;
116 }
117
118
119
120
121
122
123
124
125 public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) {
126 scan.setReversed(reversed);
127 scan.setMaxVersions(4);
128 if(mobScanRaw) {
129 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
130 }
131 }
132
133 @Test
134 public void testMobStoreScanner() throws Exception {
135 testGetFromFiles(false);
136 testGetFromMemStore(false);
137 testGetReferences(false);
138 testMobThreshold(false);
139 testGetFromArchive(false);
140 }
141
142 @Test
143 public void testReversedMobStoreScanner() throws Exception {
144 testGetFromFiles(true);
145 testGetFromMemStore(true);
146 testGetReferences(true);
147 testMobThreshold(true);
148 testGetFromArchive(true);
149 }
150
151 @Test(timeout=60000)
152 public void testGetMassive() throws Exception {
153 setUp(defaultThreshold, TableName.valueOf("testGetMassive"));
154
155
156
157 byte[] bigValue = new byte[25*1024*1024];
158
159 Put put = new Put(row1);
160 put.addColumn(family, qf1, bigValue);
161 put.addColumn(family, qf2, bigValue);
162 put.addColumn(family, qf3, bigValue);
163 table.put(put);
164
165 Get g = new Get(row1);
166 Result r = table.get(g);
167
168 }
169
170 @Test
171 public void testReadPt() throws Exception {
172 TableName tn = TableName.valueOf("testReadPt");
173 setUp(0L, tn);
174 long ts = System.currentTimeMillis();
175 byte[] value1 = Bytes.toBytes("value1");
176 Put put1 = new Put(row1);
177 put1.addColumn(family, qf1, ts, value1);
178 table.put(put1);
179 Put put2 = new Put(row2);
180 byte[] value2 = Bytes.toBytes("value2");
181 put2.addColumn(family, qf1, ts, value2);
182 table.put(put2);
183
184 Scan scan = new Scan();
185 scan.setCaching(1);
186 ResultScanner rs = table.getScanner(scan);
187
188 Put put3 = new Put(row1);
189 byte[] value3 = Bytes.toBytes("value3");
190 put3.addColumn(family, qf1, ts, value3);
191 table.put(put3);
192 Put put4 = new Put(row2);
193 byte[] value4 = Bytes.toBytes("value4");
194 put4.addColumn(family, qf1, ts, value4);
195 table.put(put4);
196 Result result = rs.next();
197 Cell cell = result.getColumnLatestCell(family, qf1);
198 Assert.assertEquals("value1", Bytes.toString(cell.getValue()));
199
200 admin.flush(tn);
201 result = rs.next();
202 cell = result.getColumnLatestCell(family, qf1);
203 Assert.assertEquals("value2", Bytes.toString(cell.getValue()));
204 }
205
206 @Test
207 public void testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss() throws Exception {
208 TableName tn = TableName.valueOf("testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss");
209 setUp(0, tn);
210 createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
211 Get get = new Get(row1);
212 get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
213 Result result = table.get(get);
214 Cell cell = result.getColumnLatestCell(family, qf1);
215 Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
216 }
217
218 @Test
219 public void testReadFromCorruptMobFiles() throws Exception {
220 TableName tn = TableName.valueOf("testReadFromCorruptMobFiles");
221 setUp(0, tn);
222 createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
223 Get get = new Get(row1);
224 IOException ioe = null;
225 try {
226 table.get(get);
227 } catch (IOException e) {
228 ioe = e;
229 }
230 Assert.assertNotNull(ioe);
231 Assert.assertEquals(CorruptHFileException.class.getName(), ioe.getClass().getName());
232 }
233
234 private void createRecordAndCorruptMobFile(TableName tn, byte[] row, byte[] family, byte[] qf,
235 byte[] value) throws IOException {
236 Put put1 = new Put(row);
237 put1.addColumn(family, qf, value);
238 table.put(put1);
239 admin.flush(tn);
240 Path mobFile = getFlushedMobFile(conf, fs, tn, Bytes.toString(family));
241 Assert.assertNotNull(mobFile);
242
243 Path corruptFile = new Path(mobFile.getParent(), "dummy");
244 TestHFile.truncateFile(fs, mobFile, corruptFile);
245 fs.delete(mobFile, true);
246 fs.rename(corruptFile, mobFile);
247 }
248
249 private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
250 throws IOException {
251 Path regionDir = MobUtils.getMobRegionPath(conf, table);
252 Path famDir = new Path(regionDir, family);
253 FileStatus[] hfFss = fs.listStatus(famDir);
254 for (FileStatus hfs : hfFss) {
255 if (!hfs.isDirectory()) {
256 return hfs.getPath();
257 }
258 }
259 return null;
260 }
261
262 private void testGetFromFiles(boolean reversed) throws Exception {
263 TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
264 setUp(defaultThreshold, tn);
265 long ts1 = System.currentTimeMillis();
266 long ts2 = ts1 + 1;
267 long ts3 = ts1 + 2;
268 byte [] value = generateMobValue((int)defaultThreshold+1);
269
270 Put put1 = new Put(row1);
271 put1.addColumn(family, qf1, ts3, value);
272 put1.addColumn(family, qf2, ts2, value);
273 put1.addColumn(family, qf3, ts1, value);
274 table.put(put1);
275
276 admin.flush(tn);
277
278 Scan scan = new Scan();
279 setScan(scan, reversed, false);
280
281 ResultScanner results = table.getScanner(scan);
282 int count = 0;
283 for (Result res : results) {
284 List<Cell> cells = res.listCells();
285 for(Cell cell : cells) {
286
287 Assert.assertEquals(Bytes.toString(value),
288 Bytes.toString(CellUtil.cloneValue(cell)));
289 count++;
290 }
291 }
292 results.close();
293 Assert.assertEquals(3, count);
294 }
295
296 private void testGetFromMemStore(boolean reversed) throws Exception {
297 setUp(defaultThreshold, TableName.valueOf("testGetFromMemStore" + reversed));
298 long ts1 = System.currentTimeMillis();
299 long ts2 = ts1 + 1;
300 long ts3 = ts1 + 2;
301 byte [] value = generateMobValue((int)defaultThreshold+1);;
302
303 Put put1 = new Put(row1);
304 put1.addColumn(family, qf1, ts3, value);
305 put1.addColumn(family, qf2, ts2, value);
306 put1.addColumn(family, qf3, ts1, value);
307 table.put(put1);
308
309 Scan scan = new Scan();
310 setScan(scan, reversed, false);
311
312 ResultScanner results = table.getScanner(scan);
313 int count = 0;
314 for (Result res : results) {
315 List<Cell> cells = res.listCells();
316 for(Cell cell : cells) {
317
318 Assert.assertEquals(Bytes.toString(value),
319 Bytes.toString(CellUtil.cloneValue(cell)));
320 count++;
321 }
322 }
323 results.close();
324 Assert.assertEquals(3, count);
325 }
326
327 private void testGetReferences(boolean reversed) throws Exception {
328 TableName tn = TableName.valueOf("testGetReferences" + reversed);
329 setUp(defaultThreshold, tn);
330 long ts1 = System.currentTimeMillis();
331 long ts2 = ts1 + 1;
332 long ts3 = ts1 + 2;
333 byte [] value = generateMobValue((int)defaultThreshold+1);;
334
335 Put put1 = new Put(row1);
336 put1.addColumn(family, qf1, ts3, value);
337 put1.addColumn(family, qf2, ts2, value);
338 put1.addColumn(family, qf3, ts1, value);
339 table.put(put1);
340
341 admin.flush(tn);
342
343 Scan scan = new Scan();
344 setScan(scan, reversed, true);
345
346 ResultScanner results = table.getScanner(scan);
347 int count = 0;
348 for (Result res : results) {
349 List<Cell> cells = res.listCells();
350 for(Cell cell : cells) {
351
352 assertIsMobReference(cell, row1, family, value, tn);
353 count++;
354 }
355 }
356 results.close();
357 Assert.assertEquals(3, count);
358 }
359
360 private void testMobThreshold(boolean reversed) throws Exception {
361 TableName tn = TableName.valueOf("testMobThreshold" + reversed);
362 setUp(defaultThreshold, tn);
363 byte [] valueLess = generateMobValue((int)defaultThreshold-1);
364 byte [] valueEqual = generateMobValue((int)defaultThreshold);
365 byte [] valueGreater = generateMobValue((int)defaultThreshold+1);
366 long ts1 = System.currentTimeMillis();
367 long ts2 = ts1 + 1;
368 long ts3 = ts1 + 2;
369
370 Put put1 = new Put(row1);
371 put1.addColumn(family, qf1, ts3, valueLess);
372 put1.addColumn(family, qf2, ts2, valueEqual);
373 put1.addColumn(family, qf3, ts1, valueGreater);
374 table.put(put1);
375
376 admin.flush(tn);
377
378 Scan scan = new Scan();
379 setScan(scan, reversed, true);
380
381 Cell cellLess= null;
382 Cell cellEqual = null;
383 Cell cellGreater = null;
384 ResultScanner results = table.getScanner(scan);
385 int count = 0;
386 for (Result res : results) {
387 List<Cell> cells = res.listCells();
388 for(Cell cell : cells) {
389
390 String qf = Bytes.toString(CellUtil.cloneQualifier(cell));
391 if(qf.equals(Bytes.toString(qf1))) {
392 cellLess = cell;
393 }
394 if(qf.equals(Bytes.toString(qf2))) {
395 cellEqual = cell;
396 }
397 if(qf.equals(Bytes.toString(qf3))) {
398 cellGreater = cell;
399 }
400 count++;
401 }
402 }
403 Assert.assertEquals(3, count);
404 assertNotMobReference(cellLess, row1, family, valueLess);
405 assertNotMobReference(cellEqual, row1, family, valueEqual);
406 assertIsMobReference(cellGreater, row1, family, valueGreater, tn);
407 results.close();
408 }
409
410 private void testGetFromArchive(boolean reversed) throws Exception {
411 TableName tn = TableName.valueOf("testGetFromArchive" + reversed);
412 setUp(defaultThreshold, tn);
413 long ts1 = System.currentTimeMillis();
414 long ts2 = ts1 + 1;
415 long ts3 = ts1 + 2;
416 byte [] value = generateMobValue((int)defaultThreshold+1);;
417
418 Put put1 = new Put(row1);
419 put1.addColumn(family, qf1, ts3, value);
420 put1.addColumn(family, qf2, ts2, value);
421 put1.addColumn(family, qf3, ts1, value);
422 table.put(put1);
423
424 admin.flush(tn);
425
426
427 Path mobFamilyPath;
428 mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tn),
429 hcd.getNameAsString());
430 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
431 FileStatus[] files = fs.listStatus(mobFamilyPath);
432
433
434 Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
435 Path tableDir = FSUtils.getTableDir(rootDir, tn);
436 HRegionInfo regionInfo = MobUtils.getMobRegionInfo(tn);
437 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(),
438 regionInfo, tableDir, family);
439
440
441 fs.mkdirs(storeArchiveDir);
442 int fileCount = 0;
443 for(FileStatus file : files) {
444 fileCount++;
445 Path filePath = file.getPath();
446 Path src = new Path(mobFamilyPath, filePath.getName());
447 Path dst = new Path(storeArchiveDir, filePath.getName());
448 fs.rename(src, dst);
449 }
450
451
452 FileStatus[] files1 = fs.listStatus(mobFamilyPath);
453 Assert.assertEquals(0, files1.length);
454 FileStatus[] files2 = fs.listStatus(storeArchiveDir);
455 Assert.assertEquals(fileCount, files2.length);
456
457
458 Scan scan = new Scan();
459 setScan(scan, reversed, false);
460 ResultScanner results = table.getScanner(scan);
461 int count = 0;
462 for (Result res : results) {
463 List<Cell> cells = res.listCells();
464 for(Cell cell : cells) {
465
466 Assert.assertEquals(Bytes.toString(value),
467 Bytes.toString(CellUtil.cloneValue(cell)));
468 count++;
469 }
470 }
471 results.close();
472 Assert.assertEquals(3, count);
473 }
474
475
476
477
478 private static void assertNotMobReference(Cell cell, byte[] row, byte[] family,
479 byte[] value) throws IOException {
480 Assert.assertEquals(Bytes.toString(row),
481 Bytes.toString(CellUtil.cloneRow(cell)));
482 Assert.assertEquals(Bytes.toString(family),
483 Bytes.toString(CellUtil.cloneFamily(cell)));
484 Assert.assertTrue(Bytes.toString(value).equals(
485 Bytes.toString(CellUtil.cloneValue(cell))));
486 }
487
488
489
490
491 private static void assertIsMobReference(Cell cell, byte[] row, byte[] family,
492 byte[] value, TableName tn) throws IOException {
493 Assert.assertEquals(Bytes.toString(row),
494 Bytes.toString(CellUtil.cloneRow(cell)));
495 Assert.assertEquals(Bytes.toString(family),
496 Bytes.toString(CellUtil.cloneFamily(cell)));
497 Assert.assertFalse(Bytes.toString(value).equals(
498 Bytes.toString(CellUtil.cloneValue(cell))));
499 byte[] referenceValue = CellUtil.cloneValue(cell);
500 String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
501 referenceValue.length - Bytes.SIZEOF_INT);
502 int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT);
503 Assert.assertEquals(value.length, valLen);
504 Path mobFamilyPath;
505 mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
506 tn), hcd.getNameAsString());
507 Path targetPath = new Path(mobFamilyPath, fileName);
508 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
509 Assert.assertTrue(fs.exists(targetPath));
510 }
511 }