View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.List;
23  import java.util.Random;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileStatus;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.client.ConnectionFactory;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.Result;
41  import org.apache.hadoop.hbase.client.ResultScanner;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.client.Table;
44  import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
45  import org.apache.hadoop.hbase.io.hfile.TestHFile;
46  import org.apache.hadoop.hbase.mob.MobConstants;
47  import org.apache.hadoop.hbase.mob.MobUtils;
48  import org.apache.hadoop.hbase.testclassification.MediumTests;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
52  import org.junit.AfterClass;
53  import org.junit.Assert;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  import org.junit.experimental.categories.Category;
57  
58  @Category(MediumTests.class)
59  public class TestMobStoreScanner {
60  
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private final static byte [] row1 = Bytes.toBytes("row1");
63    private final static byte [] row2 = Bytes.toBytes("row2");
64    private final static byte [] family = Bytes.toBytes("family");
65    private final static byte [] qf1 = Bytes.toBytes("qualifier1");
66    private final static byte [] qf2 = Bytes.toBytes("qualifier2");
67    protected final byte[] qf3 = Bytes.toBytes("qualifier3");
68    private static Table table;
69    private static HBaseAdmin admin;
70    private static HColumnDescriptor hcd;
71    private static HTableDescriptor desc;
72    private static Random random = new Random();
73    private static long defaultThreshold = 10;
74    private FileSystem fs;
75    private Configuration conf;
76  
77    @BeforeClass
78    public static void setUpBeforeClass() throws Exception {
79      TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
80      TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
81      TEST_UTIL.getConfiguration().setInt("hbase.client.keyvalue.maxsize", 100*1024*1024);
82  
83      TEST_UTIL.startMiniCluster(1);
84    }
85  
86    @AfterClass
87    public static void tearDownAfterClass() throws Exception {
88      TEST_UTIL.shutdownMiniCluster();
89    }
90  
91    public void setUp(long threshold, TableName tn) throws Exception {
92      conf = TEST_UTIL.getConfiguration();
93      fs = FileSystem.get(conf);
94      desc = new HTableDescriptor(tn);
95      hcd = new HColumnDescriptor(family);
96      hcd.setMobEnabled(true);
97      hcd.setMobThreshold(threshold);
98      hcd.setMaxVersions(4);
99      desc.addFamily(hcd);
100     admin = TEST_UTIL.getHBaseAdmin();
101     admin.createTable(desc);
102     table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
103             .getTable(tn);
104   }
105 
106   /**
107    * Generate the mob value.
108    *
109    * @param size the size of the value
110    * @return the mob value generated
111    */
112   private static byte[] generateMobValue(int size) {
113     byte[] mobVal = new byte[size];
114     random.nextBytes(mobVal);
115     return mobVal;
116   }
117 
118   /**
119    * Set the scan attribute
120    *
121    * @param reversed if true, scan will be backward order
122    * @param mobScanRaw if true, scan will get the mob reference
123    * @return this
124    */
125   public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) {
126     scan.setReversed(reversed);
127     scan.setMaxVersions(4);
128     if(mobScanRaw) {
129       scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
130     }
131   }
132 
133   @Test
134   public void testMobStoreScanner() throws Exception {
135 	  testGetFromFiles(false);
136 	  testGetFromMemStore(false);
137     testGetReferences(false);
138     testMobThreshold(false);
139     testGetFromArchive(false);
140   }
141 
142   @Test
143   public void testReversedMobStoreScanner() throws Exception {
144 	  testGetFromFiles(true);
145 	  testGetFromMemStore(true);
146     testGetReferences(true);
147     testMobThreshold(true);
148     testGetFromArchive(true);
149   }
150 
151   @Test(timeout=60000)
152   public void testGetMassive() throws Exception {
153     setUp(defaultThreshold, TableName.valueOf("testGetMassive"));
154 
155     // Put some data 5 10, 15, 20  mb ok  (this would be right below protobuf default max size of 64MB.
156     // 25, 30, 40 fail.  these is above protobuf max size of 64MB
157     byte[] bigValue = new byte[25*1024*1024];
158 
159     Put put = new Put(row1);
160     put.addColumn(family, qf1, bigValue);
161     put.addColumn(family, qf2, bigValue);
162     put.addColumn(family, qf3, bigValue);
163     table.put(put);
164 
165     Get g = new Get(row1);
166     Result r = table.get(g);
167     // should not have blown up.
168   }
169 
170   @Test
171   public void testReadPt() throws Exception {
172     TableName tn = TableName.valueOf("testReadPt");
173     setUp(0L, tn);
174     long ts = System.currentTimeMillis();
175     byte[] value1 = Bytes.toBytes("value1");
176     Put put1 = new Put(row1);
177     put1.addColumn(family, qf1, ts, value1);
178     table.put(put1);
179     Put put2 = new Put(row2);
180     byte[] value2 = Bytes.toBytes("value2");
181     put2.addColumn(family, qf1, ts, value2);
182     table.put(put2);
183 
184     Scan scan = new Scan();
185     scan.setCaching(1);
186     ResultScanner rs = table.getScanner(scan);
187 
188     Put put3 = new Put(row1);
189     byte[] value3 = Bytes.toBytes("value3");
190     put3.addColumn(family, qf1, ts, value3);
191     table.put(put3);
192     Put put4 = new Put(row2);
193     byte[] value4 = Bytes.toBytes("value4");
194     put4.addColumn(family, qf1, ts, value4);
195     table.put(put4);
196     Result result = rs.next();
197     Cell cell = result.getColumnLatestCell(family, qf1);
198     Assert.assertEquals("value1", Bytes.toString(cell.getValue()));
199 
200     admin.flush(tn);
201     result = rs.next();
202     cell = result.getColumnLatestCell(family, qf1);
203     Assert.assertEquals("value2", Bytes.toString(cell.getValue()));
204   }
205 
206   @Test
207   public void testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss() throws Exception {
208     TableName tn = TableName.valueOf("testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss");
209     setUp(0, tn);
210     createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
211     Get get = new Get(row1);
212     get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
213     Result result = table.get(get);
214     Cell cell = result.getColumnLatestCell(family, qf1);
215     Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
216   }
217 
218   @Test
219   public void testReadFromCorruptMobFiles() throws Exception {
220     TableName tn = TableName.valueOf("testReadFromCorruptMobFiles");
221     setUp(0, tn);
222     createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
223     Get get = new Get(row1);
224     IOException ioe = null;
225     try {
226       table.get(get);
227     } catch (IOException e) {
228       ioe = e;
229     }
230     Assert.assertNotNull(ioe);
231     Assert.assertEquals(CorruptHFileException.class.getName(), ioe.getClass().getName());
232   }
233 
234   private void createRecordAndCorruptMobFile(TableName tn, byte[] row, byte[] family, byte[] qf,
235     byte[] value) throws IOException {
236     Put put1 = new Put(row);
237     put1.addColumn(family, qf, value);
238     table.put(put1);
239     admin.flush(tn);
240     Path mobFile = getFlushedMobFile(conf, fs, tn, Bytes.toString(family));
241     Assert.assertNotNull(mobFile);
242     // create new corrupt mob file.
243     Path corruptFile = new Path(mobFile.getParent(), "dummy");
244     TestHFile.truncateFile(fs, mobFile, corruptFile);
245     fs.delete(mobFile, true);
246     fs.rename(corruptFile, mobFile);
247   }
248 
249   private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
250     throws IOException {
251     Path regionDir = MobUtils.getMobRegionPath(conf, table);
252     Path famDir = new Path(regionDir, family);
253     FileStatus[] hfFss = fs.listStatus(famDir);
254     for (FileStatus hfs : hfFss) {
255       if (!hfs.isDirectory()) {
256         return hfs.getPath();
257       }
258     }
259     return null;
260   }
261 
262   private void testGetFromFiles(boolean reversed) throws Exception {
263     TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
264     setUp(defaultThreshold, tn);
265     long ts1 = System.currentTimeMillis();
266     long ts2 = ts1 + 1;
267     long ts3 = ts1 + 2;
268     byte [] value = generateMobValue((int)defaultThreshold+1);
269 
270     Put put1 = new Put(row1);
271     put1.addColumn(family, qf1, ts3, value);
272     put1.addColumn(family, qf2, ts2, value);
273     put1.addColumn(family, qf3, ts1, value);
274     table.put(put1);
275 
276     admin.flush(tn);
277 
278     Scan scan = new Scan();
279     setScan(scan, reversed, false);
280 
281     ResultScanner results = table.getScanner(scan);
282     int count = 0;
283     for (Result res : results) {
284       List<Cell> cells = res.listCells();
285       for(Cell cell : cells) {
286         // Verify the value
287         Assert.assertEquals(Bytes.toString(value),
288             Bytes.toString(CellUtil.cloneValue(cell)));
289         count++;
290       }
291     }
292     results.close();
293     Assert.assertEquals(3, count);
294   }
295 
296   private void testGetFromMemStore(boolean reversed) throws Exception {
297     setUp(defaultThreshold, TableName.valueOf("testGetFromMemStore" + reversed));
298     long ts1 = System.currentTimeMillis();
299     long ts2 = ts1 + 1;
300     long ts3 = ts1 + 2;
301     byte [] value = generateMobValue((int)defaultThreshold+1);;
302 
303     Put put1 = new Put(row1);
304     put1.addColumn(family, qf1, ts3, value);
305     put1.addColumn(family, qf2, ts2, value);
306     put1.addColumn(family, qf3, ts1, value);
307     table.put(put1);
308 
309     Scan scan = new Scan();
310     setScan(scan, reversed, false);
311 
312     ResultScanner results = table.getScanner(scan);
313     int count = 0;
314     for (Result res : results) {
315       List<Cell> cells = res.listCells();
316       for(Cell cell : cells) {
317         // Verify the value
318         Assert.assertEquals(Bytes.toString(value),
319             Bytes.toString(CellUtil.cloneValue(cell)));
320         count++;
321       }
322     }
323     results.close();
324     Assert.assertEquals(3, count);
325   }
326 
327   private void testGetReferences(boolean reversed) throws Exception {
328     TableName tn = TableName.valueOf("testGetReferences" + reversed);
329     setUp(defaultThreshold, tn);
330     long ts1 = System.currentTimeMillis();
331     long ts2 = ts1 + 1;
332     long ts3 = ts1 + 2;
333     byte [] value = generateMobValue((int)defaultThreshold+1);;
334 
335     Put put1 = new Put(row1);
336     put1.addColumn(family, qf1, ts3, value);
337     put1.addColumn(family, qf2, ts2, value);
338     put1.addColumn(family, qf3, ts1, value);
339     table.put(put1);
340 
341     admin.flush(tn);
342 
343     Scan scan = new Scan();
344     setScan(scan, reversed, true);
345 
346     ResultScanner results = table.getScanner(scan);
347     int count = 0;
348     for (Result res : results) {
349       List<Cell> cells = res.listCells();
350       for(Cell cell : cells) {
351         // Verify the value
352         assertIsMobReference(cell, row1, family, value, tn);
353         count++;
354       }
355     }
356     results.close();
357     Assert.assertEquals(3, count);
358   }
359 
360   private void testMobThreshold(boolean reversed) throws Exception {
361     TableName tn = TableName.valueOf("testMobThreshold" + reversed);
362     setUp(defaultThreshold, tn);
363     byte [] valueLess = generateMobValue((int)defaultThreshold-1);
364     byte [] valueEqual = generateMobValue((int)defaultThreshold);
365     byte [] valueGreater = generateMobValue((int)defaultThreshold+1);
366     long ts1 = System.currentTimeMillis();
367     long ts2 = ts1 + 1;
368     long ts3 = ts1 + 2;
369 
370     Put put1 = new Put(row1);
371     put1.addColumn(family, qf1, ts3, valueLess);
372     put1.addColumn(family, qf2, ts2, valueEqual);
373     put1.addColumn(family, qf3, ts1, valueGreater);
374     table.put(put1);
375 
376     admin.flush(tn);
377 
378     Scan scan = new Scan();
379     setScan(scan, reversed, true);
380 
381     Cell cellLess= null;
382     Cell cellEqual = null;
383     Cell cellGreater = null;
384     ResultScanner results = table.getScanner(scan);
385     int count = 0;
386     for (Result res : results) {
387       List<Cell> cells = res.listCells();
388       for(Cell cell : cells) {
389         // Verify the value
390         String qf = Bytes.toString(CellUtil.cloneQualifier(cell));
391         if(qf.equals(Bytes.toString(qf1))) {
392           cellLess = cell;
393         }
394         if(qf.equals(Bytes.toString(qf2))) {
395           cellEqual = cell;
396         }
397         if(qf.equals(Bytes.toString(qf3))) {
398           cellGreater = cell;
399         }
400         count++;
401       }
402     }
403     Assert.assertEquals(3, count);
404     assertNotMobReference(cellLess, row1, family, valueLess);
405     assertNotMobReference(cellEqual, row1, family, valueEqual);
406     assertIsMobReference(cellGreater, row1, family, valueGreater, tn);
407     results.close();
408   }
409 
410   private void testGetFromArchive(boolean reversed) throws Exception {
411     TableName tn = TableName.valueOf("testGetFromArchive" + reversed);
412     setUp(defaultThreshold, tn);
413     long ts1 = System.currentTimeMillis();
414     long ts2 = ts1 + 1;
415     long ts3 = ts1 + 2;
416     byte [] value = generateMobValue((int)defaultThreshold+1);;
417     // Put some data
418     Put put1 = new Put(row1);
419     put1.addColumn(family, qf1, ts3, value);
420     put1.addColumn(family, qf2, ts2, value);
421     put1.addColumn(family, qf3, ts1, value);
422     table.put(put1);
423 
424     admin.flush(tn);
425 
426     // Get the files in the mob path
427     Path mobFamilyPath;
428     mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tn),
429       hcd.getNameAsString());
430     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
431     FileStatus[] files = fs.listStatus(mobFamilyPath);
432 
433     // Get the archive path
434     Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
435     Path tableDir = FSUtils.getTableDir(rootDir, tn);
436     HRegionInfo regionInfo = MobUtils.getMobRegionInfo(tn);
437     Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(),
438         regionInfo, tableDir, family);
439 
440     // Move the files from mob path to archive path
441     fs.mkdirs(storeArchiveDir);
442     int fileCount = 0;
443     for(FileStatus file : files) {
444       fileCount++;
445       Path filePath = file.getPath();
446       Path src = new Path(mobFamilyPath, filePath.getName());
447       Path dst = new Path(storeArchiveDir, filePath.getName());
448       fs.rename(src, dst);
449     }
450 
451     // Verify the moving success
452     FileStatus[] files1 = fs.listStatus(mobFamilyPath);
453     Assert.assertEquals(0, files1.length);
454     FileStatus[] files2 = fs.listStatus(storeArchiveDir);
455     Assert.assertEquals(fileCount, files2.length);
456 
457     // Scan from archive
458     Scan scan = new Scan();
459     setScan(scan, reversed, false);
460     ResultScanner results = table.getScanner(scan);
461     int count = 0;
462     for (Result res : results) {
463       List<Cell> cells = res.listCells();
464       for(Cell cell : cells) {
465         // Verify the value
466         Assert.assertEquals(Bytes.toString(value),
467             Bytes.toString(CellUtil.cloneValue(cell)));
468         count++;
469       }
470     }
471     results.close();
472     Assert.assertEquals(3, count);
473   }
474 
475   /**
476    * Assert the value is not store in mob.
477    */
478   private static void assertNotMobReference(Cell cell, byte[] row, byte[] family,
479       byte[] value) throws IOException {
480     Assert.assertEquals(Bytes.toString(row),
481         Bytes.toString(CellUtil.cloneRow(cell)));
482     Assert.assertEquals(Bytes.toString(family),
483         Bytes.toString(CellUtil.cloneFamily(cell)));
484     Assert.assertTrue(Bytes.toString(value).equals(
485         Bytes.toString(CellUtil.cloneValue(cell))));
486   }
487 
488   /**
489    * Assert the value is store in mob.
490    */
491   private static void assertIsMobReference(Cell cell, byte[] row, byte[] family,
492       byte[] value, TableName tn) throws IOException {
493     Assert.assertEquals(Bytes.toString(row),
494         Bytes.toString(CellUtil.cloneRow(cell)));
495     Assert.assertEquals(Bytes.toString(family),
496         Bytes.toString(CellUtil.cloneFamily(cell)));
497     Assert.assertFalse(Bytes.toString(value).equals(
498         Bytes.toString(CellUtil.cloneValue(cell))));
499     byte[] referenceValue = CellUtil.cloneValue(cell);
500     String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
501         referenceValue.length - Bytes.SIZEOF_INT);
502     int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT);
503     Assert.assertEquals(value.length, valLen);
504     Path mobFamilyPath;
505     mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
506         tn), hcd.getNameAsString());
507     Path targetPath = new Path(mobFamilyPath, fileName);
508     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
509     Assert.assertTrue(fs.exists(targetPath));
510   }
511 }