View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.io;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.KeyValueUtil;
37  import org.apache.hadoop.hbase.testclassification.SmallTests;
38  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
39  import org.apache.hadoop.hbase.io.hfile.HFile;
40  import org.apache.hadoop.hbase.io.hfile.HFileContext;
41  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
42  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.AfterClass;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  @Category(SmallTests.class)
50  public class TestHalfStoreFileReader {
51    private static HBaseTestingUtility TEST_UTIL;
52  
53    @BeforeClass
54    public static void setupBeforeClass() throws Exception {
55      TEST_UTIL = new HBaseTestingUtility();
56    }
57  
58    @AfterClass
59    public static void tearDownAfterClass() throws Exception {
60      TEST_UTIL.cleanupTestDir();
61    }
62  
63    /**
64     * Test the scanner and reseek of a half hfile scanner. The scanner API
65     * demands that seekTo and reseekTo() only return < 0 if the key lies
66     * before the start of the file (with no position on the scanner). Returning
67     * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
68     *
69     * The latter case being the most common, we should generally be returning 1,
70     * and if we do, there may or may not be a 'next' in the scanner/file.
71     *
72     * A bug in the half file scanner was returning -1 at the end of the bottom
73     * half, and that was causing the infrastructure above to go null causing NPEs
74     * and other problems.  This test reproduces that failure, and also tests
75     * both the bottom and top of the file while we are at it.
76     *
77     * @throws IOException
78     */
79    @Test
80    public void testHalfScanAndReseek() throws IOException {
81      String root_dir = TEST_UTIL.getDataTestDir().toString();
82      Path p = new Path(root_dir, "test");
83  
84      Configuration conf = TEST_UTIL.getConfiguration();
85      FileSystem fs = FileSystem.get(conf);
86      CacheConfig cacheConf = new CacheConfig(conf);
87      HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
88      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
89          .withPath(fs, p)
90          .withFileContext(meta)
91          .create();
92  
93      // write some things.
94      List<KeyValue> items = genSomeKeys();
95      for (KeyValue kv : items) {
96        w.append(kv);
97      }
98      w.close();
99  
100     HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
101     r.loadFileInfo();
102     byte [] midkey = r.midkey();
103     KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
104     midkey = midKV.getRow();
105 
106     //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
107 
108     Reference bottom = new Reference(midkey, Reference.Range.bottom);
109     doTestOfScanAndReseek(p, fs, bottom, cacheConf);
110 
111     Reference top = new Reference(midkey, Reference.Range.top);
112     doTestOfScanAndReseek(p, fs, top, cacheConf);
113 
114     r.close();
115   }
116 
117   private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
118       CacheConfig cacheConf)
119       throws IOException {
120     final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
121       cacheConf, bottom, TEST_UTIL.getConfiguration());
122     halfreader.loadFileInfo();
123     final HFileScanner scanner = halfreader.getScanner(false, false);
124 
125     scanner.seekTo();
126     Cell curr;
127     do {
128       curr = scanner.getKeyValue();
129       KeyValue reseekKv =
130           getLastOnCol(curr);
131       int ret = scanner.reseekTo(reseekKv);
132       assertTrue("reseek to returned: " + ret, ret > 0);
133       //System.out.println(curr + ": " + ret);
134     } while (scanner.next());
135 
136     int ret = scanner.reseekTo(getLastOnCol(curr));
137     //System.out.println("Last reseek: " + ret);
138     assertTrue( ret > 0 );
139 
140     halfreader.close(true);
141   }
142 
143 
144   // Tests the scanner on an HFile that is backed by HalfStoreFiles
145   @Test
146   public void testHalfScanner() throws IOException {
147       String root_dir = TEST_UTIL.getDataTestDir().toString();
148       Path p = new Path(root_dir, "test");
149       Configuration conf = TEST_UTIL.getConfiguration();
150       FileSystem fs = FileSystem.get(conf);
151       CacheConfig cacheConf = new CacheConfig(conf);
152       HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
153       HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
154               .withPath(fs, p)
155               .withFileContext(meta)
156               .create();
157 
158       // write some things.
159       List<KeyValue> items = genSomeKeys();
160       for (KeyValue kv : items) {
161           w.append(kv);
162       }
163       w.close();
164 
165 
166       HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
167       r.loadFileInfo();
168       byte[] midkey = r.midkey();
169       KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
170       midkey = midKV.getRow();
171 
172       Reference bottom = new Reference(midkey, Reference.Range.bottom);
173       Reference top = new Reference(midkey, Reference.Range.top);
174 
175       // Ugly code to get the item before the midkey
176       KeyValue beforeMidKey = null;
177       for (KeyValue item : items) {
178           if (KeyValue.COMPARATOR.compare(item, midKV) >= 0) {
179               break;
180           }
181           beforeMidKey = item;
182       }
183       System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
184       System.out.println("beforeMidKey: " + beforeMidKey);
185 
186 
187       // Seek on the splitKey, should be in top, not in bottom
188       Cell foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
189       assertEquals(beforeMidKey, foundKeyValue);
190 
191       // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
192       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
193       assertEquals(items.get(items.size() - 2), foundKeyValue);
194 
195       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
196       assertEquals(beforeMidKey, foundKeyValue);
197 
198       // Try and seek before something that is in the bottom.
199       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
200       assertNull(foundKeyValue);
201 
202       // Try and seek before the first thing.
203       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
204       assertNull(foundKeyValue);
205 
206       // Try and seek before the second thing in the top and bottom.
207       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
208       assertNull(foundKeyValue);
209 
210       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
211       assertEquals(items.get(0), foundKeyValue);
212 
213       // Try to seek before the splitKey in the top file
214       foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
215       assertNull(foundKeyValue);
216     }
217 
218   private Cell doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
219                                         CacheConfig cacheConfig)
220             throws IOException {
221       final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
222               cacheConfig, bottom, TEST_UTIL.getConfiguration());
223       halfreader.loadFileInfo();
224       final HFileScanner scanner = halfreader.getScanner(false, false);
225       scanner.seekBefore(seekBefore);
226       return scanner.getKeyValue();
227   }
228 
229   private KeyValue getLastOnCol(Cell curr) {
230     return KeyValueUtil.createLastOnRow(
231         curr.getRowArray(), curr.getRowOffset(), curr.getRowLength(),
232         curr.getFamilyArray(), curr.getFamilyOffset(), curr.getFamilyLength(),
233         curr.getQualifierArray(), curr.getQualifierOffset(), curr.getQualifierLength());
234   }
235 
236   static final int SIZE = 1000;
237 
238   static byte[] _b(String s) {
239     return Bytes.toBytes(s);
240   }
241 
242   List<KeyValue> genSomeKeys() {
243     List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
244     for (int i = 0; i < SIZE; i++) {
245       KeyValue kv =
246           new KeyValue(
247               _b(String.format("row_%04d", i)),
248               _b("family"),
249               _b("qualifier"),
250               1000, // timestamp
251               _b("value"));
252       ret.add(kv);
253     }
254     return ret;
255   }
256 
257 
258 
259 }
260