View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.filter;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.ResultScanner;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.client.Table;
42  import org.apache.hadoop.hbase.filter.FilterList.Operator;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.RegionScanner;
46  import org.apache.hadoop.hbase.testclassification.MediumTests;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.Pair;
49  import org.junit.After;
50  import org.junit.AfterClass;
51  import org.junit.Before;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  import com.google.common.collect.Lists;
57  
58  /**
59   */
60  @Category(MediumTests.class)
61  public class TestFuzzyRowFilterEndToEnd {
62    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63    private final static byte fuzzyValue = (byte) 63;
64    private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
65  
66    private static int firstPartCardinality = 50;
67    private static int secondPartCardinality = 50;
68    private static int thirdPartCardinality = 50;
69    private static int colQualifiersTotal = 5;
70    private static int totalFuzzyKeys = thirdPartCardinality / 2;
71  
72    private static String table = "TestFuzzyRowFilterEndToEnd";
73  
74    /**
75     * @throws java.lang.Exception
76     */
77    @BeforeClass
78    public static void setUpBeforeClass() throws Exception {
79      Configuration conf = TEST_UTIL.getConfiguration();
80      conf.setInt("hbase.client.scanner.caching", 1000);
81      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
82        ConstantSizeRegionSplitPolicy.class.getName());
83      // set no splits
84      conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
85  
86      TEST_UTIL.startMiniCluster();
87    }
88  
89    /**
90     * @throws java.lang.Exception
91     */
92    @AfterClass
93    public static void tearDownAfterClass() throws Exception {
94      TEST_UTIL.shutdownMiniCluster();
95    }
96  
97    /**
98     * @throws java.lang.Exception
99     */
100   @Before
101   public void setUp() throws Exception {
102     // Nothing to do.
103   }
104 
105   /**
106    * @throws java.lang.Exception
107    */
108   @After
109   public void tearDown() throws Exception {
110     // Nothing to do.
111   }
112 
113   // HBASE-15676 Test that fuzzy info of all fixed bits (0s) finds matching row.
114   @Test
115   public void testAllFixedBits() throws IOException {
116     String cf = "f";
117     String cq = "q";
118     String table = "testAllFixedBits";
119 
120     Table ht =
121         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
122     // Load data
123     String[] rows = new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
124         "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
125         "\\x9B\\x00\\x044e\\x9B\\x02\\xBB", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
126         "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
127 
128     for (int i = 0; i < rows.length; i++) {
129       Put p = new Put(Bytes.toBytesBinary(rows[i]));
130       p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
131       ht.put(p);
132     }
133 
134     TEST_UTIL.flush();
135 
136     List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>();
137     byte[] fuzzyKey = Bytes.toBytesBinary("\\x9B\\x00\\x044e");
138     byte[] mask = new byte[] { 0, 0, 0, 0, 0 };
139     data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
140     FuzzyRowFilter filter = new FuzzyRowFilter(data);
141 
142     Scan scan = new Scan();
143     scan.setFilter(filter);
144 
145     ResultScanner scanner = ht.getScanner(scan);
146     int total = 0;
147     while (scanner.next() != null) {
148       total++;
149     }
150     assertEquals(2, total);
151     TEST_UTIL.deleteTable(TableName.valueOf(table));
152   }
153 
154   @Test
155   public void testHBASE14782() throws IOException
156   {
157     String cf = "f";
158     String cq = "q";
159     String table = "HBASE14872";
160 
161     Table ht =
162         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
163     // Load data
164     String[] rows = new String[]{
165         "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
166         "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", 
167         "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
168         "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
169         "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", 
170         "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", 
171     };
172     
173     String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
174     
175     for(int i=0; i < rows.length; i++){
176       Put p = new Put(Bytes.toBytesBinary(rows[i]));
177       p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
178       ht.put(p);            
179     }
180     
181     Put p = new Put(Bytes.toBytesBinary(badRow));
182     p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
183     ht.put(p);            
184 
185     TEST_UTIL.flush();
186 
187     List<Pair<byte[], byte[]>> data =  new ArrayList<Pair<byte[], byte[]>>();
188     byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
189     byte[] mask = new byte[] { 1,0,0,0};
190     data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
191     FuzzyRowFilter filter = new FuzzyRowFilter(data);
192     
193     Scan scan = new Scan();
194     scan.setFilter(filter);
195     
196     ResultScanner scanner = ht.getScanner(scan);
197     int total = 0;
198     while(scanner.next() != null){
199       total++;
200     }    
201     assertEquals(rows.length, total);
202     TEST_UTIL.deleteTable(TableName.valueOf(table));
203   }
204   
205   @Test
206   public void testEndToEnd() throws Exception {
207     String cf = "f";
208 
209     HTable ht =
210         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
211 
212     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
213     // 4 byte qualifier
214     // 4 byte value
215 
216     for (int i0 = 0; i0 < firstPartCardinality; i0++) {
217 
218       for (int i1 = 0; i1 < secondPartCardinality; i1++) {
219 
220         for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
221           byte[] rk = new byte[10];
222 
223           ByteBuffer buf = ByteBuffer.wrap(rk);
224           buf.clear();
225           buf.putShort((short) i0);
226           buf.putInt(i1);
227           buf.putInt(i2);
228           for (int c = 0; c < colQualifiersTotal; c++) {
229             byte[] cq = new byte[4];
230             Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
231 
232             Put p = new Put(rk);
233             p.setDurability(Durability.SKIP_WAL);
234             p.add(cf.getBytes(), cq, Bytes.toBytes(c));
235             ht.put(p);
236           }
237         }
238       }
239     }
240 
241     TEST_UTIL.flush();
242 
243     // test passes
244     runTest1(ht);
245     runTest2(ht);
246 
247   }
248 
249   private void runTest1(Table hTable) throws IOException {
250     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
251 
252     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
253 
254     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
255     for (int i = 0; i < totalFuzzyKeys; i++) {
256       byte[] fuzzyKey = new byte[10];
257       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
258       buf.clear();
259       buf.putShort((short) 2);
260       for (int j = 0; j < 4; j++) {
261         buf.put(fuzzyValue);
262       }
263       buf.putInt(i);
264 
265       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
266       list.add(pair);
267     }
268 
269     int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
270     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
271     // Filters are not stateless - we can't reuse them
272     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
273 
274     // regular test
275     runScanner(hTable, expectedSize, fuzzyRowFilter0);
276     // optimized from block cache
277     runScanner(hTable, expectedSize, fuzzyRowFilter1);
278 
279   }
280 
281   private void runTest2(Table hTable) throws IOException {
282     // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
283 
284     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
285 
286     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
287 
288     for (int i = 0; i < totalFuzzyKeys; i++) {
289       byte[] fuzzyKey = new byte[10];
290       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
291       buf.clear();
292       buf.putShort((short) (i * 2));
293       for (int j = 0; j < 4; j++) {
294         buf.put(fuzzyValue);
295       }
296       buf.putInt(i * 2);
297 
298       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
299       list.add(pair);
300     }
301 
302     int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
303 
304     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
305     // Filters are not stateless - we can't reuse them
306     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
307 
308     // regular test
309     runScanner(hTable, expectedSize, fuzzyRowFilter0);
310     // optimized from block cache
311     runScanner(hTable, expectedSize, fuzzyRowFilter1);
312 
313   }
314 
315   private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {
316 
317     String cf = "f";
318     Scan scan = new Scan();
319     scan.addFamily(cf.getBytes());
320     scan.setFilter(filter);
321     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
322     HRegion first = regions.get(0);
323     first.getScanner(scan);
324     RegionScanner scanner = first.getScanner(scan);
325     List<Cell> results = new ArrayList<Cell>();
326     // Result result;
327     long timeBeforeScan = System.currentTimeMillis();
328     int found = 0;
329     while (scanner.next(results)) {
330       found += results.size();
331       results.clear();
332     }
333     found += results.size();
334     long scanTime = System.currentTimeMillis() - timeBeforeScan;
335     scanner.close();
336 
337     LOG.info("\nscan time = " + scanTime + "ms");
338     LOG.info("found " + found + " results\n");
339 
340     assertEquals(expectedSize, found);
341   }
342 
343   @SuppressWarnings("deprecation")
344   @Test
345   public void testFilterList() throws Exception {
346     String cf = "f";
347     String table = "TestFuzzyRowFiltersInFilterList";
348     HTable ht =
349         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
350 
351     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
352     // 4 byte qualifier
353     // 4 byte value
354 
355     for (int i1 = 0; i1 < 5; i1++) {
356       for (int i2 = 0; i2 < 5; i2++) {
357         byte[] rk = new byte[10];
358 
359         ByteBuffer buf = ByteBuffer.wrap(rk);
360         buf.clear();
361         buf.putShort((short) 2);
362         buf.putInt(i1);
363         buf.putInt(i2);
364 
365         // Each row contains 5 columns
366         for (int c = 0; c < 5; c++) {
367           byte[] cq = new byte[4];
368           Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
369 
370           Put p = new Put(rk);
371           p.setDurability(Durability.SKIP_WAL);
372           p.add(cf.getBytes(), cq, Bytes.toBytes(c));
373           ht.put(p);
374           LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
375               + Bytes.toStringBinary(cq));
376         }
377       }
378     }
379 
380     TEST_UTIL.flush();
381 
382     // test passes if we get back 5 KV's (1 row)
383     runTest(ht, 5);
384 
385   }
386 
387   @SuppressWarnings("unchecked")
388   private void runTest(HTable hTable, int expectedSize) throws IOException {
389     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
390     byte[] fuzzyKey1 = new byte[10];
391     ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
392     buf.clear();
393     buf.putShort((short) 2);
394     for (int i = 0; i < 4; i++)
395       buf.put(fuzzyValue);
396     buf.putInt((short) 1);
397     byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
398 
399     byte[] fuzzyKey2 = new byte[10];
400     buf = ByteBuffer.wrap(fuzzyKey2);
401     buf.clear();
402     buf.putShort((short) 2);
403     buf.putInt((short) 2);
404     for (int i = 0; i < 4; i++)
405       buf.put(fuzzyValue);
406 
407     byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
408 
409     Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
410     Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
411 
412     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
413     FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
414     // regular test - we expect 1 row back (5 KVs)
415     runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
416   }
417 
418   private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
419       throws IOException {
420     String cf = "f";
421     Scan scan = new Scan();
422     scan.addFamily(cf.getBytes());
423     FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
424     scan.setFilter(filterList);
425 
426     ResultScanner scanner = hTable.getScanner(scan);
427     List<Cell> results = new ArrayList<Cell>();
428     Result result;
429     long timeBeforeScan = System.currentTimeMillis();
430     while ((result = scanner.next()) != null) {
431       for (Cell kv : result.listCells()) {
432         LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
433             + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
434         results.add(kv);
435       }
436     }
437     long scanTime = System.currentTimeMillis() - timeBeforeScan;
438     scanner.close();
439 
440     LOG.info("scan time = " + scanTime + "ms");
441     LOG.info("found " + results.size() + " results");
442 
443     assertEquals(expectedSize, results.size());
444   }
445 }