1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.filter;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellUtil;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.client.Table;
42 import org.apache.hadoop.hbase.filter.FilterList.Operator;
43 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.RegionScanner;
46 import org.apache.hadoop.hbase.testclassification.MediumTests;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.Pair;
49 import org.junit.After;
50 import org.junit.AfterClass;
51 import org.junit.Before;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 import com.google.common.collect.Lists;
57
58
59
60 @Category(MediumTests.class)
61 public class TestFuzzyRowFilterEndToEnd {
62 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63 private final static byte fuzzyValue = (byte) 63;
64 private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
65
66 private static int firstPartCardinality = 50;
67 private static int secondPartCardinality = 50;
68 private static int thirdPartCardinality = 50;
69 private static int colQualifiersTotal = 5;
70 private static int totalFuzzyKeys = thirdPartCardinality / 2;
71
72 private static String table = "TestFuzzyRowFilterEndToEnd";
73
74
75
76
77 @BeforeClass
78 public static void setUpBeforeClass() throws Exception {
79 Configuration conf = TEST_UTIL.getConfiguration();
80 conf.setInt("hbase.client.scanner.caching", 1000);
81 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
82 ConstantSizeRegionSplitPolicy.class.getName());
83
84 conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
85
86 TEST_UTIL.startMiniCluster();
87 }
88
89
90
91
92 @AfterClass
93 public static void tearDownAfterClass() throws Exception {
94 TEST_UTIL.shutdownMiniCluster();
95 }
96
97
98
99
100 @Before
101 public void setUp() throws Exception {
102
103 }
104
105
106
107
108 @After
109 public void tearDown() throws Exception {
110
111 }
112
113
114 @Test
115 public void testAllFixedBits() throws IOException {
116 String cf = "f";
117 String cq = "q";
118 String table = "testAllFixedBits";
119
120 Table ht =
121 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
122
123 String[] rows = new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
124 "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
125 "\\x9B\\x00\\x044e\\x9B\\x02\\xBB", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
126 "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
127
128 for (int i = 0; i < rows.length; i++) {
129 Put p = new Put(Bytes.toBytesBinary(rows[i]));
130 p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
131 ht.put(p);
132 }
133
134 TEST_UTIL.flush();
135
136 List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>();
137 byte[] fuzzyKey = Bytes.toBytesBinary("\\x9B\\x00\\x044e");
138 byte[] mask = new byte[] { 0, 0, 0, 0, 0 };
139 data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
140 FuzzyRowFilter filter = new FuzzyRowFilter(data);
141
142 Scan scan = new Scan();
143 scan.setFilter(filter);
144
145 ResultScanner scanner = ht.getScanner(scan);
146 int total = 0;
147 while (scanner.next() != null) {
148 total++;
149 }
150 assertEquals(2, total);
151 TEST_UTIL.deleteTable(TableName.valueOf(table));
152 }
153
154 @Test
155 public void testHBASE14782() throws IOException
156 {
157 String cf = "f";
158 String cq = "q";
159 String table = "HBASE14872";
160
161 Table ht =
162 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
163
164 String[] rows = new String[]{
165 "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
166 "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
167 "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
168 "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
169 "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01",
170 "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB",
171 };
172
173 String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
174
175 for(int i=0; i < rows.length; i++){
176 Put p = new Put(Bytes.toBytesBinary(rows[i]));
177 p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
178 ht.put(p);
179 }
180
181 Put p = new Put(Bytes.toBytesBinary(badRow));
182 p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
183 ht.put(p);
184
185 TEST_UTIL.flush();
186
187 List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>();
188 byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
189 byte[] mask = new byte[] { 1,0,0,0};
190 data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
191 FuzzyRowFilter filter = new FuzzyRowFilter(data);
192
193 Scan scan = new Scan();
194 scan.setFilter(filter);
195
196 ResultScanner scanner = ht.getScanner(scan);
197 int total = 0;
198 while(scanner.next() != null){
199 total++;
200 }
201 assertEquals(rows.length, total);
202 TEST_UTIL.deleteTable(TableName.valueOf(table));
203 }
204
205 @Test
206 public void testEndToEnd() throws Exception {
207 String cf = "f";
208
209 HTable ht =
210 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
211
212
213
214
215
216 for (int i0 = 0; i0 < firstPartCardinality; i0++) {
217
218 for (int i1 = 0; i1 < secondPartCardinality; i1++) {
219
220 for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
221 byte[] rk = new byte[10];
222
223 ByteBuffer buf = ByteBuffer.wrap(rk);
224 buf.clear();
225 buf.putShort((short) i0);
226 buf.putInt(i1);
227 buf.putInt(i2);
228 for (int c = 0; c < colQualifiersTotal; c++) {
229 byte[] cq = new byte[4];
230 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
231
232 Put p = new Put(rk);
233 p.setDurability(Durability.SKIP_WAL);
234 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
235 ht.put(p);
236 }
237 }
238 }
239 }
240
241 TEST_UTIL.flush();
242
243
244 runTest1(ht);
245 runTest2(ht);
246
247 }
248
249 private void runTest1(Table hTable) throws IOException {
250
251
252 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
253
254 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
255 for (int i = 0; i < totalFuzzyKeys; i++) {
256 byte[] fuzzyKey = new byte[10];
257 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
258 buf.clear();
259 buf.putShort((short) 2);
260 for (int j = 0; j < 4; j++) {
261 buf.put(fuzzyValue);
262 }
263 buf.putInt(i);
264
265 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
266 list.add(pair);
267 }
268
269 int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
270 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
271
272 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
273
274
275 runScanner(hTable, expectedSize, fuzzyRowFilter0);
276
277 runScanner(hTable, expectedSize, fuzzyRowFilter1);
278
279 }
280
281 private void runTest2(Table hTable) throws IOException {
282
283
284 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
285
286 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
287
288 for (int i = 0; i < totalFuzzyKeys; i++) {
289 byte[] fuzzyKey = new byte[10];
290 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
291 buf.clear();
292 buf.putShort((short) (i * 2));
293 for (int j = 0; j < 4; j++) {
294 buf.put(fuzzyValue);
295 }
296 buf.putInt(i * 2);
297
298 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
299 list.add(pair);
300 }
301
302 int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
303
304 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
305
306 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
307
308
309 runScanner(hTable, expectedSize, fuzzyRowFilter0);
310
311 runScanner(hTable, expectedSize, fuzzyRowFilter1);
312
313 }
314
315 private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {
316
317 String cf = "f";
318 Scan scan = new Scan();
319 scan.addFamily(cf.getBytes());
320 scan.setFilter(filter);
321 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
322 HRegion first = regions.get(0);
323 first.getScanner(scan);
324 RegionScanner scanner = first.getScanner(scan);
325 List<Cell> results = new ArrayList<Cell>();
326
327 long timeBeforeScan = System.currentTimeMillis();
328 int found = 0;
329 while (scanner.next(results)) {
330 found += results.size();
331 results.clear();
332 }
333 found += results.size();
334 long scanTime = System.currentTimeMillis() - timeBeforeScan;
335 scanner.close();
336
337 LOG.info("\nscan time = " + scanTime + "ms");
338 LOG.info("found " + found + " results\n");
339
340 assertEquals(expectedSize, found);
341 }
342
343 @SuppressWarnings("deprecation")
344 @Test
345 public void testFilterList() throws Exception {
346 String cf = "f";
347 String table = "TestFuzzyRowFiltersInFilterList";
348 HTable ht =
349 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
350
351
352
353
354
355 for (int i1 = 0; i1 < 5; i1++) {
356 for (int i2 = 0; i2 < 5; i2++) {
357 byte[] rk = new byte[10];
358
359 ByteBuffer buf = ByteBuffer.wrap(rk);
360 buf.clear();
361 buf.putShort((short) 2);
362 buf.putInt(i1);
363 buf.putInt(i2);
364
365
366 for (int c = 0; c < 5; c++) {
367 byte[] cq = new byte[4];
368 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
369
370 Put p = new Put(rk);
371 p.setDurability(Durability.SKIP_WAL);
372 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
373 ht.put(p);
374 LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
375 + Bytes.toStringBinary(cq));
376 }
377 }
378 }
379
380 TEST_UTIL.flush();
381
382
383 runTest(ht, 5);
384
385 }
386
387 @SuppressWarnings("unchecked")
388 private void runTest(HTable hTable, int expectedSize) throws IOException {
389
390 byte[] fuzzyKey1 = new byte[10];
391 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
392 buf.clear();
393 buf.putShort((short) 2);
394 for (int i = 0; i < 4; i++)
395 buf.put(fuzzyValue);
396 buf.putInt((short) 1);
397 byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
398
399 byte[] fuzzyKey2 = new byte[10];
400 buf = ByteBuffer.wrap(fuzzyKey2);
401 buf.clear();
402 buf.putShort((short) 2);
403 buf.putInt((short) 2);
404 for (int i = 0; i < 4; i++)
405 buf.put(fuzzyValue);
406
407 byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
408
409 Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
410 Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
411
412 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
413 FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
414
415 runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
416 }
417
418 private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
419 throws IOException {
420 String cf = "f";
421 Scan scan = new Scan();
422 scan.addFamily(cf.getBytes());
423 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
424 scan.setFilter(filterList);
425
426 ResultScanner scanner = hTable.getScanner(scan);
427 List<Cell> results = new ArrayList<Cell>();
428 Result result;
429 long timeBeforeScan = System.currentTimeMillis();
430 while ((result = scanner.next()) != null) {
431 for (Cell kv : result.listCells()) {
432 LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
433 + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
434 results.add(kv);
435 }
436 }
437 long scanTime = System.currentTimeMillis() - timeBeforeScan;
438 scanner.close();
439
440 LOG.info("scan time = " + scanTime + "ms");
441 LOG.info("found " + results.size() + " results");
442
443 assertEquals(expectedSize, results.size());
444 }
445 }