1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.filter;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Set;
29
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValueTestUtil;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.Durability;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.regionserver.HRegion;
42 import org.apache.hadoop.hbase.regionserver.InternalScanner;
43 import org.apache.hadoop.hbase.testclassification.SmallTests;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48 @Category(SmallTests.class)
49 public class TestMultipleColumnPrefixFilter {
50
51 private final static HBaseTestingUtility TEST_UTIL = new
52 HBaseTestingUtility();
53
54 @Test
55 public void testMultipleColumnPrefixFilter() throws IOException {
56 String family = "Family";
57 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
58 HColumnDescriptor hcd = new HColumnDescriptor(family);
59 hcd.setMaxVersions(3);
60 htd.addFamily(hcd);
61
62 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
63 HRegion region = HRegion.createHRegion(info, TEST_UTIL.
64 getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
65
66 List<String> rows = generateRandomWords(100, "row");
67 List<String> columns = generateRandomWords(10000, "column");
68 long maxTimestamp = 2;
69
70 List<Cell> kvList = new ArrayList<Cell>();
71
72 Map<String, List<Cell>> prefixMap = new HashMap<String,
73 List<Cell>>();
74
75 prefixMap.put("p", new ArrayList<Cell>());
76 prefixMap.put("q", new ArrayList<Cell>());
77 prefixMap.put("s", new ArrayList<Cell>());
78
79 String valueString = "ValueString";
80
81 for (String row: rows) {
82 Put p = new Put(Bytes.toBytes(row));
83 p.setDurability(Durability.SKIP_WAL);
84 for (String column: columns) {
85 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
86 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
87 valueString);
88 p.add(kv);
89 kvList.add(kv);
90 for (String s: prefixMap.keySet()) {
91 if (column.startsWith(s)) {
92 prefixMap.get(s).add(kv);
93 }
94 }
95 }
96 }
97 region.put(p);
98 }
99
100 MultipleColumnPrefixFilter filter;
101 Scan scan = new Scan();
102 scan.setMaxVersions();
103 byte [][] filter_prefix = new byte [2][];
104 filter_prefix[0] = new byte [] {'p'};
105 filter_prefix[1] = new byte [] {'q'};
106
107 filter = new MultipleColumnPrefixFilter(filter_prefix);
108 scan.setFilter(filter);
109 List<Cell> results = new ArrayList<Cell>();
110 InternalScanner scanner = region.getScanner(scan);
111 while (scanner.next(results))
112 ;
113 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
114
115 HRegion.closeHRegion(region);
116 }
117
118 @Test
119 public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
120 String family1 = "Family1";
121 String family2 = "Family2";
122 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
123 HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
124 hcd1.setMaxVersions(3);
125 htd.addFamily(hcd1);
126 HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
127 hcd2.setMaxVersions(3);
128 htd.addFamily(hcd2);
129 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
130 HRegion region = HRegion.createHRegion(info, TEST_UTIL.
131 getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
132
133 List<String> rows = generateRandomWords(100, "row");
134 List<String> columns = generateRandomWords(10000, "column");
135 long maxTimestamp = 3;
136
137 List<Cell> kvList = new ArrayList<Cell>();
138
139 Map<String, List<Cell>> prefixMap = new HashMap<String,
140 List<Cell>>();
141
142 prefixMap.put("p", new ArrayList<Cell>());
143 prefixMap.put("q", new ArrayList<Cell>());
144 prefixMap.put("s", new ArrayList<Cell>());
145
146 String valueString = "ValueString";
147
148 for (String row: rows) {
149 Put p = new Put(Bytes.toBytes(row));
150 p.setDurability(Durability.SKIP_WAL);
151 for (String column: columns) {
152 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
153 double rand = Math.random();
154 Cell kv;
155 if (rand < 0.5)
156 kv = KeyValueTestUtil.create(row, family1, column, timestamp,
157 valueString);
158 else
159 kv = KeyValueTestUtil.create(row, family2, column, timestamp,
160 valueString);
161 p.add(kv);
162 kvList.add(kv);
163 for (String s: prefixMap.keySet()) {
164 if (column.startsWith(s)) {
165 prefixMap.get(s).add(kv);
166 }
167 }
168 }
169 }
170 region.put(p);
171 }
172
173 MultipleColumnPrefixFilter filter;
174 Scan scan = new Scan();
175 scan.setMaxVersions();
176 byte [][] filter_prefix = new byte [2][];
177 filter_prefix[0] = new byte [] {'p'};
178 filter_prefix[1] = new byte [] {'q'};
179
180 filter = new MultipleColumnPrefixFilter(filter_prefix);
181 scan.setFilter(filter);
182 List<Cell> results = new ArrayList<Cell>();
183 InternalScanner scanner = region.getScanner(scan);
184 while (scanner.next(results))
185 ;
186 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
187
188 HRegion.closeHRegion(region);
189 }
190
191 @Test
192 public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
193 String family = "Family";
194 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
195 htd.addFamily(new HColumnDescriptor(family));
196 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
197 HRegion region = HRegion.createHRegion(info, TEST_UTIL.
198 getDataTestDir(), TEST_UTIL.getConfiguration(),htd);
199
200 List<String> rows = generateRandomWords(100, "row");
201 List<String> columns = generateRandomWords(10000, "column");
202 long maxTimestamp = 2;
203
204 String valueString = "ValueString";
205
206 for (String row: rows) {
207 Put p = new Put(Bytes.toBytes(row));
208 p.setDurability(Durability.SKIP_WAL);
209 for (String column: columns) {
210 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
211 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
212 valueString);
213 p.add(kv);
214 }
215 }
216 region.put(p);
217 }
218
219 MultipleColumnPrefixFilter multiplePrefixFilter;
220 Scan scan1 = new Scan();
221 scan1.setMaxVersions();
222 byte [][] filter_prefix = new byte [1][];
223 filter_prefix[0] = new byte [] {'p'};
224
225 multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
226 scan1.setFilter(multiplePrefixFilter);
227 List<Cell> results1 = new ArrayList<Cell>();
228 InternalScanner scanner1 = region.getScanner(scan1);
229 while (scanner1.next(results1))
230 ;
231
232 ColumnPrefixFilter singlePrefixFilter;
233 Scan scan2 = new Scan();
234 scan2.setMaxVersions();
235 singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
236
237 scan2.setFilter(singlePrefixFilter);
238 List<Cell> results2 = new ArrayList<Cell>();
239 InternalScanner scanner2 = region.getScanner(scan1);
240 while (scanner2.next(results2))
241 ;
242
243 assertEquals(results1.size(), results2.size());
244
245 HRegion.closeHRegion(region);
246 }
247
248 List<String> generateRandomWords(int numberOfWords, String suffix) {
249 Set<String> wordSet = new HashSet<String>();
250 for (int i = 0; i < numberOfWords; i++) {
251 int lengthOfWords = (int) (Math.random()*2) + 1;
252 char[] wordChar = new char[lengthOfWords];
253 for (int j = 0; j < wordChar.length; j++) {
254 wordChar[j] = (char) (Math.random() * 26 + 97);
255 }
256 String word;
257 if (suffix == null) {
258 word = new String(wordChar);
259 } else {
260 word = new String(wordChar) + suffix;
261 }
262 wordSet.add(word);
263 }
264 List<String> wordList = new ArrayList<String>(wordSet);
265 return wordList;
266 }
267
268 }
269
270