1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.thrift2;
19
20 import static java.nio.ByteBuffer.wrap;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.testclassification.MediumTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Admin;
43 import org.apache.hadoop.hbase.client.HBaseAdmin;
44 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
45 import org.apache.hadoop.hbase.security.User;
46 import org.apache.hadoop.hbase.security.UserProvider;
47 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
48 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
49 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
50 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
51 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
52 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
53 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
54 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
55 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
56 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
57 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
58 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
59 import org.apache.hadoop.hbase.thrift2.generated.TGet;
60 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
61 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
62 import org.apache.hadoop.hbase.thrift2.generated.TPut;
63 import org.apache.hadoop.hbase.thrift2.generated.TResult;
64 import org.apache.hadoop.hbase.thrift2.generated.TScan;
65 import org.apache.hadoop.hbase.util.Bytes;
66 import org.junit.AfterClass;
67 import org.junit.Assert;
68 import org.junit.Before;
69 import org.junit.BeforeClass;
70 import org.junit.Test;
71 import org.junit.experimental.categories.Category;
72
73 @Category(MediumTests.class)
74 public class TestThriftHBaseServiceHandlerWithLabels {
75
76 public static final Log LOG = LogFactory
77 .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
78 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79
80
81 private static byte[] tableAname = Bytes.toBytes("tableA");
82 private static byte[] familyAname = Bytes.toBytes("familyA");
83 private static byte[] familyBname = Bytes.toBytes("familyB");
84 private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
85 private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
86 private static byte[] valueAname = Bytes.toBytes("valueA");
87 private static byte[] valueBname = Bytes.toBytes("valueB");
88 private static HColumnDescriptor[] families = new HColumnDescriptor[] {
89 new HColumnDescriptor(familyAname).setMaxVersions(3),
90 new HColumnDescriptor(familyBname).setMaxVersions(2) };
91
92 private final static String TOPSECRET = "topsecret";
93 private final static String PUBLIC = "public";
94 private final static String PRIVATE = "private";
95 private final static String CONFIDENTIAL = "confidential";
96 private final static String SECRET = "secret";
97 private static User SUPERUSER;
98
99 private static Configuration conf;
100
101 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
102 List<TColumnValue> columnValuesB) {
103 assertEquals(columnValuesA.size(), columnValuesB.size());
104 Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
105 @Override
106 public int compare(TColumnValue o1, TColumnValue o2) {
107 return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
108 Bytes.add(o2.getFamily(), o2.getQualifier()));
109 }
110 };
111 Collections.sort(columnValuesA, comparator);
112 Collections.sort(columnValuesB, comparator);
113
114 for (int i = 0; i < columnValuesA.size(); i++) {
115 TColumnValue a = columnValuesA.get(i);
116 TColumnValue b = columnValuesB.get(i);
117 assertArrayEquals(a.getFamily(), b.getFamily());
118 assertArrayEquals(a.getQualifier(), b.getQualifier());
119 assertArrayEquals(a.getValue(), b.getValue());
120 }
121 }
122
123 @BeforeClass
124 public static void beforeClass() throws Exception {
125 SUPERUSER = User.createUserForTesting(conf, "admin",
126 new String[] { "supergroup" });
127 conf = UTIL.getConfiguration();
128 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
129 SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
130 conf.set("hbase.superuser", SUPERUSER.getShortName());
131 conf.set("hbase.coprocessor.master.classes",
132 VisibilityController.class.getName());
133 conf.set("hbase.coprocessor.region.classes",
134 VisibilityController.class.getName());
135 conf.setInt("hfile.format.version", 3);
136 UTIL.startMiniCluster(1);
137
138 UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
139 createLabels();
140 Admin admin = new HBaseAdmin(UTIL.getConfiguration());
141 HTableDescriptor tableDescriptor = new HTableDescriptor(
142 TableName.valueOf(tableAname));
143 for (HColumnDescriptor family : families) {
144 tableDescriptor.addFamily(family);
145 }
146 admin.createTable(tableDescriptor);
147 admin.close();
148 setAuths();
149 }
150
151 private static void createLabels() throws IOException, InterruptedException {
152 PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
153 public VisibilityLabelsResponse run() throws Exception {
154 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
155 try {
156 VisibilityClient.addLabels(conf, labels);
157 } catch (Throwable t) {
158 throw new IOException(t);
159 }
160 return null;
161 }
162 };
163 SUPERUSER.runAs(action);
164 }
165
166 private static void setAuths() throws IOException {
167 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
168 try {
169 VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
170 } catch (Throwable t) {
171 throw new IOException(t);
172 }
173 }
174
175 @AfterClass
176 public static void afterClass() throws Exception {
177 UTIL.shutdownMiniCluster();
178 }
179
180 @Before
181 public void setup() throws Exception {
182
183 }
184
185 private ThriftHBaseServiceHandler createHandler() throws IOException {
186 return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
187 }
188
189 @Test
190 public void testScanWithVisibilityLabels() throws Exception {
191 ThriftHBaseServiceHandler handler = createHandler();
192 ByteBuffer table = wrap(tableAname);
193
194
195 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
196 wrap(qualifierAname), wrap(valueAname));
197 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
198 columnValues.add(columnValue);
199 for (int i = 0; i < 10; i++) {
200 TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
201 if (i == 5) {
202 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
203 } else {
204 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
205 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
206 }
207 handler.put(table, put);
208 }
209
210
211 TScan scan = new TScan();
212 List<TColumn> columns = new ArrayList<TColumn>();
213 TColumn column = new TColumn();
214 column.setFamily(familyAname);
215 column.setQualifier(qualifierAname);
216 columns.add(column);
217 scan.setColumns(columns);
218 scan.setStartRow("testScan".getBytes());
219 scan.setStopRow("testScan\uffff".getBytes());
220
221 TAuthorization tauth = new TAuthorization();
222 List<String> labels = new ArrayList<String>();
223 labels.add(SECRET);
224 labels.add(PRIVATE);
225 tauth.setLabels(labels);
226 scan.setAuthorizations(tauth);
227
228 int scanId = handler.openScanner(table, scan);
229 List<TResult> results = handler.getScannerRows(scanId, 10);
230 assertEquals(9, results.size());
231 Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
232 ("testScan" + 5).getBytes()));
233 for (int i = 0; i < 9; i++) {
234 if (i < 5) {
235 assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
236 } else if (i == 5) {
237 continue;
238 } else {
239 assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
240 .getRow());
241 }
242 }
243
244
245 results = handler.getScannerRows(scanId, 9);
246 assertEquals(0, results.size());
247
248
249 handler.closeScanner(scanId);
250 try {
251 handler.getScannerRows(scanId, 9);
252 fail("Scanner id should be invalid");
253 } catch (TIllegalArgument e) {
254 }
255 }
256
257 @Test
258 public void testGetScannerResultsWithAuthorizations() throws Exception {
259 ThriftHBaseServiceHandler handler = createHandler();
260 ByteBuffer table = wrap(tableAname);
261
262
263 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
264 wrap(qualifierAname), wrap(valueAname));
265 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
266 columnValues.add(columnValue);
267 for (int i = 0; i < 20; i++) {
268 TPut put = new TPut(
269 wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
270 columnValues);
271 if (i == 3) {
272 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
273 } else {
274 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
275 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
276 }
277 handler.put(table, put);
278 }
279
280
281 TScan scan = new TScan();
282 List<TColumn> columns = new ArrayList<TColumn>();
283 TColumn column = new TColumn();
284 column.setFamily(familyAname);
285 column.setQualifier(qualifierAname);
286 columns.add(column);
287 scan.setColumns(columns);
288 scan.setStartRow("testGetScannerResults".getBytes());
289
290
291 scan.setStopRow("testGetScannerResults05".getBytes());
292 TAuthorization tauth = new TAuthorization();
293 List<String> labels = new ArrayList<String>();
294 labels.add(SECRET);
295 labels.add(PRIVATE);
296 tauth.setLabels(labels);
297 scan.setAuthorizations(tauth);
298 List<TResult> results = handler.getScannerResults(table, scan, 5);
299 assertEquals(4, results.size());
300 for (int i = 0; i < 4; i++) {
301 if (i < 3) {
302 assertArrayEquals(
303 ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
304 results.get(i).getRow());
305 } else if (i == 3) {
306 continue;
307 } else {
308 assertArrayEquals(
309 ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
310 .get(i).getRow());
311 }
312 }
313 }
314
315 @Test
316 public void testGetsWithLabels() throws Exception {
317 ThriftHBaseServiceHandler handler = createHandler();
318 byte[] rowName = "testPutGet".getBytes();
319 ByteBuffer table = wrap(tableAname);
320
321 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
322 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
323 wrap(valueAname)));
324 columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
325 wrap(valueBname)));
326 TPut put = new TPut(wrap(rowName), columnValues);
327
328 put.setColumnValues(columnValues);
329 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
330 + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
331 handler.put(table, put);
332 TGet get = new TGet(wrap(rowName));
333 TAuthorization tauth = new TAuthorization();
334 List<String> labels = new ArrayList<String>();
335 labels.add(SECRET);
336 labels.add(PRIVATE);
337 tauth.setLabels(labels);
338 get.setAuthorizations(tauth);
339 TResult result = handler.get(table, get);
340 assertArrayEquals(rowName, result.getRow());
341 List<TColumnValue> returnedColumnValues = result.getColumnValues();
342 assertTColumnValuesEqual(columnValues, returnedColumnValues);
343 }
344
345 @Test
346 public void testIncrementWithTags() throws Exception {
347 ThriftHBaseServiceHandler handler = createHandler();
348 byte[] rowName = "testIncrementWithTags".getBytes();
349 ByteBuffer table = wrap(tableAname);
350
351 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
352 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
353 wrap(Bytes.toBytes(1L))));
354 TPut put = new TPut(wrap(rowName), columnValues);
355 put.setColumnValues(columnValues);
356 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
357 handler.put(table, put);
358
359 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
360 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
361 wrap(qualifierAname)));
362 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
363 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
364 handler.increment(table, increment);
365
366 TGet get = new TGet(wrap(rowName));
367 TAuthorization tauth = new TAuthorization();
368 List<String> labels = new ArrayList<String>();
369 labels.add(SECRET);
370 tauth.setLabels(labels);
371 get.setAuthorizations(tauth);
372 TResult result = handler.get(table, get);
373
374 assertArrayEquals(rowName, result.getRow());
375 assertEquals(1, result.getColumnValuesSize());
376 TColumnValue columnValue = result.getColumnValues().get(0);
377 assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
378 }
379
380 @Test
381 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
382 ThriftHBaseServiceHandler handler = createHandler();
383 byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
384 ByteBuffer table = wrap(tableAname);
385
386 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
387 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
388 wrap(Bytes.toBytes(1L))));
389 TPut put = new TPut(wrap(rowName), columnValues);
390 put.setColumnValues(columnValues);
391 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
392 handler.put(table, put);
393
394 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
395 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
396 wrap(qualifierAname)));
397 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
398 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
399 handler.increment(table, increment);
400
401 TGet get = new TGet(wrap(rowName));
402 TAuthorization tauth = new TAuthorization();
403 List<String> labels = new ArrayList<String>();
404 labels.add(PUBLIC);
405 tauth.setLabels(labels);
406 get.setAuthorizations(tauth);
407 TResult result = handler.get(table, get);
408 assertNull(result.getRow());
409 }
410
411 @Test
412 public void testAppend() throws Exception {
413 ThriftHBaseServiceHandler handler = createHandler();
414 byte[] rowName = "testAppend".getBytes();
415 ByteBuffer table = wrap(tableAname);
416 byte[] v1 = Bytes.toBytes(1L);
417 byte[] v2 = Bytes.toBytes(5L);
418 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
419 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
420 wrap(Bytes.toBytes(1L))));
421 TPut put = new TPut(wrap(rowName), columnValues);
422 put.setColumnValues(columnValues);
423 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
424 handler.put(table, put);
425
426 List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
427 appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
428 wrap(v2)));
429 TAppend append = new TAppend(wrap(rowName), appendColumns);
430 append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
431 handler.append(table, append);
432
433 TGet get = new TGet(wrap(rowName));
434 TAuthorization tauth = new TAuthorization();
435 List<String> labels = new ArrayList<String>();
436 labels.add(SECRET);
437 tauth.setLabels(labels);
438 get.setAuthorizations(tauth);
439 TResult result = handler.get(table, get);
440
441 assertArrayEquals(rowName, result.getRow());
442 assertEquals(1, result.getColumnValuesSize());
443 TColumnValue columnValue = result.getColumnValues().get(0);
444 assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
445 }
446
447
448
449
450
451
452
453
454
455
456 private String pad(int n, byte pad) {
457 String res = Integer.toString(n);
458 while (res.length() < pad)
459 res = "0" + res;
460 return res;
461 }
462 }