View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.thrift2;
19  
20  import static java.nio.ByteBuffer.wrap;
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.nio.ByteBuffer;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.ArrayList;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.List;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.testclassification.MediumTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.client.Admin;
43  import org.apache.hadoop.hbase.client.HBaseAdmin;
44  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
45  import org.apache.hadoop.hbase.security.User;
46  import org.apache.hadoop.hbase.security.UserProvider;
47  import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
48  import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
49  import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
50  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
51  import org.apache.hadoop.hbase.security.visibility.VisibilityController;
52  import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
53  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
54  import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
55  import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
56  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
57  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
58  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
59  import org.apache.hadoop.hbase.thrift2.generated.TGet;
60  import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
61  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
62  import org.apache.hadoop.hbase.thrift2.generated.TPut;
63  import org.apache.hadoop.hbase.thrift2.generated.TResult;
64  import org.apache.hadoop.hbase.thrift2.generated.TScan;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.junit.AfterClass;
67  import org.junit.Assert;
68  import org.junit.Before;
69  import org.junit.BeforeClass;
70  import org.junit.Test;
71  import org.junit.experimental.categories.Category;
72  
73  @Category(MediumTests.class)
74  public class TestThriftHBaseServiceHandlerWithLabels {
75  
76  public static final Log LOG = LogFactory
77      .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
78  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79  
80  // Static names for tables, columns, rows, and values
81  private static byte[] tableAname = Bytes.toBytes("tableA");
82  private static byte[] familyAname = Bytes.toBytes("familyA");
83  private static byte[] familyBname = Bytes.toBytes("familyB");
84  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
85  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
86  private static byte[] valueAname = Bytes.toBytes("valueA");
87  private static byte[] valueBname = Bytes.toBytes("valueB");
88  private static HColumnDescriptor[] families = new HColumnDescriptor[] {
89      new HColumnDescriptor(familyAname).setMaxVersions(3),
90      new HColumnDescriptor(familyBname).setMaxVersions(2) };
91  
92  private final static String TOPSECRET = "topsecret";
93  private final static String PUBLIC = "public";
94  private final static String PRIVATE = "private";
95  private final static String CONFIDENTIAL = "confidential";
96  private final static String SECRET = "secret";
97  private static User SUPERUSER;
98  
99  private static Configuration conf;
100 
101 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
102     List<TColumnValue> columnValuesB) {
103   assertEquals(columnValuesA.size(), columnValuesB.size());
104   Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
105     @Override
106     public int compare(TColumnValue o1, TColumnValue o2) {
107       return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
108           Bytes.add(o2.getFamily(), o2.getQualifier()));
109     }
110   };
111   Collections.sort(columnValuesA, comparator);
112   Collections.sort(columnValuesB, comparator);
113 
114   for (int i = 0; i < columnValuesA.size(); i++) {
115     TColumnValue a = columnValuesA.get(i);
116     TColumnValue b = columnValuesB.get(i);
117     assertArrayEquals(a.getFamily(), b.getFamily());
118     assertArrayEquals(a.getQualifier(), b.getQualifier());
119     assertArrayEquals(a.getValue(), b.getValue());
120   }
121 }
122 
123 @BeforeClass
124 public static void beforeClass() throws Exception {
125   SUPERUSER = User.createUserForTesting(conf, "admin",
126       new String[] { "supergroup" });
127   conf = UTIL.getConfiguration();
128   conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
129       SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
130   conf.set("hbase.superuser", SUPERUSER.getShortName());
131   conf.set("hbase.coprocessor.master.classes",
132       VisibilityController.class.getName());
133   conf.set("hbase.coprocessor.region.classes",
134       VisibilityController.class.getName());
135   conf.setInt("hfile.format.version", 3);
136   UTIL.startMiniCluster(1);
137   // Wait for the labels table to become available
138   UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
139   createLabels();
140   Admin admin = new HBaseAdmin(UTIL.getConfiguration());
141   HTableDescriptor tableDescriptor = new HTableDescriptor(
142       TableName.valueOf(tableAname));
143   for (HColumnDescriptor family : families) {
144     tableDescriptor.addFamily(family);
145   }
146   admin.createTable(tableDescriptor);
147   admin.close();
148   setAuths();
149 }
150 
151 private static void createLabels() throws IOException, InterruptedException {
152   PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
153     public VisibilityLabelsResponse run() throws Exception {
154       String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
155       try {
156         VisibilityClient.addLabels(conf, labels);
157       } catch (Throwable t) {
158         throw new IOException(t);
159       }
160       return null;
161     }
162   };
163   SUPERUSER.runAs(action);
164 }
165 
166 private static void setAuths() throws IOException {
167   String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
168   try {
169     VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
170   } catch (Throwable t) {
171     throw new IOException(t);
172   }
173 }
174 
175 @AfterClass
176 public static void afterClass() throws Exception {
177   UTIL.shutdownMiniCluster();
178 }
179 
180 @Before
181 public void setup() throws Exception {
182 
183 }
184 
185 private ThriftHBaseServiceHandler createHandler() throws IOException {
186   return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
187 }
188 
189 @Test
190 public void testScanWithVisibilityLabels() throws Exception {
191   ThriftHBaseServiceHandler handler = createHandler();
192   ByteBuffer table = wrap(tableAname);
193 
194   // insert data
195   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
196       wrap(qualifierAname), wrap(valueAname));
197   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
198   columnValues.add(columnValue);
199   for (int i = 0; i < 10; i++) {
200     TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
201     if (i == 5) {
202       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
203     } else {
204       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
205           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
206     }
207     handler.put(table, put);
208   }
209 
210   // create scan instance
211   TScan scan = new TScan();
212   List<TColumn> columns = new ArrayList<TColumn>();
213   TColumn column = new TColumn();
214   column.setFamily(familyAname);
215   column.setQualifier(qualifierAname);
216   columns.add(column);
217   scan.setColumns(columns);
218   scan.setStartRow("testScan".getBytes());
219   scan.setStopRow("testScan\uffff".getBytes());
220 
221   TAuthorization tauth = new TAuthorization();
222   List<String> labels = new ArrayList<String>();
223   labels.add(SECRET);
224   labels.add(PRIVATE);
225   tauth.setLabels(labels);
226   scan.setAuthorizations(tauth);
227   // get scanner and rows
228   int scanId = handler.openScanner(table, scan);
229   List<TResult> results = handler.getScannerRows(scanId, 10);
230   assertEquals(9, results.size());
231   Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
232       ("testScan" + 5).getBytes()));
233   for (int i = 0; i < 9; i++) {
234     if (i < 5) {
235       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
236     } else if (i == 5) {
237       continue;
238     } else {
239       assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
240           .getRow());
241     }
242   }
243 
244   // check that we are at the end of the scan
245   results = handler.getScannerRows(scanId, 9);
246   assertEquals(0, results.size());
247 
248   // close scanner and check that it was indeed closed
249   handler.closeScanner(scanId);
250   try {
251     handler.getScannerRows(scanId, 9);
252     fail("Scanner id should be invalid");
253   } catch (TIllegalArgument e) {
254   }
255 }
256 
257 @Test
258 public void testGetScannerResultsWithAuthorizations() throws Exception {
259   ThriftHBaseServiceHandler handler = createHandler();
260   ByteBuffer table = wrap(tableAname);
261 
262   // insert data
263   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
264       wrap(qualifierAname), wrap(valueAname));
265   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
266   columnValues.add(columnValue);
267   for (int i = 0; i < 20; i++) {
268     TPut put = new TPut(
269         wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
270         columnValues);
271     if (i == 3) {
272       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
273     } else {
274       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
275           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
276     }
277     handler.put(table, put);
278   }
279 
280   // create scan instance
281   TScan scan = new TScan();
282   List<TColumn> columns = new ArrayList<TColumn>();
283   TColumn column = new TColumn();
284   column.setFamily(familyAname);
285   column.setQualifier(qualifierAname);
286   columns.add(column);
287   scan.setColumns(columns);
288   scan.setStartRow("testGetScannerResults".getBytes());
289 
290   // get 5 rows and check the returned results
291   scan.setStopRow("testGetScannerResults05".getBytes());
292   TAuthorization tauth = new TAuthorization();
293   List<String> labels = new ArrayList<String>();
294   labels.add(SECRET);
295   labels.add(PRIVATE);
296   tauth.setLabels(labels);
297   scan.setAuthorizations(tauth);
298   List<TResult> results = handler.getScannerResults(table, scan, 5);
299   assertEquals(4, results.size());
300   for (int i = 0; i < 4; i++) {
301     if (i < 3) {
302       assertArrayEquals(
303           ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
304           results.get(i).getRow());
305     } else if (i == 3) {
306       continue;
307     } else {
308       assertArrayEquals(
309           ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
310               .get(i).getRow());
311     }
312   }
313 }
314 
315 @Test
316 public void testGetsWithLabels() throws Exception {
317   ThriftHBaseServiceHandler handler = createHandler();
318   byte[] rowName = "testPutGet".getBytes();
319   ByteBuffer table = wrap(tableAname);
320 
321   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
322   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
323       wrap(valueAname)));
324   columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
325       wrap(valueBname)));
326   TPut put = new TPut(wrap(rowName), columnValues);
327 
328   put.setColumnValues(columnValues);
329   put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
330       + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
331   handler.put(table, put);
332   TGet get = new TGet(wrap(rowName));
333   TAuthorization tauth = new TAuthorization();
334   List<String> labels = new ArrayList<String>();
335   labels.add(SECRET);
336   labels.add(PRIVATE);
337   tauth.setLabels(labels);
338   get.setAuthorizations(tauth);
339   TResult result = handler.get(table, get);
340   assertArrayEquals(rowName, result.getRow());
341   List<TColumnValue> returnedColumnValues = result.getColumnValues();
342   assertTColumnValuesEqual(columnValues, returnedColumnValues);
343 }
344 
345 @Test
346 public void testIncrementWithTags() throws Exception {
347   ThriftHBaseServiceHandler handler = createHandler();
348   byte[] rowName = "testIncrementWithTags".getBytes();
349   ByteBuffer table = wrap(tableAname);
350 
351   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
352   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
353       wrap(Bytes.toBytes(1L))));
354   TPut put = new TPut(wrap(rowName), columnValues);
355   put.setColumnValues(columnValues);
356   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
357   handler.put(table, put);
358 
359   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
360   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
361       wrap(qualifierAname)));
362   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
363   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
364   handler.increment(table, increment);
365 
366   TGet get = new TGet(wrap(rowName));
367   TAuthorization tauth = new TAuthorization();
368   List<String> labels = new ArrayList<String>();
369   labels.add(SECRET);
370   tauth.setLabels(labels);
371   get.setAuthorizations(tauth);
372   TResult result = handler.get(table, get);
373 
374   assertArrayEquals(rowName, result.getRow());
375   assertEquals(1, result.getColumnValuesSize());
376   TColumnValue columnValue = result.getColumnValues().get(0);
377   assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
378 }
379 
380 @Test
381 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
382   ThriftHBaseServiceHandler handler = createHandler();
383   byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
384   ByteBuffer table = wrap(tableAname);
385 
386   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
387   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
388       wrap(Bytes.toBytes(1L))));
389   TPut put = new TPut(wrap(rowName), columnValues);
390   put.setColumnValues(columnValues);
391   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
392   handler.put(table, put);
393 
394   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
395   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
396       wrap(qualifierAname)));
397   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
398   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
399   handler.increment(table, increment);
400 
401   TGet get = new TGet(wrap(rowName));
402   TAuthorization tauth = new TAuthorization();
403   List<String> labels = new ArrayList<String>();
404   labels.add(PUBLIC);
405   tauth.setLabels(labels);
406   get.setAuthorizations(tauth);
407   TResult result = handler.get(table, get);
408   assertNull(result.getRow());
409 }
410 
411 @Test
412 public void testAppend() throws Exception {
413   ThriftHBaseServiceHandler handler = createHandler();
414   byte[] rowName = "testAppend".getBytes();
415   ByteBuffer table = wrap(tableAname);
416   byte[] v1 = Bytes.toBytes(1L);
417   byte[] v2 = Bytes.toBytes(5L);
418   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
419   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
420       wrap(Bytes.toBytes(1L))));
421   TPut put = new TPut(wrap(rowName), columnValues);
422   put.setColumnValues(columnValues);
423   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
424   handler.put(table, put);
425 
426   List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
427   appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
428       wrap(v2)));
429   TAppend append = new TAppend(wrap(rowName), appendColumns);
430   append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
431   handler.append(table, append);
432 
433   TGet get = new TGet(wrap(rowName));
434   TAuthorization tauth = new TAuthorization();
435   List<String> labels = new ArrayList<String>();
436   labels.add(SECRET);
437   tauth.setLabels(labels);
438   get.setAuthorizations(tauth);
439   TResult result = handler.get(table, get);
440 
441   assertArrayEquals(rowName, result.getRow());
442   assertEquals(1, result.getColumnValuesSize());
443   TColumnValue columnValue = result.getColumnValues().get(0);
444   assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
445 }
446 
447 /**
448  * Padding numbers to make comparison of sort order easier in a for loop
449  * 
450  * @param n
451  *          The number to pad.
452  * @param pad
453  *          The length to pad up to.
454  * @return The padded number as a string.
455  */
456 private String pad(int n, byte pad) {
457   String res = Integer.toString(n);
458   while (res.length() < pad)
459     res = "0" + res;
460   return res;
461 }
462 }