View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
21  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
22  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertNotNull;
26  import static org.junit.Assert.assertNull;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  
30  import java.io.IOException;
31  import java.security.PrivilegedExceptionAction;
32  import java.util.ArrayList;
33  import java.util.Collection;
34  import java.util.List;
35  
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.Cell;
38  import org.apache.hadoop.hbase.CellScanner;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.client.Admin;
46  import org.apache.hadoop.hbase.client.Append;
47  import org.apache.hadoop.hbase.client.Connection;
48  import org.apache.hadoop.hbase.client.ConnectionFactory;
49  import org.apache.hadoop.hbase.client.Get;
50  import org.apache.hadoop.hbase.client.Increment;
51  import org.apache.hadoop.hbase.client.Put;
52  import org.apache.hadoop.hbase.client.Result;
53  import org.apache.hadoop.hbase.client.ResultScanner;
54  import org.apache.hadoop.hbase.client.RowMutations;
55  import org.apache.hadoop.hbase.client.Scan;
56  import org.apache.hadoop.hbase.client.Table;
57  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
58  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
59  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
60  import org.apache.hadoop.hbase.regionserver.BloomType;
61  import org.apache.hadoop.hbase.regionserver.HRegion;
62  import org.apache.hadoop.hbase.regionserver.HRegionServer;
63  import org.apache.hadoop.hbase.regionserver.Region;
64  import org.apache.hadoop.hbase.regionserver.Store;
65  import org.apache.hadoop.hbase.regionserver.StoreFile;
66  import org.apache.hadoop.hbase.security.User;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
69  import org.junit.After;
70  import org.junit.AfterClass;
71  import org.junit.Rule;
72  import org.junit.Test;
73  import org.junit.rules.TestName;
74  
75  import com.google.protobuf.ByteString;
76  
77  /**
78   * Base test class for visibility labels basic features
79   */
80  public abstract class TestVisibilityLabels {
81  
82    public static final String TOPSECRET = "topsecret";
83    public static final String PUBLIC = "public";
84    public static final String PRIVATE = "private";
85    public static final String CONFIDENTIAL = "confidential";
86    public static final String SECRET = "secret";
87    public static final String COPYRIGHT = "\u00A9ABC";
88    public static final String ACCENT = "\u0941";
89    public static final String UNICODE_VIS_TAG = COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\""
90        + "\u0027&\\";
91    public static final String UC1 = "\u0027\"\u002b";
92    public static final String UC2 = "\u002d\u003f";
93    public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
94    public static final byte[] row1 = Bytes.toBytes("row1");
95    public static final byte[] row2 = Bytes.toBytes("row2");
96    public static final byte[] row3 = Bytes.toBytes("row3");
97    public static final byte[] row4 = Bytes.toBytes("row4");
98    public final static byte[] fam = Bytes.toBytes("info");
99    public final static byte[] qual = Bytes.toBytes("qual");
100   public final static byte[] value = Bytes.toBytes("value");
101   public static Configuration conf;
102 
103   private volatile boolean killedRS = false;
104   @Rule 
105   public final TestName TEST_NAME = new TestName();
106   public static User SUPERUSER, USER1;
107 
108   @AfterClass
109   public static void tearDownAfterClass() throws Exception {
110     TEST_UTIL.shutdownMiniCluster();
111   }
112 
113   @After
114   public void tearDown() throws Exception {
115     killedRS = false;
116   }
117 
118   @Test
119   public void testSimpleVisibilityLabels() throws Exception {
120     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
121     try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
122         PRIVATE + "|" + CONFIDENTIAL)) {
123       Scan s = new Scan();
124       s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
125       ResultScanner scanner = table.getScanner(s);
126       Result[] next = scanner.next(3);
127 
128       assertTrue(next.length == 2);
129       CellScanner cellScanner = next[0].cellScanner();
130       cellScanner.advance();
131       Cell current = cellScanner.current();
132       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
133           current.getRowLength(), row1, 0, row1.length));
134       cellScanner = next[1].cellScanner();
135       cellScanner.advance();
136       current = cellScanner.current();
137       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
138           current.getRowLength(), row2, 0, row2.length));
139     }
140   }
141   
142   @Test
143   public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
144     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
145     try (Table table = createTableAndWriteDataWithLabels(tableName,
146         SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT)
147             + "&"  + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
148         CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
149       Scan s = new Scan();
150       s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT,
151           UNICODE_VIS_TAG));
152       ResultScanner scanner = table.getScanner(s);
153       Result[] next = scanner.next(3);
154       assertTrue(next.length == 3);
155       CellScanner cellScanner = next[0].cellScanner();
156       cellScanner.advance();
157       Cell current = cellScanner.current();
158       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
159           current.getRowLength(), row1, 0, row1.length));
160       cellScanner = next[1].cellScanner();
161       cellScanner.advance();
162       current = cellScanner.current();
163       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
164           current.getRowLength(), row2, 0, row2.length));
165       cellScanner = next[2].cellScanner();
166       cellScanner.advance();
167       current = cellScanner.current();
168       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
169           current.getRowLength(), row3, 0, row3.length));
170     }
171   }
172 
173   @Test
174   public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
175     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
176     try (Table table = createTableAndWriteDataWithLabels(tableName,
177         CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
178         CellVisibility.quote(UNICODE_VIS_TAG))) {
179       Scan s = new Scan();
180       s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT,
181           UNICODE_VIS_TAG));
182       ResultScanner scanner = table.getScanner(s);
183       Result[] next = scanner.next(3);
184       assertTrue(next.length == 3);
185       CellScanner cellScanner = next[0].cellScanner();
186       cellScanner.advance();
187       Cell current = cellScanner.current();
188       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
189           current.getRowLength(), row1, 0, row1.length));
190       cellScanner = next[1].cellScanner();
191       cellScanner.advance();
192       current = cellScanner.current();
193       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
194           current.getRowLength(), row2, 0, row2.length));
195       cellScanner = next[2].cellScanner();
196       cellScanner.advance();
197       current = cellScanner.current();
198       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
199           current.getRowLength(), row3, 0, row3.length));
200     }
201   }
202 
203   @Test
204   public void testVisibilityLabelsWithComplexLabels() throws Exception {
205     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
206     try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
207         + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&"
208         + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE
209         + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
210       Scan s = new Scan();
211       s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
212       ResultScanner scanner = table.getScanner(s);
213       Result[] next = scanner.next(4);
214       assertEquals(3, next.length);
215       CellScanner cellScanner = next[0].cellScanner();
216       cellScanner.advance();
217       Cell current = cellScanner.current();
218       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
219           current.getRowLength(), row2, 0, row2.length));
220       cellScanner = next[1].cellScanner();
221       cellScanner.advance();
222       current = cellScanner.current();
223       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
224           current.getRowLength(), row3, 0, row3.length));
225       cellScanner = next[2].cellScanner();
226       cellScanner.advance();
227       current = cellScanner.current();
228       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
229           current.getRowLength(), row4, 0, row4.length));
230     }
231   }
232 
233   @Test
234   public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
235     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
236     try (Table table = createTableAndWriteDataWithLabels(tableName,
237         "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)){
238       Scan s = new Scan();
239       s.setAuthorizations(new Authorizations(PUBLIC));
240       ResultScanner scanner = table.getScanner(s);
241       Result[] next = scanner.next(3);
242       assertTrue(next.length == 0);
243     }
244   }
245 
246   @Test
247   public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
248     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
249     try {
250       createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
251       fail("Should have failed with failed sanity check exception");
252     } catch (Exception e) {
253     }
254   }
255 
256   @Test
257   public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
258     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
259     try ( Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
260         + CONFIDENTIAL + ")", PRIVATE)){
261       Scan s = new Scan();
262       s.setAuthorizations(new Authorizations("SAMPLE"));
263       ResultScanner scanner = table.getScanner(s);
264       Result[] next = scanner.next(3);
265       assertTrue(next.length == 0);
266     }
267   }
268 
269   @Test
270   public void testVisibilityLabelsWithGet() throws Exception {
271     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
272     try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
273         + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
274       Get get = new Get(row1);
275       get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
276       Result result = table.get(get);
277       assertTrue(!result.isEmpty());
278       Cell cell = result.getColumnLatestCell(fam, qual);
279       assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
280           cell.getValueLength()));
281     }
282   }
283 
284   @Test
285   public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
286     List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
287         .getRegionServerThreads();
288     int liveRS = 0;
289     for (RegionServerThread rsThreads : regionServerThreads) {
290       if (!rsThreads.getRegionServer().isAborted()) {
291         liveRS++;
292       }
293     }
294     if (liveRS == 1) {
295       TEST_UTIL.getHBaseCluster().startRegionServer();
296     }
297     Thread t1 = new Thread() {
298       public void run() {
299         List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
300             .getRegionServerThreads();
301         for (RegionServerThread rsThread : regionServerThreads) {
302           List<Region> onlineRegions = rsThread.getRegionServer().getOnlineRegions(
303               LABELS_TABLE_NAME);
304           if (onlineRegions.size() > 0) {
305             rsThread.getRegionServer().abort("Aborting ");
306             killedRS = true;
307             break;
308           }
309         }
310       }
311 
312     };
313     t1.start();
314     final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
315     Thread t = new Thread() {
316       public void run() {
317         try {
318           while (!killedRS) {
319             Thread.sleep(1);
320           }
321           createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
322               PRIVATE);
323         } catch (Exception e) {
324         }
325       }
326     };
327     t.start();
328     regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
329     while (!killedRS) {
330       Thread.sleep(10);
331     }
332     regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
333     for (RegionServerThread rsThread : regionServerThreads) {
334       while (true) {
335         if (!rsThread.getRegionServer().isAborted()) {
336           List<Region> onlineRegions = rsThread.getRegionServer().getOnlineRegions(
337               LABELS_TABLE_NAME);
338           if (onlineRegions.size() > 0) {
339             break;
340           } else {
341             Thread.sleep(10);
342           }
343         } else {
344           break;
345         }
346       }
347     }
348     TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
349     t.join();
350     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
351       Scan s = new Scan();
352       s.setAuthorizations(new Authorizations(SECRET));
353       ResultScanner scanner = table.getScanner(s);
354       Result[] next = scanner.next(3);
355       assertTrue(next.length == 1);
356     }
357   }
358 
359   @Test(timeout = 60 * 1000)
360   public void testVisibilityLabelsOnRSRestart() throws Exception {
361     final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
362     List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
363         .getRegionServerThreads();
364     for (RegionServerThread rsThread : regionServerThreads) {
365       rsThread.getRegionServer().abort("Aborting ");
366     }
367     // Start one new RS
368     RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
369     waitForLabelsRegionAvailability(rs.getRegionServer());
370     try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
371         + ")", PRIVATE);) {
372       Scan s = new Scan();
373       s.setAuthorizations(new Authorizations(SECRET));
374       ResultScanner scanner = table.getScanner(s);
375       Result[] next = scanner.next(3);
376       assertTrue(next.length == 1);
377     }
378   }
379 
380   protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
381     while (!regionServer.isOnline()) {
382       try {
383         Thread.sleep(10);
384       } catch (InterruptedException e) {
385       }
386     }
387     while (regionServer.getOnlineRegions(LABELS_TABLE_NAME).isEmpty()) {
388       try {
389         Thread.sleep(10);
390       } catch (InterruptedException e) {
391       }
392     }
393     Region labelsTableRegion = regionServer.getOnlineRegions(LABELS_TABLE_NAME).get(0);
394     while (labelsTableRegion.isRecovering()) {
395       try {
396         Thread.sleep(10);
397       } catch (InterruptedException e) {
398       }
399     }
400   }
401 
402   @Test
403   public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
404     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
405     try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
406         + ")", PRIVATE)) {
407       Get get = new Get(row1);
408       get.setAuthorizations(new Authorizations("SAMPLE"));
409       Result result = table.get(get);
410       assertTrue(result.isEmpty());
411     }
412   }
413 
414   @Test
415   public void testSetAndGetUserAuths() throws Throwable {
416     final String user = "user1";
417     PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
418       public Void run() throws Exception {
419         String[] auths = { SECRET, CONFIDENTIAL };
420         try {
421           VisibilityClient.setAuths(conf, auths, user);
422         } catch (Throwable e) {
423         }
424         return null;
425       }
426     };
427     SUPERUSER.runAs(action);
428     try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME);) {
429       Scan scan = new Scan();
430       scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
431       ResultScanner scanner = ht.getScanner(scan);
432       Result result = null;
433       List<Result> results = new ArrayList<Result>();
434       while ((result = scanner.next()) != null) {
435         results.add(result);
436       }
437       List<String> auths = extractAuths(user, results);
438       assertTrue(auths.contains(SECRET));
439       assertTrue(auths.contains(CONFIDENTIAL));
440       assertEquals(2, auths.size());
441     }
442 
443     action = new PrivilegedExceptionAction<Void>() {
444       public Void run() throws Exception {
445         GetAuthsResponse authsResponse = null;
446         try {
447           authsResponse = VisibilityClient.getAuths(conf, user);
448         } catch (Throwable e) {
449           fail("Should not have failed");
450         }
451         List<String> authsList = new ArrayList<String>();
452         for (ByteString authBS : authsResponse.getAuthList()) {
453           authsList.add(Bytes.toString(authBS.toByteArray()));
454         }
455         assertEquals(2, authsList.size());
456         assertTrue(authsList.contains(SECRET));
457         assertTrue(authsList.contains(CONFIDENTIAL));
458         return null;
459       }
460     };
461     SUPERUSER.runAs(action);
462 
463     // Try doing setAuths once again and there should not be any duplicates
464     action = new PrivilegedExceptionAction<Void>() {
465       public Void run() throws Exception {
466         String[] auths1 = { SECRET, CONFIDENTIAL };
467         GetAuthsResponse authsResponse = null;
468         try {
469           VisibilityClient.setAuths(conf, auths1, user);
470           try {
471             authsResponse = VisibilityClient.getAuths(conf, user);
472           } catch (Throwable e) {
473             fail("Should not have failed");
474           }
475         } catch (Throwable e) {
476         }
477         List<String> authsList = new ArrayList<String>();
478         for (ByteString authBS : authsResponse.getAuthList()) {
479           authsList.add(Bytes.toString(authBS.toByteArray()));
480         }
481         assertEquals(2, authsList.size());
482         assertTrue(authsList.contains(SECRET));
483         assertTrue(authsList.contains(CONFIDENTIAL));
484         return null;
485       }
486     };
487     SUPERUSER.runAs(action);
488   }
489 
490   protected List<String> extractAuths(String user, List<Result> results) {
491     List<String> auths = new ArrayList<String>();
492     for (Result result : results) {
493       Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
494       Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes());
495       if (userAuthCell != null) {
496         auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
497             labelCell.getValueLength()));
498       }
499     }
500     return auths;
501   }
502 
503   @Test
504   public void testClearUserAuths() throws Throwable {
505     PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
506       public Void run() throws Exception {
507         String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
508         String user = "testUser";
509         try {
510           VisibilityClient.setAuths(conf, auths, user);
511         } catch (Throwable e) {
512           fail("Should not have failed");
513         }
514         // Removing the auths for SECRET and CONFIDENTIAL for the user.
515         // Passing a non existing auth also.
516         auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
517         VisibilityLabelsResponse response = null;
518         try {
519           response = VisibilityClient.clearAuths(conf, auths, user);
520         } catch (Throwable e) {
521           fail("Should not have failed");
522         }
523         List<RegionActionResult> resultList = response.getResultList();
524         assertEquals(3, resultList.size());
525         assertTrue(resultList.get(0).getException().getValue().isEmpty());
526         assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
527             resultList.get(1).getException().getName());
528         assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
529             .contains(
530                 "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
531                     + "Label 'public' is not set for the user testUser"));
532         assertTrue(resultList.get(2).getException().getValue().isEmpty());
533         try (Connection connection = ConnectionFactory.createConnection(conf);
534              Table ht = connection.getTable(LABELS_TABLE_NAME)) {
535           ResultScanner scanner = ht.getScanner(new Scan());
536           Result result = null;
537           List<Result> results = new ArrayList<Result>();
538           while ((result = scanner.next()) != null) {
539             results.add(result);
540           }
541           List<String> curAuths = extractAuths(user, results);
542           assertTrue(curAuths.contains(PRIVATE));
543           assertEquals(1, curAuths.size());
544         }
545 
546         GetAuthsResponse authsResponse = null;
547         try {
548           authsResponse = VisibilityClient.getAuths(conf, user);
549         } catch (Throwable e) {
550           fail("Should not have failed");
551         }
552         List<String> authsList = new ArrayList<String>();
553         for (ByteString authBS : authsResponse.getAuthList()) {
554           authsList.add(Bytes.toString(authBS.toByteArray()));
555         }
556         assertEquals(1, authsList.size());
557         assertTrue(authsList.contains(PRIVATE));
558         return null;
559       }
560     };
561     SUPERUSER.runAs(action);
562   }
563 
564   @Test
565   public void testLabelsWithCheckAndPut() throws Throwable {
566     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
567     try (Table table = TEST_UTIL.createTable(tableName, fam)) {
568       byte[] row1 = Bytes.toBytes("row1");
569       Put put = new Put(row1);
570       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
571       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
572       table.checkAndPut(row1, fam, qual, null, put);
573       byte[] row2 = Bytes.toBytes("row2");
574       put = new Put(row2);
575       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
576       put.setCellVisibility(new CellVisibility(SECRET));
577       table.checkAndPut(row2, fam, qual, null, put);
578       
579       Scan scan = new Scan();
580       scan.setAuthorizations(new Authorizations(SECRET));
581       ResultScanner scanner = table.getScanner(scan);
582       Result result = scanner.next();
583       assertTrue(!result.isEmpty());
584       assertTrue(Bytes.equals(row2, result.getRow()));
585       result = scanner.next();
586       assertNull(result);
587     }
588   }
589 
590   @Test
591   public void testLabelsWithIncrement() throws Throwable {
592     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
593     try (Table table = TEST_UTIL.createTable(tableName, fam)) {
594       byte[] row1 = Bytes.toBytes("row1");
595       byte[] val = Bytes.toBytes(1L);
596       Put put = new Put(row1);
597       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, val);
598       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
599       table.put(put);
600       Get get = new Get(row1);
601       get.setAuthorizations(new Authorizations(SECRET));
602       Result result = table.get(get);
603       assertTrue(result.isEmpty());
604       table.incrementColumnValue(row1, fam, qual, 2L);
605       result = table.get(get);
606       assertTrue(result.isEmpty());
607       Increment increment = new Increment(row1);
608       increment.addColumn(fam, qual, 2L);
609       increment.setCellVisibility(new CellVisibility(SECRET));
610       table.increment(increment);
611       result = table.get(get);
612       assertTrue(!result.isEmpty());
613     }
614   }
615 
616   @Test
617   public void testLabelsWithAppend() throws Throwable {
618     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
619     try (Table table = TEST_UTIL.createTable(tableName, fam);) {
620       byte[] row1 = Bytes.toBytes("row1");
621       byte[] val = Bytes.toBytes("a");
622       Put put = new Put(row1);
623       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, val);
624       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
625       table.put(put);
626       Get get = new Get(row1);
627       get.setAuthorizations(new Authorizations(SECRET));
628       Result result = table.get(get);
629       assertTrue(result.isEmpty());
630       Append append = new Append(row1);
631       append.add(fam, qual, Bytes.toBytes("b"));
632       table.append(append);
633       result = table.get(get);
634       assertTrue(result.isEmpty());
635       append = new Append(row1);
636       append.add(fam, qual, Bytes.toBytes("c"));
637       append.setCellVisibility(new CellVisibility(SECRET));
638       table.append(append);
639       result = table.get(get);
640       assertTrue(!result.isEmpty());
641     }
642   }
643 
644   @Test
645   public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
646     Admin admin = TEST_UTIL.getHBaseAdmin();
647     try {
648       admin.disableTable(LABELS_TABLE_NAME);
649       fail("Lables table should not get disabled by user.");
650     } catch (Exception e) {
651     }
652     try {
653       admin.deleteTable(LABELS_TABLE_NAME);
654       fail("Lables table should not get disabled by user.");
655     } catch (Exception e) {
656     }
657     try {
658       HColumnDescriptor hcd = new HColumnDescriptor("testFamily");
659       admin.addColumn(LABELS_TABLE_NAME, hcd);
660       fail("Lables table should not get altered by user.");
661     } catch (Exception e) {
662     }
663     try {
664       admin.deleteColumn(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
665       fail("Lables table should not get altered by user.");
666     } catch (Exception e) {
667     }
668     try {
669       HColumnDescriptor hcd = new HColumnDescriptor(VisibilityConstants.LABELS_TABLE_FAMILY);
670       hcd.setBloomFilterType(BloomType.ROWCOL);
671       admin.modifyColumn(LABELS_TABLE_NAME, hcd);
672       fail("Lables table should not get altered by user.");
673     } catch (Exception e) {
674     }
675     try {
676       HTableDescriptor htd = new HTableDescriptor(LABELS_TABLE_NAME);
677       htd.addFamily(new HColumnDescriptor("f1"));
678       htd.addFamily(new HColumnDescriptor("f2"));
679       admin.modifyTable(LABELS_TABLE_NAME, htd);
680       fail("Lables table should not get altered by user.");
681     } catch (Exception e) {
682     }
683   }
684 
685   @Test
686   public void testMultipleVersions() throws Exception {
687     final byte[] r1 = Bytes.toBytes("row1");
688     final byte[] r2 = Bytes.toBytes("row2");
689     final byte[] v1 = Bytes.toBytes("100");
690     final byte[] v2 = Bytes.toBytes("101");
691     final byte[] fam2 = Bytes.toBytes("info2");
692     final byte[] qual2 = Bytes.toBytes("qual2");
693     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
694     HTableDescriptor desc = new HTableDescriptor(tableName);
695     HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1.
696     desc.addFamily(col);
697     col = new HColumnDescriptor(fam2);
698     col.setMaxVersions(5);
699     desc.addFamily(col);
700     TEST_UTIL.getHBaseAdmin().createTable(desc);
701     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
702       Put put = new Put(r1);
703       put.add(fam, qual, 3l, v1);
704       put.add(fam, qual2, 3l, v1);
705       put.add(fam2, qual, 3l, v1);
706       put.add(fam2, qual2, 3l, v1);
707       put.setCellVisibility(new CellVisibility(SECRET));
708       table.put(put);
709       put = new Put(r1);
710       put.add(fam, qual, 4l, v2);
711       put.add(fam, qual2, 4l, v2);
712       put.add(fam2, qual, 4l, v2);
713       put.add(fam2, qual2, 4l, v2);
714       put.setCellVisibility(new CellVisibility(PRIVATE));
715       table.put(put);
716 
717       put = new Put(r2);
718       put.add(fam, qual, 3l, v1);
719       put.add(fam, qual2, 3l, v1);
720       put.add(fam2, qual, 3l, v1);
721       put.add(fam2, qual2, 3l, v1);
722       put.setCellVisibility(new CellVisibility(SECRET));
723       table.put(put);
724       put = new Put(r2);
725       put.add(fam, qual, 4l, v2);
726       put.add(fam, qual2, 4l, v2);
727       put.add(fam2, qual, 4l, v2);
728       put.add(fam2, qual2, 4l, v2);
729       put.setCellVisibility(new CellVisibility(SECRET));
730       table.put(put);
731 
732       Scan s = new Scan();
733       s.setMaxVersions(1);
734       s.setAuthorizations(new Authorizations(SECRET));
735       ResultScanner scanner = table.getScanner(s);
736       Result result = scanner.next();
737       assertTrue(Bytes.equals(r1, result.getRow()));
738       // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
739       // CellVisibility with Authorizations, should not get considered in the label evaluation at
740       // all.
741       assertNull(result.getColumnLatestCell(fam, qual));
742       assertNull(result.getColumnLatestCell(fam, qual2));
743       // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
744       // are having matching CellVisibility with Authorizations, in the label evaluation. It can
745       // just skip those recent versions for which visibility is not there as per the new version's
746       // CellVisibility. The old versions which are having visibility can be send back
747       Cell cell = result.getColumnLatestCell(fam2, qual);
748       assertNotNull(cell);
749       assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
750           cell.getValueLength()));
751       cell = result.getColumnLatestCell(fam2, qual2);
752       assertNotNull(cell);
753       assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
754           cell.getValueLength()));
755 
756       result = scanner.next();
757       assertTrue(Bytes.equals(r2, result.getRow()));
758       cell = result.getColumnLatestCell(fam, qual);
759       assertNotNull(cell);
760       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
761           cell.getValueLength()));
762       cell = result.getColumnLatestCell(fam, qual2);
763       assertNotNull(cell);
764       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
765           cell.getValueLength()));
766       cell = result.getColumnLatestCell(fam2, qual);
767       assertNotNull(cell);
768       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
769           cell.getValueLength()));
770       cell = result.getColumnLatestCell(fam2, qual2);
771       assertNotNull(cell);
772       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
773           cell.getValueLength()));
774     }
775   }
776 
777   @Test
778   public void testMutateRow() throws Exception {
779     final byte[] qual2 = Bytes.toBytes("qual2");
780     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
781     HTableDescriptor desc = new HTableDescriptor(tableName);
782     HColumnDescriptor col = new HColumnDescriptor(fam);
783     desc.addFamily(col);
784     TEST_UTIL.getHBaseAdmin().createTable(desc);
785     try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
786       Put p1 = new Put(row1);
787       p1.add(fam, qual, value);
788       p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
789 
790       Put p2 = new Put(row1);
791       p2.add(fam, qual2, value);
792       p2.setCellVisibility(new CellVisibility(SECRET));
793 
794       RowMutations rm = new RowMutations(row1);
795       rm.add(p1);
796       rm.add(p2);
797 
798       table.mutateRow(rm);
799 
800       Get get = new Get(row1);
801       get.setAuthorizations(new Authorizations(CONFIDENTIAL));
802       Result result = table.get(get);
803       assertTrue(result.containsColumn(fam, qual));
804       assertFalse(result.containsColumn(fam, qual2));
805 
806       get.setAuthorizations(new Authorizations(SECRET));
807       result = table.get(get);
808       assertFalse(result.containsColumn(fam, qual));
809       assertTrue(result.containsColumn(fam, qual2));
810     }
811   }
812 
813   @Test
814   public void testFlushedFileWithVisibilityTags() throws Exception {
815     final byte[] qual2 = Bytes.toBytes("qual2");
816     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
817     HTableDescriptor desc = new HTableDescriptor(tableName);
818     HColumnDescriptor col = new HColumnDescriptor(fam);
819     desc.addFamily(col);
820     TEST_UTIL.getHBaseAdmin().createTable(desc);
821     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
822       Put p1 = new Put(row1);
823       p1.add(fam, qual, value);
824       p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
825 
826       Put p2 = new Put(row1);
827       p2.add(fam, qual2, value);
828       p2.setCellVisibility(new CellVisibility(SECRET));
829 
830       RowMutations rm = new RowMutations(row1);
831       rm.add(p1);
832       rm.add(p2);
833 
834       table.mutateRow(rm);
835     }
836     TEST_UTIL.getHBaseAdmin().flush(tableName);
837     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
838     Store store = regions.get(0).getStore(fam);
839     Collection<StoreFile> storefiles = store.getStorefiles();
840     assertTrue(storefiles.size() > 0);
841     for (StoreFile storeFile : storefiles) {
842       assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
843     }
844   }
845 
846   static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
847       throws Exception {
848     List<Put> puts = new ArrayList<Put>();
849     for (int i = 0; i < labelExps.length; i++) {
850       Put put = new Put(Bytes.toBytes("row" + (i+1)));
851       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
852       put.setCellVisibility(new CellVisibility(labelExps[i]));
853       puts.add(put);
854     }
855     Table table = TEST_UTIL.createTable(tableName, fam);
856     table.put(puts);
857     return table;
858   }
859 
860   public static void addLabels() throws Exception {
861     PrivilegedExceptionAction<VisibilityLabelsResponse> action =
862         new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
863       public VisibilityLabelsResponse run() throws Exception {
864         String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
865             UNICODE_VIS_TAG, UC1, UC2 };
866         try {
867           VisibilityClient.addLabels(conf, labels);
868         } catch (Throwable t) {
869           throw new IOException(t);
870         }
871         return null;
872       }
873     };
874     SUPERUSER.runAs(action);
875   }
876 }