View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.ArrayList;
30  import java.util.Arrays;
31  import java.util.List;
32  import java.util.Random;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellUtil;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HRegionLocation;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
45  import org.apache.hadoop.hbase.testclassification.LargeTests;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Pair;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  @Category(LargeTests.class)
56  public class TestFromClientSide3 {
57    final Log LOG = LogFactory.getLog(getClass());
58    private final static HBaseTestingUtility TEST_UTIL
59      = new HBaseTestingUtility();
60    private static byte[] FAMILY = Bytes.toBytes("testFamily");
61    private static Random random = new Random();
62    private static int SLAVES = 3;
63    private static byte [] ROW = Bytes.toBytes("testRow");
64    private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
65    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
66    private static byte [] VALUE = Bytes.toBytes("testValue");
67    private final static byte[] COL_QUAL = Bytes.toBytes("f1");
68    private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
69    private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
70  
71    /**
72     * @throws java.lang.Exception
73     */
74    @BeforeClass
75    public static void setUpBeforeClass() throws Exception {
76      TEST_UTIL.getConfiguration().setBoolean(
77          "hbase.online.schema.update.enable", true);
78      TEST_UTIL.startMiniCluster(SLAVES);
79    }
80  
81    /**
82     * @throws java.lang.Exception
83     */
84    @AfterClass
85    public static void tearDownAfterClass() throws Exception {
86      TEST_UTIL.shutdownMiniCluster();
87    }
88  
89    /**
90     * @throws java.lang.Exception
91     */
92    @Before
93    public void setUp() throws Exception {
94      // Nothing to do.
95    }
96  
97    /**
98     * @throws java.lang.Exception
99     */
100   @After
101   public void tearDown() throws Exception {
102     for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
103       LOG.info("Tear down, remove table=" + htd.getTableName());
104       TEST_UTIL.deleteTable(htd.getTableName());
105     }
106   }
107 
108   private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
109       throws Exception {
110     Put put = new Put(row);
111     for (int i = 0; i < nPuts; i++) {
112       byte[] qualifier = Bytes.toBytes(random.nextInt());
113       byte[] value = Bytes.toBytes(random.nextInt());
114       put.add(family, qualifier, value);
115     }
116     table.put(put);
117   }
118 
119   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
120       byte[] row, byte[] family, int nFlushes, int nPuts)
121   throws Exception {
122 
123     // connection needed for poll-wait
124     HRegionLocation loc = table.getRegionLocation(row, true);
125     AdminProtos.AdminService.BlockingInterface server =
126       admin.getConnection().getAdmin(loc.getServerName());
127     byte[] regName = loc.getRegionInfo().getRegionName();
128 
129     for (int i = 0; i < nFlushes; i++) {
130       randomCFPuts(table, row, family, nPuts);
131       List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
132       int sfCount = sf.size();
133 
134       // TODO: replace this api with a synchronous flush after HBASE-2949
135       admin.flush(table.getTableName());
136 
137       // synchronously poll wait for a new storefile to appear (flush happened)
138       while (ProtobufUtil.getStoreFiles(
139           server, regName, FAMILY).size() == sfCount) {
140         Thread.sleep(40);
141       }
142     }
143   }
144 
145   private static List<Cell> toList(ResultScanner scanner) {
146     try {
147       List<Cell> cells = new ArrayList<>();
148       for (Result r : scanner) {
149         cells.addAll(r.listCells());
150       }
151       return cells;
152     } finally {
153       scanner.close();
154     }
155   }
156 
157   @Test
158   public void testScanAfterDeletingSpecifiedRow() throws IOException {
159     TableName tableName = TableName.valueOf("testScanAfterDeletingSpecifiedRow");
160     HTableDescriptor desc = new HTableDescriptor(tableName)
161             .addFamily(new HColumnDescriptor(FAMILY));
162     TEST_UTIL.getHBaseAdmin().createTable(desc);
163     byte[] row = Bytes.toBytes("SpecifiedRow");
164     byte[] value0 = Bytes.toBytes("value_0");
165     byte[] value1 = Bytes.toBytes("value_1");
166     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
167       Put put = new Put(row);
168       put.addColumn(FAMILY, QUALIFIER, VALUE);
169       t.put(put);
170       Delete d = new Delete(row);
171       t.delete(d);
172       put = new Put(row);
173       put.addColumn(FAMILY, null, value0);
174       t.put(put);
175       put = new Put(row);
176       put.addColumn(FAMILY, null, value1);
177       t.put(put);
178       List<Cell> cells = toList(t.getScanner(new Scan()));
179       assertEquals(1, cells.size());
180       assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
181 
182       cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
183       assertEquals(1, cells.size());
184       assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
185 
186       cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
187       assertEquals(0, cells.size());
188 
189       TEST_UTIL.getHBaseAdmin().flush(tableName);
190       cells = toList(t.getScanner(new Scan()));
191       assertEquals(1, cells.size());
192       assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
193 
194       cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
195       assertEquals(1, cells.size());
196       assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
197 
198       cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
199       assertEquals(0, cells.size());
200     } finally {
201       TEST_UTIL.deleteTable(tableName);
202     }
203   }
204 
205   // override the config settings at the CF level and ensure priority
206   @Test(timeout = 60000)
207   public void testAdvancedConfigOverride() throws Exception {
208     /*
209      * Overall idea: (1) create 3 store files and issue a compaction. config's
210      * compaction.min == 3, so should work. (2) Increase the compaction.min
211      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
212      * of the default config value, adding 3 files and issuing a compaction
213      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
214      * and modify table. The CF schema should override the Table schema and now
215      * cause a minor compaction.
216      */
217     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
218 
219     String tableName = "testAdvancedConfigOverride";
220     TableName TABLE = TableName.valueOf(tableName);
221     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
222     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
223     ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
224 
225     // Create 3 store files.
226     byte[] row = Bytes.toBytes(random.nextInt());
227     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
228 
229     // Verify we have multiple store files.
230     HRegionLocation loc = hTable.getRegionLocation(row, true);
231     byte[] regionName = loc.getRegionInfo().getRegionName();
232     AdminProtos.AdminService.BlockingInterface server =
233       connection.getAdmin(loc.getServerName());
234     assertTrue(ProtobufUtil.getStoreFiles(
235       server, regionName, FAMILY).size() > 1);
236 
237     // Issue a compaction request
238     admin.compact(TABLE.getName());
239 
240     // poll wait for the compactions to happen
241     for (int i = 0; i < 10 * 1000 / 40; ++i) {
242       // The number of store files after compaction should be lesser.
243       loc = hTable.getRegionLocation(row, true);
244       if (!loc.getRegionInfo().isOffline()) {
245         regionName = loc.getRegionInfo().getRegionName();
246         server = connection.getAdmin(loc.getServerName());
247         if (ProtobufUtil.getStoreFiles(
248             server, regionName, FAMILY).size() <= 1) {
249           break;
250         }
251       }
252       Thread.sleep(40);
253     }
254     // verify the compactions took place and that we didn't just time out
255     assertTrue(ProtobufUtil.getStoreFiles(
256       server, regionName, FAMILY).size() <= 1);
257 
258     // change the compaction.min config option for this table to 5
259     LOG.info("hbase.hstore.compaction.min should now be 5");
260     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
261     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
262     admin.modifyTable(TABLE, htd);
263     Pair<Integer, Integer> st;
264     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
265       LOG.debug(st.getFirst() + " regions left to update");
266       Thread.sleep(40);
267     }
268     LOG.info("alter status finished");
269 
270     // Create 3 more store files.
271     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
272 
273     // Issue a compaction request
274     admin.compact(TABLE.getName());
275 
276     // This time, the compaction request should not happen
277     Thread.sleep(10 * 1000);
278     loc = hTable.getRegionLocation(row, true);
279     regionName = loc.getRegionInfo().getRegionName();
280     server = connection.getAdmin(loc.getServerName());
281     int sfCount = ProtobufUtil.getStoreFiles(
282       server, regionName, FAMILY).size();
283     assertTrue(sfCount > 1);
284 
285     // change an individual CF's config option to 2 & online schema update
286     LOG.info("hbase.hstore.compaction.min should now be 2");
287     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
288     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
289     htd.modifyFamily(hcd);
290     admin.modifyTable(TABLE, htd);
291     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
292       LOG.debug(st.getFirst() + " regions left to update");
293       Thread.sleep(40);
294     }
295     LOG.info("alter status finished");
296 
297     // Issue a compaction request
298     admin.compact(TABLE.getName());
299 
300     // poll wait for the compactions to happen
301     for (int i = 0; i < 10 * 1000 / 40; ++i) {
302       loc = hTable.getRegionLocation(row, true);
303       regionName = loc.getRegionInfo().getRegionName();
304       try {
305         server = connection.getAdmin(loc.getServerName());
306         if (ProtobufUtil.getStoreFiles(
307             server, regionName, FAMILY).size() < sfCount) {
308           break;
309         }
310       } catch (Exception e) {
311         LOG.debug("Waiting for region to come online: " + regionName);
312       }
313       Thread.sleep(40);
314     }
315     // verify the compaction took place and that we didn't just time out
316     assertTrue(ProtobufUtil.getStoreFiles(
317       server, regionName, FAMILY).size() < sfCount);
318 
319     // Finally, ensure that we can remove a custom config value after we made it
320     LOG.info("Removing CF config value");
321     LOG.info("hbase.hstore.compaction.min should now be 5");
322     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
323     hcd.setValue("hbase.hstore.compaction.min", null);
324     htd.modifyFamily(hcd);
325     admin.modifyTable(TABLE, htd);
326     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
327       LOG.debug(st.getFirst() + " regions left to update");
328       Thread.sleep(40);
329     }
330     LOG.info("alter status finished");
331     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
332         "hbase.hstore.compaction.min"));
333   }
334 
335   @Test
336   public void testHTableBatchWithEmptyPut() throws Exception {
337     Table table = TEST_UTIL.createTable(
338       Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
339     try {
340       List actions = (List) new ArrayList();
341       Object[] results = new Object[2];
342       // create an empty Put
343       Put put1 = new Put(ROW);
344       actions.add(put1);
345 
346       Put put2 = new Put(ANOTHERROW);
347       put2.add(FAMILY, QUALIFIER, VALUE);
348       actions.add(put2);
349 
350       table.batch(actions, results);
351       fail("Empty Put should have failed the batch call");
352     } catch (IllegalArgumentException iae) {
353 
354     } finally {
355       table.close();
356     }
357   }
358 
359   @Test
360   public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
361 
362     // Test with a single region table.
363 
364     Table table = TEST_UTIL.createTable(
365       Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
366 
367     Put put = new Put(ROW);
368     put.add(FAMILY, QUALIFIER, VALUE);
369 
370     Get get = new Get(ROW);
371 
372     boolean exist = table.exists(get);
373     assertEquals(exist, false);
374 
375     table.put(put);
376 
377     exist = table.exists(get);
378     assertEquals(exist, true);
379   }
380 
381   public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
382 
383     HTable table = TEST_UTIL.createTable(
384       Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
385 
386     Put put = new Put(ROW);
387     put.add(FAMILY, QUALIFIER, VALUE);
388     table.put(put);
389 
390     List<Get> gets = new ArrayList<Get>();
391     gets.add(new Get(ROW));
392     gets.add(null);
393     gets.add(new Get(ANOTHERROW));
394 
395     Boolean[] results = table.exists(gets);
396     assertEquals(results[0], true);
397     assertEquals(results[1], false);
398     assertEquals(results[2], false);
399   }
400 
401   @Test
402   public void testHTableExistsBeforeGet() throws Exception {
403     Table table = TEST_UTIL.createTable(
404       Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
405     try {
406       Put put = new Put(ROW);
407       put.add(FAMILY, QUALIFIER, VALUE);
408       table.put(put);
409 
410       Get get = new Get(ROW);
411 
412       boolean exist = table.exists(get);
413       assertEquals(true, exist);
414 
415       Result result = table.get(get);
416       assertEquals(false, result.isEmpty());
417       assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
418     } finally {
419       table.close();
420     }
421   }
422 
423   @Test
424   public void testHTableExistsAllBeforeGet() throws Exception {
425     final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
426     Table table = TEST_UTIL.createTable(
427       Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
428     try {
429       Put put = new Put(ROW);
430       put.add(FAMILY, QUALIFIER, VALUE);
431       table.put(put);
432       put = new Put(ROW2);
433       put.add(FAMILY, QUALIFIER, VALUE);
434       table.put(put);
435 
436       Get get = new Get(ROW);
437       Get get2 = new Get(ROW2);
438       ArrayList<Get> getList = new ArrayList(2);
439       getList.add(get);
440       getList.add(get2);
441 
442       boolean[] exists = table.existsAll(getList);
443       assertEquals(true, exists[0]);
444       assertEquals(true, exists[1]);
445 
446       Result[] result = table.get(getList);
447       assertEquals(false, result[0].isEmpty());
448       assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
449       assertEquals(false, result[1].isEmpty());
450       assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
451     } finally {
452       table.close();
453     }
454   }
455 
456   @Test
457   public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
458 
459     Table table = TEST_UTIL.createTable(
460       TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
461       1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
462     Put put = new Put(ROW);
463     put.add(FAMILY, QUALIFIER, VALUE);
464 
465     Get get = new Get(ROW);
466 
467     boolean exist = table.exists(get);
468     assertEquals(exist, false);
469 
470     table.put(put);
471 
472     exist = table.exists(get);
473     assertEquals(exist, true);
474   }
475 
476   @Test
477   public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
478     HTable table = TEST_UTIL.createTable(
479       TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
480       new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
481     Put put = new Put(ROW);
482     put.add(FAMILY, QUALIFIER, VALUE);
483     table.put (put);
484 
485     List<Get> gets = new ArrayList<Get>();
486     gets.add(new Get(ANOTHERROW));
487     gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
488     gets.add(new Get(ROW));
489     gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
490 
491     LOG.info("Calling exists");
492     Boolean[] results = table.exists(gets);
493     assertEquals(results[0], false);
494     assertEquals(results[1], false);
495     assertEquals(results[2], true);
496     assertEquals(results[3], false);
497 
498     // Test with the first region.
499     put = new Put(new byte[] { 0x00 });
500     put.add(FAMILY, QUALIFIER, VALUE);
501     table.put(put);
502 
503     gets = new ArrayList<Get>();
504     gets.add(new Get(new byte[] { 0x00 }));
505     gets.add(new Get(new byte[] { 0x00, 0x00 }));
506     results = table.exists(gets);
507     assertEquals(results[0], true);
508     assertEquals(results[1], false);
509 
510     // Test with the last region
511     put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
512     put.add(FAMILY, QUALIFIER, VALUE);
513     table.put(put);
514 
515     gets = new ArrayList<Get>();
516     gets.add(new Get(new byte[] { (byte) 0xff }));
517     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
518     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
519     results = table.exists(gets);
520     assertEquals(results[0], false);
521     assertEquals(results[1], true);
522     assertEquals(results[2], false);
523   }
524 
525   @Test
526   public void testGetEmptyRow() throws Exception {
527     //Create a table and put in 1 row
528     Admin admin = TEST_UTIL.getHBaseAdmin();
529     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
530     desc.addFamily(new HColumnDescriptor(FAMILY));
531     admin.createTable(desc);
532     Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
533 
534     Put put = new Put(ROW_BYTES);
535     put.add(FAMILY, COL_QUAL, VAL_BYTES);
536     table.put(put);
537 
538     //Try getting the row with an empty row key
539     Result res = null;
540     try {
541       res = table.get(new Get(new byte[0]));
542       fail();
543     } catch (IllegalArgumentException e) {
544       // Expected.
545     }
546     assertTrue(res == null);
547     res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
548     assertTrue(res.isEmpty() == true);
549     res = table.get(new Get(ROW_BYTES));
550     assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
551     table.close();
552   }
553 }