1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.List;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.hbase.Cell;
37 import org.apache.hadoop.hbase.CellUtil;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HRegionLocation;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
45 import org.apache.hadoop.hbase.testclassification.LargeTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 @Category(LargeTests.class)
56 public class TestFromClientSide3 {
57 final Log LOG = LogFactory.getLog(getClass());
58 private final static HBaseTestingUtility TEST_UTIL
59 = new HBaseTestingUtility();
60 private static byte[] FAMILY = Bytes.toBytes("testFamily");
61 private static Random random = new Random();
62 private static int SLAVES = 3;
63 private static byte [] ROW = Bytes.toBytes("testRow");
64 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
65 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
66 private static byte [] VALUE = Bytes.toBytes("testValue");
67 private final static byte[] COL_QUAL = Bytes.toBytes("f1");
68 private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
69 private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
70
71
72
73
74 @BeforeClass
75 public static void setUpBeforeClass() throws Exception {
76 TEST_UTIL.getConfiguration().setBoolean(
77 "hbase.online.schema.update.enable", true);
78 TEST_UTIL.startMiniCluster(SLAVES);
79 }
80
81
82
83
84 @AfterClass
85 public static void tearDownAfterClass() throws Exception {
86 TEST_UTIL.shutdownMiniCluster();
87 }
88
89
90
91
92 @Before
93 public void setUp() throws Exception {
94
95 }
96
97
98
99
100 @After
101 public void tearDown() throws Exception {
102 for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
103 LOG.info("Tear down, remove table=" + htd.getTableName());
104 TEST_UTIL.deleteTable(htd.getTableName());
105 }
106 }
107
108 private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
109 throws Exception {
110 Put put = new Put(row);
111 for (int i = 0; i < nPuts; i++) {
112 byte[] qualifier = Bytes.toBytes(random.nextInt());
113 byte[] value = Bytes.toBytes(random.nextInt());
114 put.add(family, qualifier, value);
115 }
116 table.put(put);
117 }
118
119 private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
120 byte[] row, byte[] family, int nFlushes, int nPuts)
121 throws Exception {
122
123
124 HRegionLocation loc = table.getRegionLocation(row, true);
125 AdminProtos.AdminService.BlockingInterface server =
126 admin.getConnection().getAdmin(loc.getServerName());
127 byte[] regName = loc.getRegionInfo().getRegionName();
128
129 for (int i = 0; i < nFlushes; i++) {
130 randomCFPuts(table, row, family, nPuts);
131 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
132 int sfCount = sf.size();
133
134
135 admin.flush(table.getTableName());
136
137
138 while (ProtobufUtil.getStoreFiles(
139 server, regName, FAMILY).size() == sfCount) {
140 Thread.sleep(40);
141 }
142 }
143 }
144
145 private static List<Cell> toList(ResultScanner scanner) {
146 try {
147 List<Cell> cells = new ArrayList<>();
148 for (Result r : scanner) {
149 cells.addAll(r.listCells());
150 }
151 return cells;
152 } finally {
153 scanner.close();
154 }
155 }
156
157 @Test
158 public void testScanAfterDeletingSpecifiedRow() throws IOException {
159 TableName tableName = TableName.valueOf("testScanAfterDeletingSpecifiedRow");
160 HTableDescriptor desc = new HTableDescriptor(tableName)
161 .addFamily(new HColumnDescriptor(FAMILY));
162 TEST_UTIL.getHBaseAdmin().createTable(desc);
163 byte[] row = Bytes.toBytes("SpecifiedRow");
164 byte[] value0 = Bytes.toBytes("value_0");
165 byte[] value1 = Bytes.toBytes("value_1");
166 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
167 Put put = new Put(row);
168 put.addColumn(FAMILY, QUALIFIER, VALUE);
169 t.put(put);
170 Delete d = new Delete(row);
171 t.delete(d);
172 put = new Put(row);
173 put.addColumn(FAMILY, null, value0);
174 t.put(put);
175 put = new Put(row);
176 put.addColumn(FAMILY, null, value1);
177 t.put(put);
178 List<Cell> cells = toList(t.getScanner(new Scan()));
179 assertEquals(1, cells.size());
180 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
181
182 cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
183 assertEquals(1, cells.size());
184 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
185
186 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
187 assertEquals(0, cells.size());
188
189 TEST_UTIL.getHBaseAdmin().flush(tableName);
190 cells = toList(t.getScanner(new Scan()));
191 assertEquals(1, cells.size());
192 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
193
194 cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
195 assertEquals(1, cells.size());
196 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
197
198 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
199 assertEquals(0, cells.size());
200 } finally {
201 TEST_UTIL.deleteTable(tableName);
202 }
203 }
204
205
206 @Test(timeout = 60000)
207 public void testAdvancedConfigOverride() throws Exception {
208
209
210
211
212
213
214
215
216
217 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
218
219 String tableName = "testAdvancedConfigOverride";
220 TableName TABLE = TableName.valueOf(tableName);
221 HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
222 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
223 ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
224
225
226 byte[] row = Bytes.toBytes(random.nextInt());
227 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
228
229
230 HRegionLocation loc = hTable.getRegionLocation(row, true);
231 byte[] regionName = loc.getRegionInfo().getRegionName();
232 AdminProtos.AdminService.BlockingInterface server =
233 connection.getAdmin(loc.getServerName());
234 assertTrue(ProtobufUtil.getStoreFiles(
235 server, regionName, FAMILY).size() > 1);
236
237
238 admin.compact(TABLE.getName());
239
240
241 for (int i = 0; i < 10 * 1000 / 40; ++i) {
242
243 loc = hTable.getRegionLocation(row, true);
244 if (!loc.getRegionInfo().isOffline()) {
245 regionName = loc.getRegionInfo().getRegionName();
246 server = connection.getAdmin(loc.getServerName());
247 if (ProtobufUtil.getStoreFiles(
248 server, regionName, FAMILY).size() <= 1) {
249 break;
250 }
251 }
252 Thread.sleep(40);
253 }
254
255 assertTrue(ProtobufUtil.getStoreFiles(
256 server, regionName, FAMILY).size() <= 1);
257
258
259 LOG.info("hbase.hstore.compaction.min should now be 5");
260 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
261 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
262 admin.modifyTable(TABLE, htd);
263 Pair<Integer, Integer> st;
264 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
265 LOG.debug(st.getFirst() + " regions left to update");
266 Thread.sleep(40);
267 }
268 LOG.info("alter status finished");
269
270
271 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
272
273
274 admin.compact(TABLE.getName());
275
276
277 Thread.sleep(10 * 1000);
278 loc = hTable.getRegionLocation(row, true);
279 regionName = loc.getRegionInfo().getRegionName();
280 server = connection.getAdmin(loc.getServerName());
281 int sfCount = ProtobufUtil.getStoreFiles(
282 server, regionName, FAMILY).size();
283 assertTrue(sfCount > 1);
284
285
286 LOG.info("hbase.hstore.compaction.min should now be 2");
287 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
288 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
289 htd.modifyFamily(hcd);
290 admin.modifyTable(TABLE, htd);
291 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
292 LOG.debug(st.getFirst() + " regions left to update");
293 Thread.sleep(40);
294 }
295 LOG.info("alter status finished");
296
297
298 admin.compact(TABLE.getName());
299
300
301 for (int i = 0; i < 10 * 1000 / 40; ++i) {
302 loc = hTable.getRegionLocation(row, true);
303 regionName = loc.getRegionInfo().getRegionName();
304 try {
305 server = connection.getAdmin(loc.getServerName());
306 if (ProtobufUtil.getStoreFiles(
307 server, regionName, FAMILY).size() < sfCount) {
308 break;
309 }
310 } catch (Exception e) {
311 LOG.debug("Waiting for region to come online: " + regionName);
312 }
313 Thread.sleep(40);
314 }
315
316 assertTrue(ProtobufUtil.getStoreFiles(
317 server, regionName, FAMILY).size() < sfCount);
318
319
320 LOG.info("Removing CF config value");
321 LOG.info("hbase.hstore.compaction.min should now be 5");
322 hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
323 hcd.setValue("hbase.hstore.compaction.min", null);
324 htd.modifyFamily(hcd);
325 admin.modifyTable(TABLE, htd);
326 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
327 LOG.debug(st.getFirst() + " regions left to update");
328 Thread.sleep(40);
329 }
330 LOG.info("alter status finished");
331 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
332 "hbase.hstore.compaction.min"));
333 }
334
335 @Test
336 public void testHTableBatchWithEmptyPut() throws Exception {
337 Table table = TEST_UTIL.createTable(
338 Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
339 try {
340 List actions = (List) new ArrayList();
341 Object[] results = new Object[2];
342
343 Put put1 = new Put(ROW);
344 actions.add(put1);
345
346 Put put2 = new Put(ANOTHERROW);
347 put2.add(FAMILY, QUALIFIER, VALUE);
348 actions.add(put2);
349
350 table.batch(actions, results);
351 fail("Empty Put should have failed the batch call");
352 } catch (IllegalArgumentException iae) {
353
354 } finally {
355 table.close();
356 }
357 }
358
359 @Test
360 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
361
362
363
364 Table table = TEST_UTIL.createTable(
365 Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
366
367 Put put = new Put(ROW);
368 put.add(FAMILY, QUALIFIER, VALUE);
369
370 Get get = new Get(ROW);
371
372 boolean exist = table.exists(get);
373 assertEquals(exist, false);
374
375 table.put(put);
376
377 exist = table.exists(get);
378 assertEquals(exist, true);
379 }
380
381 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
382
383 HTable table = TEST_UTIL.createTable(
384 Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
385
386 Put put = new Put(ROW);
387 put.add(FAMILY, QUALIFIER, VALUE);
388 table.put(put);
389
390 List<Get> gets = new ArrayList<Get>();
391 gets.add(new Get(ROW));
392 gets.add(null);
393 gets.add(new Get(ANOTHERROW));
394
395 Boolean[] results = table.exists(gets);
396 assertEquals(results[0], true);
397 assertEquals(results[1], false);
398 assertEquals(results[2], false);
399 }
400
401 @Test
402 public void testHTableExistsBeforeGet() throws Exception {
403 Table table = TEST_UTIL.createTable(
404 Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
405 try {
406 Put put = new Put(ROW);
407 put.add(FAMILY, QUALIFIER, VALUE);
408 table.put(put);
409
410 Get get = new Get(ROW);
411
412 boolean exist = table.exists(get);
413 assertEquals(true, exist);
414
415 Result result = table.get(get);
416 assertEquals(false, result.isEmpty());
417 assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
418 } finally {
419 table.close();
420 }
421 }
422
423 @Test
424 public void testHTableExistsAllBeforeGet() throws Exception {
425 final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
426 Table table = TEST_UTIL.createTable(
427 Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
428 try {
429 Put put = new Put(ROW);
430 put.add(FAMILY, QUALIFIER, VALUE);
431 table.put(put);
432 put = new Put(ROW2);
433 put.add(FAMILY, QUALIFIER, VALUE);
434 table.put(put);
435
436 Get get = new Get(ROW);
437 Get get2 = new Get(ROW2);
438 ArrayList<Get> getList = new ArrayList(2);
439 getList.add(get);
440 getList.add(get2);
441
442 boolean[] exists = table.existsAll(getList);
443 assertEquals(true, exists[0]);
444 assertEquals(true, exists[1]);
445
446 Result[] result = table.get(getList);
447 assertEquals(false, result[0].isEmpty());
448 assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
449 assertEquals(false, result[1].isEmpty());
450 assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
451 } finally {
452 table.close();
453 }
454 }
455
456 @Test
457 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
458
459 Table table = TEST_UTIL.createTable(
460 TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
461 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
462 Put put = new Put(ROW);
463 put.add(FAMILY, QUALIFIER, VALUE);
464
465 Get get = new Get(ROW);
466
467 boolean exist = table.exists(get);
468 assertEquals(exist, false);
469
470 table.put(put);
471
472 exist = table.exists(get);
473 assertEquals(exist, true);
474 }
475
476 @Test
477 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
478 HTable table = TEST_UTIL.createTable(
479 TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
480 new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
481 Put put = new Put(ROW);
482 put.add(FAMILY, QUALIFIER, VALUE);
483 table.put (put);
484
485 List<Get> gets = new ArrayList<Get>();
486 gets.add(new Get(ANOTHERROW));
487 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
488 gets.add(new Get(ROW));
489 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
490
491 LOG.info("Calling exists");
492 Boolean[] results = table.exists(gets);
493 assertEquals(results[0], false);
494 assertEquals(results[1], false);
495 assertEquals(results[2], true);
496 assertEquals(results[3], false);
497
498
499 put = new Put(new byte[] { 0x00 });
500 put.add(FAMILY, QUALIFIER, VALUE);
501 table.put(put);
502
503 gets = new ArrayList<Get>();
504 gets.add(new Get(new byte[] { 0x00 }));
505 gets.add(new Get(new byte[] { 0x00, 0x00 }));
506 results = table.exists(gets);
507 assertEquals(results[0], true);
508 assertEquals(results[1], false);
509
510
511 put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
512 put.add(FAMILY, QUALIFIER, VALUE);
513 table.put(put);
514
515 gets = new ArrayList<Get>();
516 gets.add(new Get(new byte[] { (byte) 0xff }));
517 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
518 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
519 results = table.exists(gets);
520 assertEquals(results[0], false);
521 assertEquals(results[1], true);
522 assertEquals(results[2], false);
523 }
524
525 @Test
526 public void testGetEmptyRow() throws Exception {
527
528 Admin admin = TEST_UTIL.getHBaseAdmin();
529 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
530 desc.addFamily(new HColumnDescriptor(FAMILY));
531 admin.createTable(desc);
532 Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
533
534 Put put = new Put(ROW_BYTES);
535 put.add(FAMILY, COL_QUAL, VAL_BYTES);
536 table.put(put);
537
538
539 Result res = null;
540 try {
541 res = table.get(new Get(new byte[0]));
542 fail();
543 } catch (IllegalArgumentException e) {
544
545 }
546 assertTrue(res == null);
547 res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
548 assertTrue(res.isEmpty() == true);
549 res = table.get(new Get(ROW_BYTES));
550 assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
551 table.close();
552 }
553 }