1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Mockito.doReturn;
25 import static org.mockito.Mockito.when;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.CoordinatedStateManager;
36 import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.client.Durability;
45 import org.apache.hadoop.hbase.client.Put;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.testclassification.SmallTests;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.FSUtils;
50 import org.apache.hadoop.hbase.wal.WALFactory;
51 import org.apache.zookeeper.KeeperException;
52 import org.junit.After;
53 import org.junit.Before;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56 import org.mockito.Mockito;
57
58 import com.google.common.collect.ImmutableList;
59
60
61
62
63
64 @Category(SmallTests.class)
65 public class TestRegionMergeTransaction {
66 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass()
68 .getName());
69 private HRegion region_a;
70 private HRegion region_b;
71 private HRegion region_c;
72 private WALFactory wals;
73 private FileSystem fs;
74
75 private static final byte[] STARTROW_A = new byte[] { 'a', 'a', 'a' };
76 private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' };
77 private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' };
78 private static final byte[] ENDROW = new byte[] { '{', '{', '{' };
79 private static final byte[] CF = HConstants.CATALOG_FAMILY;
80
81 @Before
82 public void setup() throws IOException {
83 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
84 this.fs.delete(this.testdir, true);
85 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
86 FSUtils.setRootDir(walConf, this.testdir);
87 this.wals = new WALFactory(walConf, null, TestRegionMergeTransaction.class.getName());
88 this.region_a = createRegion(this.testdir, this.wals, STARTROW_A, STARTROW_B);
89 this.region_b = createRegion(this.testdir, this.wals, STARTROW_B, STARTROW_C);
90 this.region_c = createRegion(this.testdir, this.wals, STARTROW_C, ENDROW);
91 assert region_a != null && region_b != null && region_c != null;
92 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
93 }
94
95 @After
96 public void teardown() throws IOException {
97 for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
98 if (region != null && !region.isClosed()) region.close();
99 if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
100 && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
101 throw new IOException("Failed deleting of "
102 + region.getRegionFileSystem().getRegionDir());
103 }
104 }
105 if (this.wals != null) {
106 this.wals.close();
107 }
108 this.fs.delete(this.testdir, true);
109 }
110
111
112
113
114
115
116 @Test
117 public void testPrepare() throws IOException {
118 prepareOnGoodRegions();
119 }
120
121 private RegionMergeTransactionImpl prepareOnGoodRegions() throws IOException {
122 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(region_a, region_b,
123 false);
124 RegionMergeTransactionImpl spyMT = Mockito.spy(mt);
125 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
126 region_a.getRegionInfo().getRegionName());
127 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
128 region_b.getRegionInfo().getRegionName());
129 assertTrue(spyMT.prepare(null));
130 return spyMT;
131 }
132
133
134
135
136 @Test
137 public void testPrepareWithSameRegion() throws IOException {
138 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(this.region_a,
139 this.region_a, true);
140 assertFalse("should not merge the same region even if it is forcible ",
141 mt.prepare(null));
142 }
143
144
145
146
147 @Test
148 public void testPrepareWithRegionsNotAdjacent() throws IOException {
149 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(this.region_a,
150 this.region_c, false);
151 assertFalse("should not merge two regions if they are adjacent except it is forcible",
152 mt.prepare(null));
153 }
154
155
156
157
158 @Test
159 public void testPrepareWithRegionsNotAdjacentUnderCompulsory()
160 throws IOException {
161 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(region_a, region_c,
162 true);
163 RegionMergeTransactionImpl spyMT = Mockito.spy(mt);
164 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
165 region_a.getRegionInfo().getRegionName());
166 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
167 region_c.getRegionInfo().getRegionName());
168 assertTrue("Since focible is true, should merge two regions even if they are not adjacent",
169 spyMT.prepare(null));
170 }
171
172
173
174
175 @Test
176 public void testPrepareWithRegionsWithReference() throws IOException {
177 HStore storeMock = Mockito.mock(HStore.class);
178 when(storeMock.hasReferences()).thenReturn(true);
179 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
180 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
181 this.region_a.stores.put(Bytes.toBytes(""), storeMock);
182 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(this.region_a,
183 this.region_b, false);
184 assertFalse(
185 "a region should not be mergeable if it has instances of store file references",
186 mt.prepare(null));
187 }
188
189 @Test
190 public void testPrepareWithClosedRegion() throws IOException {
191 this.region_a.close();
192 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(this.region_a,
193 this.region_b, false);
194 assertFalse(mt.prepare(null));
195 }
196
197
198
199
200
201 @Test
202 public void testPrepareWithRegionsWithMergeReference() throws IOException {
203 RegionMergeTransactionImpl mt = new RegionMergeTransactionImpl(region_a, region_b,
204 false);
205 RegionMergeTransactionImpl spyMT = Mockito.spy(mt);
206 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
207 region_a.getRegionInfo().getRegionName());
208 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
209 region_b.getRegionInfo().getRegionName());
210 assertFalse(spyMT.prepare(null));
211 }
212
213 @Test
214 public void testWholesomeMerge() throws IOException, InterruptedException {
215 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
216 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
217 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
218 assertEquals(rowCountOfRegionA, countRows(this.region_a));
219 assertEquals(rowCountOfRegionB, countRows(this.region_b));
220
221
222 RegionMergeTransactionImpl mt = prepareOnGoodRegions();
223
224
225 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
226 CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
227 TEST_UTIL.getConfiguration());
228 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration(), cp);
229 HRegion mergedRegion = (HRegion)mt.execute(mockServer, null);
230
231 assertTrue(this.fs.exists(mt.getMergesDir()));
232
233 assertTrue(region_a.isClosed());
234 assertTrue(region_b.isClosed());
235
236
237
238 assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
239
240 assertTrue(Bytes.equals(this.region_a.getRegionInfo().getStartKey(),
241 mergedRegion.getRegionInfo().getStartKey()));
242 assertTrue(Bytes.equals(this.region_b.getRegionInfo().getEndKey(),
243 mergedRegion.getRegionInfo().getEndKey()));
244
245 try {
246 int mergedRegionRowCount = countRows(mergedRegion);
247 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
248 mergedRegionRowCount);
249 } finally {
250 HRegion.closeHRegion(mergedRegion);
251 }
252
253 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
254 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
255 }
256
257 @Test
258 public void testRollback() throws IOException, InterruptedException {
259 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
260 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
261 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
262 assertEquals(rowCountOfRegionA, countRows(this.region_a));
263 assertEquals(rowCountOfRegionB, countRows(this.region_b));
264
265
266 RegionMergeTransactionImpl mt = prepareOnGoodRegions();
267
268 when(mt.createMergedRegionFromMerges(region_a, region_b,
269 mt.getMergedRegionInfo())).thenThrow(
270 new MockedFailedMergedRegionCreation());
271
272
273 boolean expectedException = false;
274 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
275 CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
276 TEST_UTIL.getConfiguration());
277 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration(), cp);
278 try {
279 mt.execute(mockServer, null);
280 } catch (MockedFailedMergedRegionCreation e) {
281 expectedException = true;
282 }
283 assertTrue(expectedException);
284
285 assertTrue(mt.rollback(null, null));
286
287
288 int rowCountOfRegionA2 = countRows(this.region_a);
289 assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
290 int rowCountOfRegionB2 = countRows(this.region_b);
291 assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
292
293
294 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir,
295 mt.getMergedRegionInfo())));
296
297 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
298 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
299
300
301 assertTrue(mt.prepare(null));
302 HRegion mergedRegion = (HRegion)mt.execute(mockServer, null);
303
304
305 try {
306 int mergedRegionRowCount = countRows(mergedRegion);
307 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
308 mergedRegionRowCount);
309 } finally {
310 HRegion.closeHRegion(mergedRegion);
311 }
312
313 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
314 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
315 }
316
317 @Test
318 public void testFailAfterPONR() throws IOException, KeeperException, InterruptedException {
319 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
320 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
321 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
322 assertEquals(rowCountOfRegionA, countRows(this.region_a));
323 assertEquals(rowCountOfRegionB, countRows(this.region_b));
324
325
326 RegionMergeTransactionImpl mt = prepareOnGoodRegions();
327 Mockito.doThrow(new MockedFailedMergedRegionOpen())
328 .when(mt)
329 .openMergedRegion((Server) Mockito.anyObject(),
330 (RegionServerServices) Mockito.anyObject(),
331 (HRegion) Mockito.anyObject());
332
333
334 boolean expectedException = false;
335 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
336 CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
337 TEST_UTIL.getConfiguration());
338 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration(), cp);
339 try {
340 mt.execute(mockServer, null);
341 } catch (MockedFailedMergedRegionOpen e) {
342 expectedException = true;
343 }
344 assertTrue(expectedException);
345
346 assertFalse(mt.rollback(null, null));
347
348
349
350 Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
351 .getParent();
352 Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
353 .getEncodedName());
354 assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
355 }
356
357 @Test
358 public void testMeregedRegionBoundary() {
359 TableName tableName =
360 TableName.valueOf("testMeregedRegionBoundary");
361 byte[] a = Bytes.toBytes("a");
362 byte[] b = Bytes.toBytes("b");
363 byte[] z = Bytes.toBytes("z");
364 HRegionInfo r1 = new HRegionInfo(tableName);
365 HRegionInfo r2 = new HRegionInfo(tableName, a, z);
366 HRegionInfo m = RegionMergeTransactionImpl.getMergedRegionInfo(r1, r2);
367 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
368 && Bytes.equals(m.getEndKey(), r1.getEndKey()));
369
370 r1 = new HRegionInfo(tableName, null, a);
371 r2 = new HRegionInfo(tableName, a, z);
372 m = RegionMergeTransactionImpl.getMergedRegionInfo(r1, r2);
373 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
374 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
375
376 r1 = new HRegionInfo(tableName, null, a);
377 r2 = new HRegionInfo(tableName, z, null);
378 m = RegionMergeTransactionImpl.getMergedRegionInfo(r1, r2);
379 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
380 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
381
382 r1 = new HRegionInfo(tableName, a, z);
383 r2 = new HRegionInfo(tableName, z, null);
384 m = RegionMergeTransactionImpl.getMergedRegionInfo(r1, r2);
385 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
386 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
387
388 r1 = new HRegionInfo(tableName, a, b);
389 r2 = new HRegionInfo(tableName, b, z);
390 m = RegionMergeTransactionImpl.getMergedRegionInfo(r1, r2);
391 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
392 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
393 }
394
395
396
397
398 @SuppressWarnings("serial")
399 private class MockedFailedMergedRegionCreation extends IOException {
400 }
401
402 @SuppressWarnings("serial")
403 private class MockedFailedMergedRegionOpen extends IOException {
404 }
405
406 private HRegion createRegion(final Path testdir, final WALFactory wals,
407 final byte[] startrow, final byte[] endrow)
408 throws IOException {
409
410 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
411 HColumnDescriptor hcd = new HColumnDescriptor(CF);
412 htd.addFamily(hcd);
413 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
414 HRegion a = HRegion.createHRegion(hri, testdir,
415 TEST_UTIL.getConfiguration(), htd);
416 HRegion.closeHRegion(a);
417 return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()),
418 TEST_UTIL.getConfiguration());
419 }
420
421 private int countRows(final HRegion r) throws IOException {
422 int rowcount = 0;
423 InternalScanner scanner = r.getScanner(new Scan());
424 try {
425 List<Cell> kvs = new ArrayList<Cell>();
426 boolean hasNext = true;
427 while (hasNext) {
428 hasNext = scanner.next(kvs);
429 if (!kvs.isEmpty())
430 rowcount++;
431 }
432 } finally {
433 scanner.close();
434 }
435 return rowcount;
436 }
437
438
439
440
441
442
443
444
445
446
447 private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
448 throws IOException {
449 byte[] k = new byte[3];
450 int rowCount = 0;
451 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
452 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
453 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
454 k[0] = b1;
455 k[1] = b2;
456 k[2] = b3;
457 if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
458 continue;
459 }
460 Put put = new Put(k);
461 put.add(f, null, k);
462 if (r.getWAL() == null)
463 put.setDurability(Durability.SKIP_WAL);
464 r.put(put);
465 rowCount++;
466 }
467 }
468 if (flush) {
469 r.flush(true);
470 }
471 }
472 return rowCount;
473 }
474
475 }