1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.commons.lang.math.RandomUtils;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.testclassification.LargeTests;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.UnknownRegionException;
44 import org.apache.hadoop.hbase.MetaTableAccessor;
45 import org.apache.hadoop.hbase.client.Admin;
46 import org.apache.hadoop.hbase.client.Put;
47 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
48 import org.apache.hadoop.hbase.client.Result;
49 import org.apache.hadoop.hbase.client.ResultScanner;
50 import org.apache.hadoop.hbase.client.Scan;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
53 import org.apache.hadoop.hbase.master.AssignmentManager;
54 import org.apache.hadoop.hbase.master.HMaster;
55 import org.apache.hadoop.hbase.master.RegionState.State;
56 import org.apache.hadoop.hbase.master.RegionStates;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
59 import org.apache.hadoop.hbase.util.FSUtils;
60 import org.apache.hadoop.hbase.util.Pair;
61 import org.apache.hadoop.hbase.util.PairOfSameType;
62 import org.apache.hadoop.util.StringUtils;
63 import org.junit.AfterClass;
64 import org.junit.BeforeClass;
65 import org.junit.Test;
66 import org.junit.experimental.categories.Category;
67
68 import com.google.common.base.Joiner;
69
70
71
72
73
74
75
76 @Category(LargeTests.class)
77 public class TestRegionMergeTransactionOnCluster {
78 private static final Log LOG = LogFactory
79 .getLog(TestRegionMergeTransactionOnCluster.class);
80 private static final int NB_SERVERS = 3;
81
82 private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
83 private static final byte[] QUALIFIER = Bytes.toBytes("q");
84
85 private static byte[] ROW = Bytes.toBytes("testRow");
86 private static final int INITIAL_REGION_NUM = 10;
87 private static final int ROWSIZE = 200;
88 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
89
90 private static int waitTime = 60 * 1000;
91
92 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
93
94 private static HMaster master;
95 private static Admin admin;
96
97 static void setupOnce() throws Exception {
98
99 TEST_UTIL.startMiniCluster(NB_SERVERS);
100 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
101 master = cluster.getMaster();
102 master.balanceSwitch(false);
103 admin = TEST_UTIL.getHBaseAdmin();
104 }
105
106 @BeforeClass
107 public static void beforeAllTests() throws Exception {
108
109 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
110 setupOnce();
111 }
112
113 @AfterClass
114 public static void afterAllTests() throws Exception {
115 TEST_UTIL.shutdownMiniCluster();
116 }
117
118 @Test
119 public void testWholesomeMerge() throws Exception {
120 LOG.info("Starting testWholesomeMerge");
121 final TableName tableName =
122 TableName.valueOf("testWholesomeMerge");
123
124
125 Table table = createTableAndLoadData(master, tableName);
126
127 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
128 INITIAL_REGION_NUM - 1);
129
130
131 PairOfSameType<HRegionInfo> mergedRegions =
132 mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2,
133 INITIAL_REGION_NUM - 2);
134
135 verifyRowCount(table, ROWSIZE);
136
137
138 HRegionInfo hri = RandomUtils.nextBoolean() ?
139 mergedRegions.getFirst() : mergedRegions.getSecond();
140 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
141 AssignmentManager am = cluster.getMaster().getAssignmentManager();
142 RegionStates regionStates = am.getRegionStates();
143 long start = EnvironmentEdgeManager.currentTime();
144 while (!regionStates.isRegionInState(hri, State.MERGED)) {
145 assertFalse("Timed out in waiting one merged region to be in state MERGED",
146 EnvironmentEdgeManager.currentTime() - start > 60000);
147 Thread.sleep(500);
148 }
149
150
151 am.assign(hri, true, true);
152 assertFalse("Merged region can't be assigned",
153 regionStates.isRegionInTransition(hri));
154 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
155
156
157 am.unassign(hri, true, null);
158 assertFalse("Merged region can't be unassigned",
159 regionStates.isRegionInTransition(hri));
160 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
161
162 table.close();
163 }
164
165 @Test
166 public void testCleanMergeReference() throws Exception {
167 LOG.info("Starting testCleanMergeReference");
168 admin.enableCatalogJanitor(false);
169 try {
170 final TableName tableName =
171 TableName.valueOf("testCleanMergeReference");
172
173 Table table = createTableAndLoadData(master, tableName);
174
175 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
176 INITIAL_REGION_NUM - 1);
177 verifyRowCount(table, ROWSIZE);
178 table.close();
179
180 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
181 .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tableName);
182 HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
183 HTableDescriptor tableDescritor = master.getTableDescriptors().get(
184 tableName);
185 Result mergedRegionResult = MetaTableAccessor.getRegionResult(
186 master.getConnection(), mergedRegionInfo.getRegionName());
187
188
189 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
190 HConstants.MERGEA_QUALIFIER) != null);
191 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
192 HConstants.MERGEB_QUALIFIER) != null);
193
194
195 HRegionInfo regionA = HRegionInfo.getHRegionInfo(mergedRegionResult,
196 HConstants.MERGEA_QUALIFIER);
197 HRegionInfo regionB = HRegionInfo.getHRegionInfo(mergedRegionResult,
198 HConstants.MERGEB_QUALIFIER);
199 FileSystem fs = master.getMasterFileSystem().getFileSystem();
200 Path rootDir = master.getMasterFileSystem().getRootDir();
201
202 Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
203 Path regionAdir = new Path(tabledir, regionA.getEncodedName());
204 Path regionBdir = new Path(tabledir, regionB.getEncodedName());
205 assertTrue(fs.exists(regionAdir));
206 assertTrue(fs.exists(regionBdir));
207
208 admin.compactRegion(mergedRegionInfo.getRegionName());
209
210 long timeout = System.currentTimeMillis() + waitTime;
211 HRegionFileSystem hrfs = new HRegionFileSystem(
212 TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
213 while (System.currentTimeMillis() < timeout) {
214 if (!hrfs.hasReferences(tableDescritor)) {
215 break;
216 }
217 Thread.sleep(50);
218 }
219 assertFalse(hrfs.hasReferences(tableDescritor));
220
221
222
223 int cleaned = admin.runCatalogScan();
224 assertTrue(cleaned > 0);
225 assertFalse(fs.exists(regionAdir));
226 assertFalse(fs.exists(regionBdir));
227
228 mergedRegionResult = MetaTableAccessor.getRegionResult(
229 master.getConnection(), mergedRegionInfo.getRegionName());
230 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
231 HConstants.MERGEA_QUALIFIER) != null);
232 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
233 HConstants.MERGEB_QUALIFIER) != null);
234
235 } finally {
236 admin.enableCatalogJanitor(true);
237 }
238 }
239
240
241
242
243
244
245
246 @Test
247 public void testMerge() throws Exception {
248 LOG.info("Starting testMerge");
249 final TableName tableName = TableName.valueOf("testMerge");
250
251 try {
252
253 Table table = createTableAndLoadData(master, tableName);
254 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
255 List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
256
257 HRegionInfo a = regions.get(0);
258 HRegionInfo b = regions.get(1);
259 regionStates.regionOffline(a);
260 try {
261
262 admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
263 fail("Offline regions should not be able to merge");
264 } catch (IOException ie) {
265 System.out.println(ie);
266 assertTrue("Exception should mention regions not online",
267 StringUtils.stringifyException(ie).contains("regions not online")
268 && ie instanceof MergeRegionException);
269 }
270 try {
271
272 admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
273 fail("A region should not be able to merge with itself, even forcifully");
274 } catch (IOException ie) {
275 assertTrue("Exception should mention regions not online",
276 StringUtils.stringifyException(ie).contains("region to itself")
277 && ie instanceof MergeRegionException);
278 }
279 try {
280
281 admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
282 fail("Unknown region could not be merged");
283 } catch (IOException ie) {
284 assertTrue("UnknownRegionException should be thrown",
285 ie instanceof UnknownRegionException);
286 }
287 table.close();
288 } finally {
289 TEST_UTIL.deleteTable(tableName);
290 }
291 }
292
293 @Test
294 public void testMergeWithReplicas() throws Exception {
295 final TableName tableName = TableName.valueOf("testMergeWithReplicas");
296
297 createTableAndLoadData(master, tableName, 5, 2);
298 List<Pair<HRegionInfo, ServerName>> initialRegionToServers =
299 MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
300 tableName);
301
302 PairOfSameType<HRegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(master, tableName,
303 0, 2, 5 * 2 - 2);
304 List<Pair<HRegionInfo, ServerName>> currentRegionToServers =
305 MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
306 tableName);
307 List<HRegionInfo> initialRegions = new ArrayList<HRegionInfo>();
308 for (Pair<HRegionInfo, ServerName> p : initialRegionToServers) {
309 initialRegions.add(p.getFirst());
310 }
311 List<HRegionInfo> currentRegions = new ArrayList<HRegionInfo>();
312 for (Pair<HRegionInfo, ServerName> p : currentRegionToServers) {
313 currentRegions.add(p.getFirst());
314 }
315 assertTrue(initialRegions.contains(mergedRegions.getFirst()));
316 assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
317 mergedRegions.getFirst(), 1)));
318 assertTrue(initialRegions.contains(mergedRegions.getSecond()));
319 assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
320 mergedRegions.getSecond(), 1)));
321 assertTrue(!initialRegions.contains(currentRegions.get(0)));
322 assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
323 currentRegions.get(0), 1)));
324 assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
325 currentRegions.get(0), 1)));
326 assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
327 mergedRegions.getFirst(), 1)));
328 assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
329 mergedRegions.getSecond(), 1)));
330 }
331
332 private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum(
333 HMaster master, TableName tablename,
334 int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
335 PairOfSameType<HRegionInfo> mergedRegions =
336 requestMergeRegion(master, tablename, regionAnum, regionBnum);
337 waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
338 return mergedRegions;
339 }
340
341 private PairOfSameType<HRegionInfo> requestMergeRegion(
342 HMaster master, TableName tablename,
343 int regionAnum, int regionBnum) throws Exception {
344 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
345 .getTableRegionsAndLocations(master.getZooKeeper(),
346 master.getConnection(), tablename);
347 HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
348 HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
349 TEST_UTIL.getHBaseAdmin().mergeRegions(
350 regionA.getEncodedNameAsBytes(),
351 regionB.getEncodedNameAsBytes(), false);
352 return new PairOfSameType<HRegionInfo>(regionA, regionB);
353 }
354
355 private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
356 int expectedRegionNum) throws Exception {
357 List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
358 List<HRegionInfo> tableRegionsInMaster;
359 long timeout = System.currentTimeMillis() + waitTime;
360 while (System.currentTimeMillis() < timeout) {
361 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
362 master.getConnection(), tablename);
363 tableRegionsInMaster = master.getAssignmentManager().getRegionStates()
364 .getRegionsOfTable(tablename);
365 if (tableRegionsInMeta.size() == expectedRegionNum
366 && tableRegionsInMaster.size() == expectedRegionNum) {
367 break;
368 }
369 Thread.sleep(250);
370 }
371
372 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
373 master.getConnection(), tablename);
374 LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
375 assertEquals(expectedRegionNum, tableRegionsInMeta.size());
376 }
377
378 private Table createTableAndLoadData(HMaster master, TableName tablename)
379 throws Exception {
380 return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
381 }
382
383 private Table createTableAndLoadData(HMaster master, TableName tablename,
384 int numRegions, int replication) throws Exception {
385 assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
386 byte[][] splitRows = new byte[numRegions - 1][];
387 for (int i = 0; i < splitRows.length; i++) {
388 splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
389 }
390
391 Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
392 if (replication > 1) {
393 HBaseTestingUtility.setReplicas(admin, tablename, replication);
394 }
395 loadData(table);
396 verifyRowCount(table, ROWSIZE);
397
398
399 long timeout = System.currentTimeMillis() + waitTime;
400 List<Pair<HRegionInfo, ServerName>> tableRegions;
401 while (System.currentTimeMillis() < timeout) {
402 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
403 master.getConnection(), tablename);
404 if (tableRegions.size() == numRegions * replication)
405 break;
406 Thread.sleep(250);
407 }
408
409 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
410 master.getZooKeeper(),
411 master.getConnection(), tablename);
412 LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
413 assertEquals(numRegions * replication, tableRegions.size());
414 return table;
415 }
416
417 private static byte[][] makeN(byte[] base, int n) {
418 byte[][] ret = new byte[n][];
419 for (int i = 0; i < n; i++) {
420 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
421 }
422 return ret;
423 }
424
425 private void loadData(Table table) throws IOException {
426 for (int i = 0; i < ROWSIZE; i++) {
427 Put put = new Put(ROWS[i]);
428 put.add(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
429 table.put(put);
430 }
431 }
432
433 private void verifyRowCount(Table table, int expectedRegionNum)
434 throws IOException {
435 ResultScanner scanner = table.getScanner(new Scan());
436 int rowCount = 0;
437 while (scanner.next() != null) {
438 rowCount++;
439 }
440 assertEquals(expectedRegionNum, rowCount);
441 scanner.close();
442 }
443 }