1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.net.InetSocketAddress;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Random;
35 import java.util.concurrent.atomic.AtomicInteger;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.client.Admin;
51 import org.apache.hadoop.hbase.client.Connection;
52 import org.apache.hadoop.hbase.client.HTable;
53 import org.apache.hadoop.hbase.client.MetaScanner;
54 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
55 import org.apache.hadoop.hbase.client.Result;
56 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
57 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
58 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
59 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
60 import org.apache.hadoop.hbase.regionserver.HRegionServer;
61 import org.apache.hadoop.hbase.regionserver.Region;
62 import org.apache.hadoop.hbase.util.Bytes;
63 import org.apache.hadoop.hbase.util.Pair;
64 import org.apache.zookeeper.KeeperException;
65 import org.junit.AfterClass;
66 import org.junit.BeforeClass;
67 import org.junit.Test;
68 import org.junit.experimental.categories.Category;
69
70
71 @Category(MediumTests.class)
72 public class TestRegionPlacement {
73 final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
74 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
75 private final static int SLAVES = 10;
76 private static Connection CONNECTION;
77 private static Admin admin;
78 private static RegionPlacementMaintainer rp;
79 private static Position[] positions = Position.values();
80 private int lastRegionOnPrimaryRSCount = 0;
81 private int REGION_NUM = 10;
82 private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
83 new HashMap<HRegionInfo, ServerName[]>();
84
85 @BeforeClass
86 public static void setupBeforeClass() throws Exception {
87 Configuration conf = TEST_UTIL.getConfiguration();
88
89 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
90 FavoredNodeLoadBalancer.class, LoadBalancer.class);
91 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
92 TEST_UTIL.startMiniCluster(SLAVES);
93 CONNECTION = TEST_UTIL.getConnection();
94 admin = CONNECTION.getAdmin();
95 rp = new RegionPlacementMaintainer(conf);
96 }
97
98 @AfterClass
99 public static void tearDownAfterClass() throws Exception {
100 TEST_UTIL.shutdownMiniCluster();
101 }
102
103 @Test
104 public void testRegionPlacement() throws Exception {
105 String tableStr = "testRegionAssignment";
106 TableName table = TableName.valueOf(tableStr);
107
108 createTable(table, REGION_NUM);
109
110 TEST_UTIL.waitTableAvailable(table);
111
112
113
114 verifyRegionOnPrimaryRS(REGION_NUM);
115
116 FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
117
118 verifyRegionServerUpdated(currentPlan);
119
120
121
122
123
124
125
126
127 FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
128 FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
129
130 rp.updateAssignmentPlan(shuffledPlan);
131
132
133
134 verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
135
136
137
138 shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
139 FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
140
141
142 rp.updateAssignmentPlan(shuffledPlan);
143
144 verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
145
146
147 RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
148
149 rp.setTargetTableName(new String[]{tableStr});
150 List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
151 AssignmentVerificationReport report = reports.get(0);
152 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
153 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
154 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
155 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
156 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
157 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
158 assertTrue(report.getUnassignedRegions().size() == 0);
159
160
161 killRandomServerAndVerifyAssignment();
162
163
164 reports = rp.verifyRegionPlacement(false);
165 report = reports.get(0);
166 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
167 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
168 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
169 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
170 assertTrue("secondary " +
171 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
172 + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
173 (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
174 || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
175 assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
176 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
177 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
178 RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
179 }
180
181 private void killRandomServerAndVerifyAssignment()
182 throws IOException, InterruptedException, KeeperException {
183 ServerName serverToKill = null;
184 int killIndex = 0;
185 Random random = new Random(System.currentTimeMillis());
186 ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
187 LOG.debug("Server holding meta " + metaServer);
188 boolean isNamespaceServer = false;
189 do {
190
191 killIndex = random.nextInt(SLAVES);
192 serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
193 Collection<Region> regs =
194 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
195 isNamespaceServer = false;
196 for (Region r : regs) {
197 if (r.getRegionInfo().getTable().getNamespaceAsString()
198 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
199 isNamespaceServer = true;
200 break;
201 }
202 }
203 } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
204 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
205 LOG.debug("Stopping RS " + serverToKill);
206 Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
207 new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
208
209 for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
210 ServerName s = entry.getValue()[0];
211 if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
212 regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
213 entry.getValue()[1], entry.getValue()[2]));
214 LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
215 entry.getValue()[1] + " " + entry.getValue()[2]);
216 }
217 }
218 int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
219 TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
220 TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
221 int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
222 while (curr - orig < regionsToVerify.size()) {
223 LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
224 " Current #regions " + curr + " Original #regions " + orig);
225 Thread.sleep(200);
226 curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
227 }
228
229 for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
230 ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
231 .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
232 Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
233 LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
234 " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
235 + "/" + secondaryTertiaryServers.getSecond());
236 if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
237 ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
238 fail("Region " + entry.getKey() + " not present on any of the expected servers");
239 }
240 }
241
242 TEST_UTIL.getHBaseCluster().startRegionServer();
243 }
244
245
246
247
248 @Test
249 public void testRandomizedMatrix() {
250 int rows = 100;
251 int cols = 100;
252 float[][] matrix = new float[rows][cols];
253 Random random = new Random();
254 for (int i = 0; i < rows; i++) {
255 for (int j = 0; j < cols; j++) {
256 matrix[i][j] = random.nextFloat();
257 }
258 }
259
260
261 RegionPlacementMaintainer.RandomizedMatrix rm =
262 new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
263 float[][] transformed = rm.transform(matrix);
264 float[][] invertedTransformed = rm.invert(transformed);
265 for (int i = 0; i < rows; i++) {
266 for (int j = 0; j < cols; j++) {
267 if (matrix[i][j] != invertedTransformed[i][j]) {
268 throw new RuntimeException();
269 }
270 }
271 }
272
273
274
275 int[] transformedIndices = new int[rows];
276 for (int i = 0; i < rows; i++) {
277 transformedIndices[i] = random.nextInt(cols);
278 }
279 int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
280 float[] transformedValues = new float[rows];
281 float[] invertedTransformedValues = new float[rows];
282 for (int i = 0; i < rows; i++) {
283 transformedValues[i] = transformed[i][transformedIndices[i]];
284 invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
285 }
286 Arrays.sort(transformedValues);
287 Arrays.sort(invertedTransformedValues);
288 if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
289 throw new RuntimeException();
290 }
291 }
292
293
294
295
296
297
298
299
300 private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
301 FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
302 FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
303
304 for (Map.Entry<HRegionInfo, List<ServerName>> entry :
305 plan.getAssignmentMap().entrySet()) {
306 HRegionInfo region = entry.getKey();
307
308
309 List<ServerName> shuffledServerList = new ArrayList<ServerName>();
310 shuffledServerList.addAll(entry.getValue());
311
312
313 shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
314 shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
315
316
317 shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
318 }
319 return shuffledPlan;
320 }
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335 private void verifyRegionAssignment(FavoredNodesPlan plan,
336 int regionMovementNum, int numRegionsOnPrimaryRS)
337 throws InterruptedException, IOException {
338
339 verifyMETAUpdated(plan);
340
341
342 verifyRegionMovementNum(regionMovementNum);
343
344
345
346 verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
347
348
349 verifyRegionServerUpdated(plan);
350 }
351
352
353
354
355
356
357 private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
358 throws IOException {
359 FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
360 assertTrue("The assignment plan is NOT consistent with the expected plan ",
361 planFromMETA.equals(expectedPlan));
362 }
363
364
365
366
367 private void verifyRegionMovementNum(int expected)
368 throws InterruptedException, IOException {
369 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
370 HMaster m = cluster.getMaster();
371 int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
372
373 m.balance();
374
375 int retry = 10;
376 long sleep = 3000;
377 int attempt = 0;
378 int currentRegionOpened, regionMovement;
379 do {
380 currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
381 regionMovement= currentRegionOpened - lastRegionOpenedCount;
382 LOG.debug("There are " + regionMovement + "/" + expected +
383 " regions moved after " + attempt + " attempts");
384 Thread.sleep((++attempt) * sleep);
385 } while (regionMovement != expected && attempt <= retry);
386
387
388 lastRegionOpenedCount = currentRegionOpened;
389
390 assertEquals("There are only " + regionMovement + " instead of "
391 + expected + " region movement for " + attempt + " attempts",
392 regionMovement, expected);
393 }
394
395
396
397
398
399
400
401 private void verifyRegionOnPrimaryRS(int expectedNum)
402 throws IOException {
403 lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
404 assertEquals("Only " + expectedNum + " of user regions running " +
405 "on the primary region server", expectedNum ,
406 lastRegionOnPrimaryRSCount);
407 }
408
409
410
411
412
413
414
415 private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
416
417 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
418 for (int i = 0; i < SLAVES; i++) {
419 HRegionServer rs = cluster.getRegionServer(i);
420 for (Region region: rs.getOnlineRegions(TableName.valueOf("testRegionAssignment"))) {
421 InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
422 region.getRegionInfo().getEncodedName());
423 List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
424
425
426
427 if (favoredServerList == null) {
428 HTableDescriptor desc = region.getTableDesc();
429
430 assertNull(favoredSocketAddress);
431 assertTrue("User region " +
432 region.getTableDesc().getTableName() +
433 " should have favored nodes",
434 (desc.isRootRegion() || desc.isMetaRegion()));
435 } else {
436
437
438 assertTrue(favoredSocketAddress.length == favoredServerList.size());
439 assertTrue(favoredServerList.size() > 0);
440 for (int j = 0; j < favoredServerList.size(); j++) {
441 InetSocketAddress addrFromRS = favoredSocketAddress[j];
442 InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
443 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
444
445 assertNotNull(addrFromRS);
446 assertNotNull(addrFromPlan);
447 assertTrue("Region server " + rs.getServerName().getHostAndPort()
448 + " has the " + positions[j] +
449 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
450 addrFromRS + " which is inconsistent with the plan "
451 + addrFromPlan, addrFromRS.equals(addrFromPlan));
452 }
453 }
454 }
455 }
456 }
457
458
459
460
461
462
463
464
465
466 private int getNumRegionisOnPrimaryRS() throws IOException {
467 final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
468 final AtomicInteger totalRegionNum = new AtomicInteger(0);
469 LOG.info("The start of region placement verification");
470 MetaScannerVisitor visitor = new MetaScannerVisitor() {
471 public boolean processRow(Result result) throws IOException {
472 try {
473 @SuppressWarnings("deprecation")
474 HRegionInfo info = MetaScanner.getHRegionInfo(result);
475 if(info.getTable().getNamespaceAsString()
476 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
477 return true;
478 }
479 byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
480 HConstants.SERVER_QUALIFIER);
481 byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
482 FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
483
484 ServerName[] favoredServerList =
485 FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
486 favoredNodesAssignmentPlan.put(info, favoredServerList);
487
488 Position[] positions = Position.values();
489 if (info != null) {
490 totalRegionNum.incrementAndGet();
491 if (server != null) {
492 ServerName serverName =
493 ServerName.valueOf(Bytes.toString(server), -1);
494 if (favoredNodes != null) {
495 String placement = "[NOT FAVORED NODE]";
496 for (int i = 0; i < favoredServerList.length; i++) {
497 if (favoredServerList[i].equals(serverName)) {
498 placement = positions[i].toString();
499 if (i == Position.PRIMARY.ordinal()) {
500 regionOnPrimaryNum.incrementAndGet();
501 }
502 break;
503 }
504 }
505 LOG.info(info.getRegionNameAsString() + " on " +
506 serverName + " " + placement);
507 } else {
508 LOG.info(info.getRegionNameAsString() + " running on " +
509 serverName + " but there is no favored region server");
510 }
511 } else {
512 LOG.info(info.getRegionNameAsString() +
513 " not assigned to any server");
514 }
515 }
516 return true;
517 } catch (RuntimeException e) {
518 LOG.error("Result=" + result);
519 throw e;
520 }
521 }
522
523 @Override
524 public void close() throws IOException {}
525 };
526 MetaScanner.metaScan(CONNECTION, visitor);
527 LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
528 totalRegionNum.intValue() + " regions running on the primary" +
529 " region servers" );
530 return regionOnPrimaryNum.intValue() ;
531 }
532
533
534
535
536
537
538
539
540 private static void createTable(TableName tableName, int regionNum)
541 throws IOException {
542 int expectedRegions = regionNum;
543 byte[][] splitKeys = new byte[expectedRegions - 1][];
544 for (int i = 1; i < expectedRegions; i++) {
545 byte splitKey = (byte) i;
546 splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
547 }
548
549 HTableDescriptor desc = new HTableDescriptor(tableName);
550 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
551 admin.createTable(desc, splitKeys);
552
553 HTable ht = (HTable) CONNECTION.getTable(tableName);
554 @SuppressWarnings("deprecation")
555 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
556 assertEquals("Tried to create " + expectedRegions + " regions "
557 + "but only found " + regions.size(), expectedRegions, regions.size());
558 ht.close();
559 }
560 }