View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.net.InetSocketAddress;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Random;
35  import java.util.concurrent.atomic.AtomicInteger;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.NamespaceDescriptor;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.client.Admin;
51  import org.apache.hadoop.hbase.client.Connection;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.client.MetaScanner;
54  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
55  import org.apache.hadoop.hbase.client.Result;
56  import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
57  import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
58  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
59  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
60  import org.apache.hadoop.hbase.regionserver.HRegionServer;
61  import org.apache.hadoop.hbase.regionserver.Region;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.hbase.util.Pair;
64  import org.apache.zookeeper.KeeperException;
65  import org.junit.AfterClass;
66  import org.junit.BeforeClass;
67  import org.junit.Test;
68  import org.junit.experimental.categories.Category;
69  
70  
71  @Category(MediumTests.class)
72  public class TestRegionPlacement {
73    final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
74    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
75    private final static int SLAVES = 10;
76    private static Connection CONNECTION;
77    private static Admin admin;
78    private static RegionPlacementMaintainer rp;
79    private static Position[] positions = Position.values();
80    private int lastRegionOnPrimaryRSCount = 0;
81    private int REGION_NUM = 10;
82    private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
83        new HashMap<HRegionInfo, ServerName[]>();
84  
85    @BeforeClass
86    public static void setupBeforeClass() throws Exception {
87      Configuration conf = TEST_UTIL.getConfiguration();
88      // Enable the favored nodes based load balancer
89      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
90          FavoredNodeLoadBalancer.class, LoadBalancer.class);
91      conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
92      TEST_UTIL.startMiniCluster(SLAVES);
93      CONNECTION = TEST_UTIL.getConnection();
94      admin = CONNECTION.getAdmin();
95      rp = new RegionPlacementMaintainer(conf);
96    }
97  
98    @AfterClass
99    public static void tearDownAfterClass() throws Exception {
100     TEST_UTIL.shutdownMiniCluster();
101   }
102 
103   @Test
104   public void testRegionPlacement() throws Exception {
105     String tableStr = "testRegionAssignment";
106     TableName table = TableName.valueOf(tableStr);
107     // Create a table with REGION_NUM regions.
108     createTable(table, REGION_NUM);
109 
110     TEST_UTIL.waitTableAvailable(table);
111 
112     // Verify all the user regions are assigned to the primary region server
113     // based on the plan
114     verifyRegionOnPrimaryRS(REGION_NUM);
115 
116     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
117     // Verify all the region server are update with the latest favored nodes
118     verifyRegionServerUpdated(currentPlan);
119     // Test Case 2: To verify whether the region placement tools can
120     // correctly update the new assignment plan to hbase:meta and Region Server.
121     // The new assignment plan is generated by shuffle the existing assignment
122     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
123     // Shuffle the plan by switching the secondary region server with
124     // the tertiary.
125 
126     // Shuffle the secondary with tertiary favored nodes
127     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
128         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
129     // Let the region placement update the hbase:meta and Region Servers
130     rp.updateAssignmentPlan(shuffledPlan);
131 
132     // Verify the region assignment. There are supposed to no region reassignment
133     // All the regions are still on the primary region server
134     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
135 
136     // Shuffle the plan by switching the primary with secondary and
137     // verify the region reassignment is consistent with the plan.
138     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
139         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
140 
141     // Let the region placement update the hbase:meta and Region Servers
142     rp.updateAssignmentPlan(shuffledPlan);
143 
144     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
145 
146     // also verify that the AssignmentVerificationReport has the correct information
147     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
148     // we are interested in only one table (and hence one report)
149     rp.setTargetTableName(new String[]{tableStr});
150     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
151     AssignmentVerificationReport report = reports.get(0);
152     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
153     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
154     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
155     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
156     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
157     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
158     assertTrue(report.getUnassignedRegions().size() == 0);
159 
160     // Check when a RS stops, the regions get assigned to their secondary/tertiary
161     killRandomServerAndVerifyAssignment();
162     
163     // also verify that the AssignmentVerificationReport has the correct information
164     reports = rp.verifyRegionPlacement(false);
165     report = reports.get(0);
166     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
167     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
168     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
169     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
170     assertTrue("secondary " +
171     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
172         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
173         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
174         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
175     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
176         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
177         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
178     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
179   }
180 
181   private void killRandomServerAndVerifyAssignment() 
182       throws IOException, InterruptedException, KeeperException {
183     ServerName serverToKill = null;
184     int killIndex = 0;
185     Random random = new Random(System.currentTimeMillis());
186     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
187     LOG.debug("Server holding meta " + metaServer);
188     boolean isNamespaceServer = false;
189     do {
190       // kill a random non-meta server carrying at least one region
191       killIndex = random.nextInt(SLAVES);
192       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
193       Collection<Region> regs =
194           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
195       isNamespaceServer = false;
196       for (Region r : regs) {
197         if (r.getRegionInfo().getTable().getNamespaceAsString()
198             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
199           isNamespaceServer = true;
200           break;
201         }
202       }
203     } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
204         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
205     LOG.debug("Stopping RS " + serverToKill);
206     Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
207         new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
208     // mark the regions to track
209     for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
210       ServerName s = entry.getValue()[0];
211       if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
212         regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
213             entry.getValue()[1], entry.getValue()[2]));
214         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
215             entry.getValue()[1] + " " + entry.getValue()[2]);
216       }
217     }
218     int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
219     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
220     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
221     int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
222     while (curr - orig < regionsToVerify.size()) {
223       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
224           " Current #regions " + curr + " Original #regions " + orig);
225       Thread.sleep(200);
226       curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
227     }
228     // now verify
229     for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
230       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
231           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
232       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
233       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
234           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
235           + "/" + secondaryTertiaryServers.getSecond());
236       if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
237           ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
238         fail("Region " + entry.getKey() + " not present on any of the expected servers");
239       }
240     }
241     // start(reinstate) region server since we killed one before
242     TEST_UTIL.getHBaseCluster().startRegionServer();
243   }
244 
245   /**
246    * Used to test the correctness of this class.
247    */
248   @Test
249   public void testRandomizedMatrix() {
250     int rows = 100;
251     int cols = 100;
252     float[][] matrix = new float[rows][cols];
253     Random random = new Random();
254     for (int i = 0; i < rows; i++) {
255       for (int j = 0; j < cols; j++) {
256         matrix[i][j] = random.nextFloat();
257       }
258     }
259 
260     // Test that inverting a transformed matrix gives the original matrix.
261     RegionPlacementMaintainer.RandomizedMatrix rm =
262       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
263     float[][] transformed = rm.transform(matrix);
264     float[][] invertedTransformed = rm.invert(transformed);
265     for (int i = 0; i < rows; i++) {
266       for (int j = 0; j < cols; j++) {
267         if (matrix[i][j] != invertedTransformed[i][j]) {
268           throw new RuntimeException();
269         }
270       }
271     }
272 
273     // Test that the indices on a transformed matrix can be inverted to give
274     // the same values on the original matrix.
275     int[] transformedIndices = new int[rows];
276     for (int i = 0; i < rows; i++) {
277       transformedIndices[i] = random.nextInt(cols);
278     }
279     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
280     float[] transformedValues = new float[rows];
281     float[] invertedTransformedValues = new float[rows];
282     for (int i = 0; i < rows; i++) {
283       transformedValues[i] = transformed[i][transformedIndices[i]];
284       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
285     }
286     Arrays.sort(transformedValues);
287     Arrays.sort(invertedTransformedValues);
288     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
289       throw new RuntimeException();
290     }
291   }
292 
293   /**
294    * Shuffle the assignment plan by switching two favored node positions.
295    * @param plan The assignment plan
296    * @param p1 The first switch position
297    * @param p2 The second switch position
298    * @return
299    */
300   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
301       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
302     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
303 
304     for (Map.Entry<HRegionInfo, List<ServerName>> entry :
305       plan.getAssignmentMap().entrySet()) {
306       HRegionInfo region = entry.getKey();
307 
308       // copy the server list from the original plan
309       List<ServerName> shuffledServerList = new ArrayList<ServerName>();
310       shuffledServerList.addAll(entry.getValue());
311 
312       // start to shuffle
313       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
314       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
315 
316       // update the plan
317       shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
318     }
319     return shuffledPlan;
320   }
321 
322   /**
323    * To verify the region assignment status.
324    * It will check the assignment plan consistency between hbase:meta and
325    * region servers.
326    * Also it will verify weather the number of region movement and
327    * the number regions on the primary region server are expected
328    *
329    * @param plan
330    * @param regionMovementNum
331    * @param numRegionsOnPrimaryRS
332    * @throws InterruptedException
333    * @throws IOException
334    */
335   private void verifyRegionAssignment(FavoredNodesPlan plan,
336       int regionMovementNum, int numRegionsOnPrimaryRS)
337   throws InterruptedException, IOException {
338     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
339     verifyMETAUpdated(plan);
340 
341     // Verify the number of region movement is expected
342     verifyRegionMovementNum(regionMovementNum);
343 
344     // Verify the number of regions is assigned to the primary region server
345     // based on the plan is expected
346     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
347 
348     // Verify all the online region server are updated with the assignment plan
349     verifyRegionServerUpdated(plan);
350   }
351 
352   /**
353    * Verify the meta has updated to the latest assignment plan
354    * @param plan
355    * @throws IOException
356    */
357   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
358   throws IOException {
359     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
360     assertTrue("The assignment plan is NOT consistent with the expected plan ",
361         planFromMETA.equals(expectedPlan));
362   }
363 
364   /**
365    * Verify the number of region movement is expected
366    */
367   private void verifyRegionMovementNum(int expected)
368       throws InterruptedException, IOException {
369     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
370     HMaster m = cluster.getMaster();
371     int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
372     // get the assignments start to execute
373     m.balance();
374 
375     int retry = 10;
376     long sleep = 3000;
377     int attempt = 0;
378     int currentRegionOpened, regionMovement;
379     do {
380       currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
381       regionMovement= currentRegionOpened - lastRegionOpenedCount;
382       LOG.debug("There are " + regionMovement + "/" + expected +
383           " regions moved after " + attempt + " attempts");
384       Thread.sleep((++attempt) * sleep);
385     } while (regionMovement != expected && attempt <= retry);
386 
387     // update the lastRegionOpenedCount
388     lastRegionOpenedCount = currentRegionOpened;
389 
390     assertEquals("There are only " + regionMovement + " instead of "
391           + expected + " region movement for " + attempt + " attempts",
392           regionMovement, expected);
393   }
394 
395   /**
396    * Verify the number of user regions is assigned to the primary
397    * region server based on the plan is expected
398    * @param expectedNum.
399    * @throws IOException
400    */
401   private void verifyRegionOnPrimaryRS(int expectedNum)
402       throws IOException {
403     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
404     assertEquals("Only " +  expectedNum + " of user regions running " +
405         "on the primary region server", expectedNum ,
406         lastRegionOnPrimaryRSCount);
407   }
408 
409   /**
410    * Verify all the online region servers has been updated to the
411    * latest assignment plan
412    * @param plan
413    * @throws IOException
414    */
415   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
416     // Verify all region servers contain the correct favored nodes information
417     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
418     for (int i = 0; i < SLAVES; i++) {
419       HRegionServer rs = cluster.getRegionServer(i);
420       for (Region region: rs.getOnlineRegions(TableName.valueOf("testRegionAssignment"))) {
421         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
422             region.getRegionInfo().getEncodedName());
423         List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
424 
425         // All regions are supposed to have favored nodes,
426         // except for hbase:meta and ROOT
427         if (favoredServerList == null) {
428           HTableDescriptor desc = region.getTableDesc();
429           // Verify they are ROOT and hbase:meta regions since no favored nodes
430           assertNull(favoredSocketAddress);
431           assertTrue("User region " +
432               region.getTableDesc().getTableName() +
433               " should have favored nodes",
434               (desc.isRootRegion() || desc.isMetaRegion()));
435         } else {
436           // For user region, the favored nodes in the region server should be
437           // identical to favored nodes in the assignmentPlan
438           assertTrue(favoredSocketAddress.length == favoredServerList.size());
439           assertTrue(favoredServerList.size() > 0);
440           for (int j = 0; j < favoredServerList.size(); j++) {
441             InetSocketAddress addrFromRS = favoredSocketAddress[j];
442             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
443                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
444 
445             assertNotNull(addrFromRS);
446             assertNotNull(addrFromPlan);
447             assertTrue("Region server " + rs.getServerName().getHostAndPort()
448                 + " has the " + positions[j] +
449                 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
450                 addrFromRS + " which is inconsistent with the plan "
451                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
452           }
453         }
454       }
455     }
456   }
457 
458   /**
459    * Check whether regions are assigned to servers consistent with the explicit
460    * hints that are persisted in the hbase:meta table.
461    * Also keep track of the number of the regions are assigned to the
462    * primary region server.
463    * @return the number of regions are assigned to the primary region server
464    * @throws IOException
465    */
466   private int getNumRegionisOnPrimaryRS() throws IOException {
467     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
468     final AtomicInteger totalRegionNum = new AtomicInteger(0);
469     LOG.info("The start of region placement verification");
470     MetaScannerVisitor visitor = new MetaScannerVisitor() {
471       public boolean processRow(Result result) throws IOException {
472         try {
473           @SuppressWarnings("deprecation")
474           HRegionInfo info = MetaScanner.getHRegionInfo(result);
475           if(info.getTable().getNamespaceAsString()
476               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
477             return true;
478           }
479           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
480               HConstants.SERVER_QUALIFIER);
481           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
482               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
483           // Add the favored nodes into assignment plan
484           ServerName[] favoredServerList =
485               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
486           favoredNodesAssignmentPlan.put(info, favoredServerList);
487 
488           Position[] positions = Position.values();
489           if (info != null) {
490             totalRegionNum.incrementAndGet();
491             if (server != null) {
492               ServerName serverName =
493                   ServerName.valueOf(Bytes.toString(server), -1);
494               if (favoredNodes != null) {
495                 String placement = "[NOT FAVORED NODE]";
496                 for (int i = 0; i < favoredServerList.length; i++) {
497                   if (favoredServerList[i].equals(serverName)) {
498                     placement = positions[i].toString();
499                     if (i == Position.PRIMARY.ordinal()) {
500                       regionOnPrimaryNum.incrementAndGet();
501                     }
502                     break;
503                   }
504                 }
505                 LOG.info(info.getRegionNameAsString() + " on " +
506                     serverName + " " + placement);
507               } else {
508                 LOG.info(info.getRegionNameAsString() + " running on " +
509                     serverName + " but there is no favored region server");
510               }
511             } else {
512               LOG.info(info.getRegionNameAsString() +
513                   " not assigned to any server");
514             }
515           }
516           return true;
517         } catch (RuntimeException e) {
518           LOG.error("Result=" + result);
519           throw e;
520         }
521       }
522 
523       @Override
524       public void close() throws IOException {}
525     };
526     MetaScanner.metaScan(CONNECTION, visitor);
527     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
528         totalRegionNum.intValue() + " regions running on the primary" +
529         " region servers" );
530     return regionOnPrimaryNum.intValue() ;
531   }
532 
533   /**
534    * Create a table with specified table name and region number.
535    * @param tablename
536    * @param regionNum
537    * @return
538    * @throws IOException
539    */
540   private static void createTable(TableName tableName, int regionNum)
541       throws IOException {
542     int expectedRegions = regionNum;
543     byte[][] splitKeys = new byte[expectedRegions - 1][];
544     for (int i = 1; i < expectedRegions; i++) {
545       byte splitKey = (byte) i;
546       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
547     }
548 
549     HTableDescriptor desc = new HTableDescriptor(tableName);
550     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
551     admin.createTable(desc, splitKeys);
552 
553     HTable ht = (HTable) CONNECTION.getTable(tableName);
554     @SuppressWarnings("deprecation")
555     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
556     assertEquals("Tried to create " + expectedRegions + " regions "
557         + "but only found " + regions.size(), expectedRegions, regions.size());
558     ht.close();
559   }
560 }