View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22  import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.concurrent.ExecutorService;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.Abortable;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HRegionLocation;
40  import org.apache.hadoop.hbase.MetaTableAccessor;
41  import org.apache.hadoop.hbase.RegionLocations;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.TableNotFoundException;
45  import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
46  import org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
47  import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
48  import org.apache.hadoop.hbase.testclassification.MediumTests;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.HBaseFsck;
51  import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
52  import org.apache.hadoop.hbase.util.HBaseFsckRepair;
53  import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
54  import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
55  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
56  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
57  import org.junit.After;
58  import org.junit.Before;
59  import org.junit.Test;
60  import org.junit.experimental.categories.Category;
61  
62  /**
63   * Tests the scenarios where replicas are enabled for the meta table
64   */
65  @Category(MediumTests.class)
66  public class TestMetaWithReplicas {
67    static final Log LOG = LogFactory.getLog(TestMetaWithReplicas.class);
68    private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
69  
70    @Before
71    public void setup() throws Exception {
72      TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
73      TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
74      TEST_UTIL.getConfiguration().setInt(
75          StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
76      TEST_UTIL.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart", 3);
77      TEST_UTIL.startMiniCluster(4);
78      
79      TEST_UTIL.waitUntilAllSystemRegionsAssigned();
80  
81      // disable the balancer
82      LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
83          new Abortable() {
84        boolean aborted = false;
85        @Override
86        public boolean isAborted() {
87          return aborted;
88        }
89        @Override
90        public void abort(String why, Throwable e) {
91          aborted = true;
92        }
93      });
94      l.setBalancerOn(false);
95      for (int replicaId = 1; replicaId < 3; replicaId ++) {
96        HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
97          replicaId);
98        TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(h);
99      }
100     LOG.debug("All meta replicas assigned");
101   }
102 
103   @After
104   public void tearDown() throws Exception {
105     TEST_UTIL.shutdownMiniCluster();
106   }
107 
108   @Test
109   public void testMetaHTDReplicaCount() throws Exception {
110     assertTrue(TEST_UTIL.getHBaseAdmin().getTableDescriptor(TableName.META_TABLE_NAME)
111         .getRegionReplication() == 3);
112   }
113 
114   @Test
115   public void testZookeeperNodesForReplicas() throws Exception {
116     // Checks all the znodes exist when meta's replicas are enabled
117     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
118     Configuration conf = TEST_UTIL.getConfiguration();
119     String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
120         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
121     String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
122         conf.get("zookeeper.znode.metaserver", "meta-region-server"));
123     // check that the data in the znode is parseable (this would also mean the znode exists)
124     byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
125     ServerName.parseFrom(data);
126     for (int i = 1; i < 3; i++) {
127       String secZnode = ZKUtil.joinZNode(baseZNode,
128           conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
129       String str = zkw.getZNodeForReplica(i);
130       assertTrue(str.equals(secZnode));
131       // check that the data in the znode is parseable (this would also mean the znode exists)
132       data = ZKUtil.getData(zkw, secZnode);
133       ServerName.parseFrom(data);
134     }
135   }
136 
137   @Test
138   public void testShutdownHandling() throws Exception {
139     // This test creates a table, flushes the meta (with 3 replicas), kills the
140     // server holding the primary meta replica. Then it does a put/get into/from
141     // the test table. The put/get operations would use the replicas to locate the
142     // location of the test table's region
143     shutdownMetaAndDoValidations(TEST_UTIL);
144   }
145 
146   public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
147     // This test creates a table, flushes the meta (with 3 replicas), kills the
148     // server holding the primary meta replica. Then it does a put/get into/from
149     // the test table. The put/get operations would use the replicas to locate the
150     // location of the test table's region
151     ZooKeeperWatcher zkw = util.getZooKeeperWatcher();
152     Configuration conf = util.getConfiguration();
153     conf.setBoolean(HConstants.USE_META_REPLICAS, true);
154 
155     String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
156         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
157     String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
158         conf.get("zookeeper.znode.metaserver", "meta-region-server"));
159     byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
160     ServerName primary = ServerName.parseFrom(data);
161 
162     byte[] TABLE = Bytes.toBytes("testShutdownHandling");
163     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
164     if (util.getHBaseAdmin().tableExists(TABLE)) {
165       util.getHBaseAdmin().disableTable(TABLE);
166       util.getHBaseAdmin().deleteTable(TABLE);
167     }
168     Table htable = util.createTable(TABLE, FAMILIES, conf);
169 
170     util.getHBaseAdmin().flush(TableName.META_TABLE_NAME);
171     Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
172        30000) * 6);
173     Connection c = ConnectionFactory.createConnection(util.getConfiguration());
174     List<HRegionInfo> regions = MetaTableAccessor.getTableRegions(zkw, c,
175         TableName.valueOf(TABLE));
176     HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
177     // Ensure that the primary server for test table is not the same one as the primary
178     // of the meta region since we will be killing the srv holding the meta's primary...
179     // We want to be able to write to the test table even when the meta is not present ..
180     // If the servers are the same, then move the test table's region out of the server
181     // to another random server
182     if (hrl.getServerName().equals(primary)) {
183       util.getHBaseAdmin().move(hrl.getRegionInfo().getEncodedNameAsBytes(), null);
184       // wait for the move to complete
185       do {
186         Thread.sleep(10);
187         hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
188       } while (primary.equals(hrl.getServerName()));
189       util.getHBaseAdmin().flush(TableName.META_TABLE_NAME);
190       Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
191          30000) * 3);
192     }
193     ServerName master = util.getHBaseClusterInterface().getClusterStatus().getMaster();
194     // kill the master so that regionserver recovery is not triggered at all
195     // for the meta server
196     util.getHBaseClusterInterface().stopMaster(master);
197     util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
198     if (!master.equals(primary)) {
199       util.getHBaseClusterInterface().killRegionServer(primary);
200       util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
201     }
202     ((ClusterConnection)c).clearRegionCache();
203     htable.close();
204     htable = c.getTable(TableName.valueOf(TABLE));
205     byte[] row = "test".getBytes();
206     Put put = new Put(row);
207     put.add("foo".getBytes(), row, row);
208     BufferedMutator m = c.getBufferedMutator(TableName.valueOf(TABLE));
209     m.mutate(put);
210     m.flush();
211     // Try to do a get of the row that was just put
212     Get get = new Get(row);
213     Result r = htable.get(get);
214     assertTrue(Arrays.equals(r.getRow(), row));
215     // now start back the killed servers and disable use of replicas. That would mean
216     // calls go to the primary
217     util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
218     util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
219     util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
220     ((ClusterConnection)c).clearRegionCache();
221     htable.close();
222     conf.setBoolean(HConstants.USE_META_REPLICAS, false);
223     htable = c.getTable(TableName.valueOf(TABLE));
224     r = htable.get(get);
225     assertTrue(Arrays.equals(r.getRow(), row));
226   }
227 
228   @Test
229   public void testMetaLookupThreadPoolCreated() throws Exception {
230     byte[] TABLE = Bytes.toBytes("testMetaLookupThreadPoolCreated");
231     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
232     if (TEST_UTIL.getHBaseAdmin().tableExists(TABLE)) {
233       TEST_UTIL.getHBaseAdmin().disableTable(TABLE);
234       TEST_UTIL.getHBaseAdmin().deleteTable(TABLE);
235     }
236     Table htable = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration());
237     byte[] row = "test".getBytes();
238     HConnectionImplementation c = ((HConnectionImplementation)((HTable)htable).connection);
239     // check that metalookup pool would get created
240     c.relocateRegion(TABLE, row);
241     ExecutorService ex = c.getCurrentMetaLookupPool();
242     assert(ex != null);
243   }
244 
245   @Test
246   public void testChangingReplicaCount() throws Exception {
247     // tests changing the replica count across master restarts
248     // reduce the replica count from 3 to 2
249     stopMasterAndValidateReplicaCount(3, 2);
250     // increase the replica count from 2 to 3
251     stopMasterAndValidateReplicaCount(2, 3);
252   }
253 
254   private void stopMasterAndValidateReplicaCount(int originalReplicaCount, int newReplicaCount)
255       throws Exception {
256     ServerName sn = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
257     TEST_UTIL.getHBaseClusterInterface().stopMaster(sn);
258     TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(sn, 60000);
259     List<String> metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes();
260     assert(metaZnodes.size() == originalReplicaCount); //we should have what was configured before
261     TEST_UTIL.getHBaseClusterInterface().getConf().setInt(HConstants.META_REPLICAS_NUM,
262         newReplicaCount);
263     TEST_UTIL.getHBaseClusterInterface().startMaster(sn.getHostname(), 0);
264     TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
265     int count = 0;
266     do {
267       metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes();
268       Thread.sleep(10);
269       count++;
270       // wait for the count to be different from the originalReplicaCount. When the
271       // replica count is reduced, that will happen when the master unassigns excess
272       // replica, and deletes the excess znodes
273     } while (metaZnodes.size() == originalReplicaCount && count < 1000);
274     assert(metaZnodes.size() == newReplicaCount);
275     // also check if hbck returns without errors
276     TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM,
277         newReplicaCount);
278     HBaseFsck hbck = HbckTestingUtil.doFsck(TEST_UTIL.getConfiguration(), false);
279     HbckTestingUtil.assertNoErrors(hbck);
280   }
281 
282   @Test
283   public void testHBaseFsckWithMetaReplicas() throws Exception {
284     HBaseFsck hbck = HbckTestingUtil.doFsck(TEST_UTIL.getConfiguration(), false);
285     HbckTestingUtil.assertNoErrors(hbck);
286   }
287 
288   @Test
289   public void testHBaseFsckWithFewerMetaReplicas() throws Exception {
290     ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection(
291         TEST_UTIL.getConfiguration());
292     RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW,
293         false, false);
294     HBaseFsckRepair.closeRegionSilentlyAndWait(c,
295         rl.getRegionLocation(1).getServerName(), rl.getRegionLocation(1).getRegionInfo());
296     // check that problem exists
297     HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
298     assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION});
299     // fix the problem
300     hbck = doFsck(TEST_UTIL.getConfiguration(), true);
301     // run hbck again to make sure we don't see any errors
302     hbck = doFsck(TEST_UTIL.getConfiguration(), false);
303     assertErrors(hbck, new ERROR_CODE[]{});
304   }
305 
306   @Test
307   public void testHBaseFsckWithFewerMetaReplicaZnodes() throws Exception {
308     ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection(
309         TEST_UTIL.getConfiguration());
310     RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW,
311         false, false);
312     HBaseFsckRepair.closeRegionSilentlyAndWait(c,
313         rl.getRegionLocation(2).getServerName(), rl.getRegionLocation(2).getRegionInfo());
314     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
315     ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(2));
316     // check that problem exists
317     HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
318     assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION});
319     // fix the problem
320     hbck = doFsck(TEST_UTIL.getConfiguration(), true);
321     // run hbck again to make sure we don't see any errors
322     hbck = doFsck(TEST_UTIL.getConfiguration(), false);
323     assertErrors(hbck, new ERROR_CODE[]{});
324   }
325 
326   @Test
327   public void testAccessingUnknownTables() throws Exception {
328     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
329     conf.setBoolean(HConstants.USE_META_REPLICAS, true);
330     Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf("RandomTable"));
331     Get get = new Get(Bytes.toBytes("foo"));
332     try {
333       table.get(get);
334     } catch (TableNotFoundException t) {
335       return;
336     }
337     fail("Expected TableNotFoundException");
338   }
339 
340   @Test
341   public void testMetaAddressChange() throws Exception {
342     // checks that even when the meta's location changes, the various
343     // caches update themselves. Uses the master operations to test
344     // this
345     Configuration conf = TEST_UTIL.getConfiguration();
346     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
347     String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
348         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
349     String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
350         conf.get("zookeeper.znode.metaserver", "meta-region-server"));
351     // check that the data in the znode is parseable (this would also mean the znode exists)
352     byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
353     ServerName currentServer = ServerName.parseFrom(data);
354     Collection<ServerName> liveServers = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers();
355     ServerName moveToServer = null;
356     for (ServerName s : liveServers) {
357       if (!currentServer.equals(s)) {
358         moveToServer = s;
359       }
360     }
361     assert(moveToServer != null);
362     String tableName = "randomTable5678";
363     TEST_UTIL.createTable(TableName.valueOf(tableName), "f");
364     assertTrue(TEST_UTIL.getHBaseAdmin().tableExists(tableName));
365     TEST_UTIL.getHBaseAdmin().move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
366         Bytes.toBytes(moveToServer.getServerName()));
367     int i = 0;
368     do {
369       Thread.sleep(10);
370       data = ZKUtil.getData(zkw, primaryMetaZnode);
371       currentServer = ServerName.parseFrom(data);
372       i++;
373     } while (!moveToServer.equals(currentServer) && i < 1000); //wait for 10 seconds overall
374     assert(i != 1000);
375     TEST_UTIL.getHBaseAdmin().disableTable("randomTable5678");
376     assertTrue(TEST_UTIL.getHBaseAdmin().isTableDisabled("randomTable5678"));
377   }
378 
379   @Test
380   public void testShutdownOfReplicaHolder() throws Exception {
381     // checks that the when the server holding meta replica is shut down, the meta replica
382     // can be recovered
383     RegionLocations rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()).
384         locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true);
385     HRegionLocation hrl = rl.getRegionLocation(1);
386     ServerName oldServer = hrl.getServerName();
387     TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
388     int i = 0;
389     do {
390       LOG.debug("Waiting for the replica " + hrl.getRegionInfo() + " to come up");
391       Thread.sleep(30000); //wait for the detection/recovery
392       rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()).
393           locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true);
394       hrl = rl.getRegionLocation(1);
395       i++;
396     } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3);
397     assertTrue(i != 3);
398   }
399 
400   @Test
401   public void testHBaseFsckWithExcessMetaReplicas() throws Exception {
402     HBaseFsck hbck = new HBaseFsck(TEST_UTIL.getConfiguration());
403     // Create a meta replica (this will be the 4th one) and assign it
404     HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
405         HRegionInfo.FIRST_META_REGIONINFO, 3);
406     // create in-memory state otherwise master won't assign
407     TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
408              .getRegionStates().createRegionState(h);
409     TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h);
410     HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h);
411     // check that problem exists
412     hbck = doFsck(TEST_UTIL.getConfiguration(), false);
413     assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN, ERROR_CODE.SHOULD_NOT_BE_DEPLOYED});
414     // fix the problem
415     hbck = doFsck(TEST_UTIL.getConfiguration(), true);
416     // run hbck again to make sure we don't see any errors
417     hbck = doFsck(TEST_UTIL.getConfiguration(), false);
418     assertErrors(hbck, new ERROR_CODE[]{});
419   }
420 
421   @Test
422   public void testMetaTableReplicaAssignment() throws Exception {
423     final ClusterConnection c =
424         ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
425     final RegionLocations rl =
426         c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
427 
428     final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
429     LOG.debug("The hbase:meta default replica region is in server: " + meta0SN);
430     final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
431     LOG.debug("The hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo() +
432       " is in server: " + meta1SN);
433 
434     LOG.debug("Killing the region server " + meta1SN +
435       " that hosts hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo());
436     TEST_UTIL.getHBaseClusterInterface().killRegionServer(meta1SN);
437     TEST_UTIL.getHBaseClusterInterface().waitForRegionServerToStop(meta1SN, 60000);
438 
439     ServerName masterSN = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
440     LOG.debug("Killing the master server " + masterSN);
441     TEST_UTIL.getHBaseClusterInterface().stopMaster(masterSN);
442     TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(masterSN, 60000);
443     LOG.debug("Restarting the master server " + masterSN);
444     TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), masterSN.getPort());
445     TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
446 
447     // wait for replica 1 to be re-assigned
448     TEST_UTIL.waitFor(60000, 100, new ExplainingPredicate<IOException>() {
449 
450       @Override
451       public boolean evaluate() throws IOException {
452         RegionLocations rls =
453             c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
454         HRegionLocation loc = rls.getRegionLocation(1);
455         if (loc != null && !meta1SN.equals(loc.getServerName())) {
456           LOG.debug("The hbase:meta replica 1 region " + rls.getRegionLocation(1).getRegionInfo() +
457               " is now moved from server " + meta1SN + " to server " + loc.getServerName());
458           return true;
459         } else {
460           return false;
461         }
462       }
463 
464       @Override
465       public String explainFailure() throws IOException {
466         return "The hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo() +
467             " has not been assigned in time";
468       }
469     });
470 
471     LOG.debug("Killing the region server " + meta0SN +
472       " that hosts hbase:meta default replica region " + rl.getRegionLocation(0).getRegionInfo());
473     TEST_UTIL.getHBaseClusterInterface().killRegionServer(meta0SN);
474     TEST_UTIL.getHBaseClusterInterface().waitForRegionServerToStop(meta0SN, 60000);
475 
476     TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(
477       HRegionInfo.FIRST_META_REGIONINFO);
478 
479     // wait for default replica to be re-assigned
480     TEST_UTIL.waitFor(60000, 100, new ExplainingPredicate<IOException>() {
481 
482       @Override
483       public boolean evaluate() throws IOException {
484         RegionLocations rls =
485             c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
486         HRegionLocation loc = rls.getRegionLocation(0);
487         if (loc != null && !meta0SN.equals(loc.getServerName())) {
488           LOG.debug(
489             "The hbase:meta default replica region " + rls.getRegionLocation(0).getRegionInfo() +
490                 " is now moved from server " + meta0SN + " to server " + loc.getServerName());
491           return true;
492         } else {
493           return false;
494         }
495       }
496 
497       @Override
498       public String explainFailure() throws IOException {
499         return "The hbase:meta default replica region " + rl.getRegionLocation(0).getRegionInfo() +
500             " has not been assigned in time";
501       }
502     });
503   }
504 }