1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.List;
30 import java.util.concurrent.ExecutorService;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.Abortable;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HRegionLocation;
40 import org.apache.hadoop.hbase.MetaTableAccessor;
41 import org.apache.hadoop.hbase.RegionLocations;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.TableNotFoundException;
45 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
46 import org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
47 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
48 import org.apache.hadoop.hbase.testclassification.MediumTests;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.HBaseFsck;
51 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
52 import org.apache.hadoop.hbase.util.HBaseFsckRepair;
53 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
54 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
55 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
56 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
57 import org.junit.After;
58 import org.junit.Before;
59 import org.junit.Test;
60 import org.junit.experimental.categories.Category;
61
62
63
64
65 @Category(MediumTests.class)
66 public class TestMetaWithReplicas {
67 static final Log LOG = LogFactory.getLog(TestMetaWithReplicas.class);
68 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
69
70 @Before
71 public void setup() throws Exception {
72 TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
73 TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
74 TEST_UTIL.getConfiguration().setInt(
75 StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
76 TEST_UTIL.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart", 3);
77 TEST_UTIL.startMiniCluster(4);
78
79 TEST_UTIL.waitUntilAllSystemRegionsAssigned();
80
81
82 LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
83 new Abortable() {
84 boolean aborted = false;
85 @Override
86 public boolean isAborted() {
87 return aborted;
88 }
89 @Override
90 public void abort(String why, Throwable e) {
91 aborted = true;
92 }
93 });
94 l.setBalancerOn(false);
95 for (int replicaId = 1; replicaId < 3; replicaId ++) {
96 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
97 replicaId);
98 TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(h);
99 }
100 LOG.debug("All meta replicas assigned");
101 }
102
103 @After
104 public void tearDown() throws Exception {
105 TEST_UTIL.shutdownMiniCluster();
106 }
107
108 @Test
109 public void testMetaHTDReplicaCount() throws Exception {
110 assertTrue(TEST_UTIL.getHBaseAdmin().getTableDescriptor(TableName.META_TABLE_NAME)
111 .getRegionReplication() == 3);
112 }
113
114 @Test
115 public void testZookeeperNodesForReplicas() throws Exception {
116
117 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
118 Configuration conf = TEST_UTIL.getConfiguration();
119 String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
120 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
121 String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
122 conf.get("zookeeper.znode.metaserver", "meta-region-server"));
123
124 byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
125 ServerName.parseFrom(data);
126 for (int i = 1; i < 3; i++) {
127 String secZnode = ZKUtil.joinZNode(baseZNode,
128 conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
129 String str = zkw.getZNodeForReplica(i);
130 assertTrue(str.equals(secZnode));
131
132 data = ZKUtil.getData(zkw, secZnode);
133 ServerName.parseFrom(data);
134 }
135 }
136
137 @Test
138 public void testShutdownHandling() throws Exception {
139
140
141
142
143 shutdownMetaAndDoValidations(TEST_UTIL);
144 }
145
146 public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
147
148
149
150
151 ZooKeeperWatcher zkw = util.getZooKeeperWatcher();
152 Configuration conf = util.getConfiguration();
153 conf.setBoolean(HConstants.USE_META_REPLICAS, true);
154
155 String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
156 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
157 String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
158 conf.get("zookeeper.znode.metaserver", "meta-region-server"));
159 byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
160 ServerName primary = ServerName.parseFrom(data);
161
162 byte[] TABLE = Bytes.toBytes("testShutdownHandling");
163 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
164 if (util.getHBaseAdmin().tableExists(TABLE)) {
165 util.getHBaseAdmin().disableTable(TABLE);
166 util.getHBaseAdmin().deleteTable(TABLE);
167 }
168 Table htable = util.createTable(TABLE, FAMILIES, conf);
169
170 util.getHBaseAdmin().flush(TableName.META_TABLE_NAME);
171 Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
172 30000) * 6);
173 Connection c = ConnectionFactory.createConnection(util.getConfiguration());
174 List<HRegionInfo> regions = MetaTableAccessor.getTableRegions(zkw, c,
175 TableName.valueOf(TABLE));
176 HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
177
178
179
180
181
182 if (hrl.getServerName().equals(primary)) {
183 util.getHBaseAdmin().move(hrl.getRegionInfo().getEncodedNameAsBytes(), null);
184
185 do {
186 Thread.sleep(10);
187 hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
188 } while (primary.equals(hrl.getServerName()));
189 util.getHBaseAdmin().flush(TableName.META_TABLE_NAME);
190 Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
191 30000) * 3);
192 }
193 ServerName master = util.getHBaseClusterInterface().getClusterStatus().getMaster();
194
195
196 util.getHBaseClusterInterface().stopMaster(master);
197 util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
198 if (!master.equals(primary)) {
199 util.getHBaseClusterInterface().killRegionServer(primary);
200 util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
201 }
202 ((ClusterConnection)c).clearRegionCache();
203 htable.close();
204 htable = c.getTable(TableName.valueOf(TABLE));
205 byte[] row = "test".getBytes();
206 Put put = new Put(row);
207 put.add("foo".getBytes(), row, row);
208 BufferedMutator m = c.getBufferedMutator(TableName.valueOf(TABLE));
209 m.mutate(put);
210 m.flush();
211
212 Get get = new Get(row);
213 Result r = htable.get(get);
214 assertTrue(Arrays.equals(r.getRow(), row));
215
216
217 util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
218 util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
219 util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
220 ((ClusterConnection)c).clearRegionCache();
221 htable.close();
222 conf.setBoolean(HConstants.USE_META_REPLICAS, false);
223 htable = c.getTable(TableName.valueOf(TABLE));
224 r = htable.get(get);
225 assertTrue(Arrays.equals(r.getRow(), row));
226 }
227
228 @Test
229 public void testMetaLookupThreadPoolCreated() throws Exception {
230 byte[] TABLE = Bytes.toBytes("testMetaLookupThreadPoolCreated");
231 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
232 if (TEST_UTIL.getHBaseAdmin().tableExists(TABLE)) {
233 TEST_UTIL.getHBaseAdmin().disableTable(TABLE);
234 TEST_UTIL.getHBaseAdmin().deleteTable(TABLE);
235 }
236 Table htable = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration());
237 byte[] row = "test".getBytes();
238 HConnectionImplementation c = ((HConnectionImplementation)((HTable)htable).connection);
239
240 c.relocateRegion(TABLE, row);
241 ExecutorService ex = c.getCurrentMetaLookupPool();
242 assert(ex != null);
243 }
244
245 @Test
246 public void testChangingReplicaCount() throws Exception {
247
248
249 stopMasterAndValidateReplicaCount(3, 2);
250
251 stopMasterAndValidateReplicaCount(2, 3);
252 }
253
254 private void stopMasterAndValidateReplicaCount(int originalReplicaCount, int newReplicaCount)
255 throws Exception {
256 ServerName sn = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
257 TEST_UTIL.getHBaseClusterInterface().stopMaster(sn);
258 TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(sn, 60000);
259 List<String> metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes();
260 assert(metaZnodes.size() == originalReplicaCount);
261 TEST_UTIL.getHBaseClusterInterface().getConf().setInt(HConstants.META_REPLICAS_NUM,
262 newReplicaCount);
263 TEST_UTIL.getHBaseClusterInterface().startMaster(sn.getHostname(), 0);
264 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
265 int count = 0;
266 do {
267 metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes();
268 Thread.sleep(10);
269 count++;
270
271
272
273 } while (metaZnodes.size() == originalReplicaCount && count < 1000);
274 assert(metaZnodes.size() == newReplicaCount);
275
276 TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM,
277 newReplicaCount);
278 HBaseFsck hbck = HbckTestingUtil.doFsck(TEST_UTIL.getConfiguration(), false);
279 HbckTestingUtil.assertNoErrors(hbck);
280 }
281
282 @Test
283 public void testHBaseFsckWithMetaReplicas() throws Exception {
284 HBaseFsck hbck = HbckTestingUtil.doFsck(TEST_UTIL.getConfiguration(), false);
285 HbckTestingUtil.assertNoErrors(hbck);
286 }
287
288 @Test
289 public void testHBaseFsckWithFewerMetaReplicas() throws Exception {
290 ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection(
291 TEST_UTIL.getConfiguration());
292 RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW,
293 false, false);
294 HBaseFsckRepair.closeRegionSilentlyAndWait(c,
295 rl.getRegionLocation(1).getServerName(), rl.getRegionLocation(1).getRegionInfo());
296
297 HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
298 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION});
299
300 hbck = doFsck(TEST_UTIL.getConfiguration(), true);
301
302 hbck = doFsck(TEST_UTIL.getConfiguration(), false);
303 assertErrors(hbck, new ERROR_CODE[]{});
304 }
305
306 @Test
307 public void testHBaseFsckWithFewerMetaReplicaZnodes() throws Exception {
308 ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection(
309 TEST_UTIL.getConfiguration());
310 RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW,
311 false, false);
312 HBaseFsckRepair.closeRegionSilentlyAndWait(c,
313 rl.getRegionLocation(2).getServerName(), rl.getRegionLocation(2).getRegionInfo());
314 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
315 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(2));
316
317 HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
318 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION});
319
320 hbck = doFsck(TEST_UTIL.getConfiguration(), true);
321
322 hbck = doFsck(TEST_UTIL.getConfiguration(), false);
323 assertErrors(hbck, new ERROR_CODE[]{});
324 }
325
326 @Test
327 public void testAccessingUnknownTables() throws Exception {
328 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
329 conf.setBoolean(HConstants.USE_META_REPLICAS, true);
330 Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf("RandomTable"));
331 Get get = new Get(Bytes.toBytes("foo"));
332 try {
333 table.get(get);
334 } catch (TableNotFoundException t) {
335 return;
336 }
337 fail("Expected TableNotFoundException");
338 }
339
340 @Test
341 public void testMetaAddressChange() throws Exception {
342
343
344
345 Configuration conf = TEST_UTIL.getConfiguration();
346 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
347 String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
348 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
349 String primaryMetaZnode = ZKUtil.joinZNode(baseZNode,
350 conf.get("zookeeper.znode.metaserver", "meta-region-server"));
351
352 byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
353 ServerName currentServer = ServerName.parseFrom(data);
354 Collection<ServerName> liveServers = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers();
355 ServerName moveToServer = null;
356 for (ServerName s : liveServers) {
357 if (!currentServer.equals(s)) {
358 moveToServer = s;
359 }
360 }
361 assert(moveToServer != null);
362 String tableName = "randomTable5678";
363 TEST_UTIL.createTable(TableName.valueOf(tableName), "f");
364 assertTrue(TEST_UTIL.getHBaseAdmin().tableExists(tableName));
365 TEST_UTIL.getHBaseAdmin().move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
366 Bytes.toBytes(moveToServer.getServerName()));
367 int i = 0;
368 do {
369 Thread.sleep(10);
370 data = ZKUtil.getData(zkw, primaryMetaZnode);
371 currentServer = ServerName.parseFrom(data);
372 i++;
373 } while (!moveToServer.equals(currentServer) && i < 1000);
374 assert(i != 1000);
375 TEST_UTIL.getHBaseAdmin().disableTable("randomTable5678");
376 assertTrue(TEST_UTIL.getHBaseAdmin().isTableDisabled("randomTable5678"));
377 }
378
379 @Test
380 public void testShutdownOfReplicaHolder() throws Exception {
381
382
383 RegionLocations rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()).
384 locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true);
385 HRegionLocation hrl = rl.getRegionLocation(1);
386 ServerName oldServer = hrl.getServerName();
387 TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
388 int i = 0;
389 do {
390 LOG.debug("Waiting for the replica " + hrl.getRegionInfo() + " to come up");
391 Thread.sleep(30000);
392 rl = ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration()).
393 locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true);
394 hrl = rl.getRegionLocation(1);
395 i++;
396 } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3);
397 assertTrue(i != 3);
398 }
399
400 @Test
401 public void testHBaseFsckWithExcessMetaReplicas() throws Exception {
402 HBaseFsck hbck = new HBaseFsck(TEST_UTIL.getConfiguration());
403
404 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
405 HRegionInfo.FIRST_META_REGIONINFO, 3);
406
407 TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
408 .getRegionStates().createRegionState(h);
409 TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h);
410 HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h);
411
412 hbck = doFsck(TEST_UTIL.getConfiguration(), false);
413 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN, ERROR_CODE.SHOULD_NOT_BE_DEPLOYED});
414
415 hbck = doFsck(TEST_UTIL.getConfiguration(), true);
416
417 hbck = doFsck(TEST_UTIL.getConfiguration(), false);
418 assertErrors(hbck, new ERROR_CODE[]{});
419 }
420
421 @Test
422 public void testMetaTableReplicaAssignment() throws Exception {
423 final ClusterConnection c =
424 ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
425 final RegionLocations rl =
426 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
427
428 final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
429 LOG.debug("The hbase:meta default replica region is in server: " + meta0SN);
430 final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
431 LOG.debug("The hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo() +
432 " is in server: " + meta1SN);
433
434 LOG.debug("Killing the region server " + meta1SN +
435 " that hosts hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo());
436 TEST_UTIL.getHBaseClusterInterface().killRegionServer(meta1SN);
437 TEST_UTIL.getHBaseClusterInterface().waitForRegionServerToStop(meta1SN, 60000);
438
439 ServerName masterSN = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
440 LOG.debug("Killing the master server " + masterSN);
441 TEST_UTIL.getHBaseClusterInterface().stopMaster(masterSN);
442 TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(masterSN, 60000);
443 LOG.debug("Restarting the master server " + masterSN);
444 TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), masterSN.getPort());
445 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
446
447
448 TEST_UTIL.waitFor(60000, 100, new ExplainingPredicate<IOException>() {
449
450 @Override
451 public boolean evaluate() throws IOException {
452 RegionLocations rls =
453 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
454 HRegionLocation loc = rls.getRegionLocation(1);
455 if (loc != null && !meta1SN.equals(loc.getServerName())) {
456 LOG.debug("The hbase:meta replica 1 region " + rls.getRegionLocation(1).getRegionInfo() +
457 " is now moved from server " + meta1SN + " to server " + loc.getServerName());
458 return true;
459 } else {
460 return false;
461 }
462 }
463
464 @Override
465 public String explainFailure() throws IOException {
466 return "The hbase:meta replica 1 region " + rl.getRegionLocation(1).getRegionInfo() +
467 " has not been assigned in time";
468 }
469 });
470
471 LOG.debug("Killing the region server " + meta0SN +
472 " that hosts hbase:meta default replica region " + rl.getRegionLocation(0).getRegionInfo());
473 TEST_UTIL.getHBaseClusterInterface().killRegionServer(meta0SN);
474 TEST_UTIL.getHBaseClusterInterface().waitForRegionServerToStop(meta0SN, 60000);
475
476 TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().waitForAssignment(
477 HRegionInfo.FIRST_META_REGIONINFO);
478
479
480 TEST_UTIL.waitFor(60000, 100, new ExplainingPredicate<IOException>() {
481
482 @Override
483 public boolean evaluate() throws IOException {
484 RegionLocations rls =
485 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, true);
486 HRegionLocation loc = rls.getRegionLocation(0);
487 if (loc != null && !meta0SN.equals(loc.getServerName())) {
488 LOG.debug(
489 "The hbase:meta default replica region " + rls.getRegionLocation(0).getRegionInfo() +
490 " is now moved from server " + meta0SN + " to server " + loc.getServerName());
491 return true;
492 } else {
493 return false;
494 }
495 }
496
497 @Override
498 public String explainFailure() throws IOException {
499 return "The hbase:meta default replica region " + rl.getRegionLocation(0).getRegionInfo() +
500 " has not been assigned in time";
501 }
502 });
503 }
504 }