1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertNotEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.util.List;
26 import java.util.Map;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.MiniHBaseCluster;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableExistsException;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.Connection;
39 import org.apache.hadoop.hbase.client.ConnectionFactory;
40 import org.apache.hadoop.hbase.client.MetaScanner;
41 import org.apache.hadoop.hbase.executor.EventType;
42 import org.apache.hadoop.hbase.testclassification.LargeTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.JVMClusterUtil;
45 import org.apache.hadoop.hbase.util.Threads;
46 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
47 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
48 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
49 import org.junit.After;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53 @Category(LargeTests.class)
54 public class TestRestartCluster {
55 private static final Log LOG = LogFactory.getLog(TestRestartCluster.class);
56 private HBaseTestingUtility UTIL = new HBaseTestingUtility();
57
58 private static final byte[] TABLENAME = Bytes.toBytes("master_transitions");
59 private static final byte [][] FAMILIES = {Bytes.toBytes("a")};
60 private static final TableName[] TABLES = {
61 TableName.valueOf("restartTableOne"),
62 TableName.valueOf("restartTableTwo"),
63 TableName.valueOf("restartTableThree")
64 };
65 private static final byte [] FAMILY = Bytes.toBytes("family");
66
67 @After public void tearDown() throws Exception {
68 UTIL.shutdownMiniCluster();
69 }
70
71 @Test (timeout=300000) public void testRestartClusterAfterKill()
72 throws Exception {
73 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
74 UTIL.startMiniZKCluster();
75 ZooKeeperWatcher zooKeeper =
76 new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null, true);
77
78
79 String unassignedZNode = zooKeeper.assignmentZNode;
80 ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode);
81
82 ServerName sn = ServerName.valueOf(HMaster.MASTER, 1, System.currentTimeMillis());
83
84 ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO, sn);
85
86 LOG.debug("Created UNASSIGNED zNode for ROOT and hbase:meta regions in state " +
87 EventType.M_ZK_REGION_OFFLINE);
88
89
90 LOG.info("Starting HBase cluster...");
91 UTIL.startMiniCluster(2);
92
93 UTIL.createTable(TABLENAME, FAMILIES);
94 LOG.info("Created a table, waiting for table to be available...");
95 UTIL.waitTableAvailable(TABLENAME, 60*1000);
96
97 LOG.info("Master deleted unassigned region and started up successfully.");
98 }
99
100 @Test (timeout=300000)
101 public void testClusterRestart() throws Exception {
102 UTIL.startMiniCluster(3);
103 Connection connection = UTIL.getConnection();
104
105 while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
106 Threads.sleep(1);
107 }
108 LOG.info("\n\nCreating tables");
109 for(TableName TABLE : TABLES) {
110 UTIL.createTable(TABLE, FAMILY);
111 }
112 for(TableName TABLE : TABLES) {
113 UTIL.waitTableEnabled(TABLE);
114 }
115
116 List<HRegionInfo> allRegions =
117 MetaScanner.listAllRegions(UTIL.getConfiguration(), connection, true);
118 assertTrue(4 <= allRegions.size());
119
120 LOG.info("\n\nShutting down cluster");
121 UTIL.shutdownMiniHBaseCluster();
122
123 LOG.info("\n\nSleeping a bit");
124 Thread.sleep(2000);
125
126 LOG.info("\n\nStarting cluster the second time");
127 UTIL.restartHBaseCluster(3);
128
129
130
131
132 allRegions =
133 MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), connection, true);
134 assertTrue(4 <= allRegions.size());
135 LOG.info("\n\nWaiting for tables to be available");
136 for(TableName TABLE: TABLES) {
137 try {
138 UTIL.createTable(TABLE, FAMILY);
139 assertTrue("Able to create table that should already exist", false);
140 } catch(TableExistsException tee) {
141 LOG.info("Table already exists as expected");
142 }
143 UTIL.waitTableAvailable(TABLE);
144 }
145 }
146
147
148
149
150 @Test (timeout=300000)
151 public void testRetainAssignmentOnRestart() throws Exception {
152 UTIL.startMiniCluster(2);
153 while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
154 Threads.sleep(1);
155 }
156
157 UTIL.getMiniHBaseCluster().getMaster().
158 getMasterRpcServices().synchronousBalanceSwitch(false);
159 LOG.info("\n\nCreating tables");
160 for(TableName TABLE : TABLES) {
161 UTIL.createTable(TABLE, FAMILY);
162 }
163 for(TableName TABLE : TABLES) {
164 UTIL.waitTableEnabled(TABLE);
165 }
166
167 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
168 AssignmentManager am = master.getAssignmentManager();
169 am.waitUntilNoRegionsInTransition(120000);
170
171
172
173 SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
174 master.getConnection());
175 snapshot.initialize();
176 Map<HRegionInfo, ServerName> regionToRegionServerMap
177 = snapshot.getRegionToRegionServerMap();
178
179 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
180 List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
181 assertEquals(2, threads.size());
182 int[] rsPorts = new int[2];
183 for (int i = 0; i < 2; i++) {
184 rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
185 }
186 for (ServerName serverName: regionToRegionServerMap.values()) {
187 boolean found = false;
188 for (int k = 0; k < 2 && !found; k++) {
189 found = serverName.getPort() == rsPorts[k];
190 }
191 assertTrue(found);
192 }
193
194 LOG.info("\n\nShutting down HBase cluster");
195 cluster.shutdown();
196 cluster.waitUntilShutDown();
197
198 LOG.info("\n\nSleeping a bit");
199 Thread.sleep(2000);
200
201 LOG.info("\n\nStarting cluster the second time with the same ports");
202 try {
203 cluster.getConf().setInt(
204 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
205 master = cluster.startMaster().getMaster();
206 for (int i = 0; i < 2; i++) {
207 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
208 cluster.startRegionServer();
209 }
210 } finally {
211
212 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
213 cluster.getConf().setInt(
214 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
215 }
216
217
218 List<ServerName> localServers = master.getServerManager().getOnlineServersList();
219 assertEquals(2, localServers.size());
220 for (int i = 0; i < 2; i++) {
221 boolean found = false;
222 for (ServerName serverName: localServers) {
223 if (serverName.getPort() == rsPorts[i]) {
224 found = true;
225 break;
226 }
227 }
228 assertTrue(found);
229 }
230
231
232 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
233 int expectedRegions = regionToRegionServerMap.size() + 1;
234 while (!master.isInitialized()
235 || regionStates.getRegionAssignments().size() != expectedRegions) {
236 Threads.sleep(100);
237 }
238
239 snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection());
240 snapshot.initialize();
241 Map<HRegionInfo, ServerName> newRegionToRegionServerMap =
242 snapshot.getRegionToRegionServerMap();
243 assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
244 for (Map.Entry<HRegionInfo, ServerName> entry: newRegionToRegionServerMap.entrySet()) {
245 if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
246 ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
247 ServerName currentServer = entry.getValue();
248 assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
249 assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
250 }
251 }
252 }