1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
25 import org.apache.hadoop.hbase.executor.EventType;
26 import org.apache.hadoop.hbase.executor.ExecutorService;
27 import org.apache.hadoop.hbase.executor.ExecutorType;
28 import org.apache.hadoop.hbase.master.AssignmentManager;
29 import org.apache.hadoop.hbase.master.HMaster;
30 import org.apache.hadoop.hbase.master.LoadBalancer;
31 import org.apache.hadoop.hbase.master.MasterServices;
32 import org.apache.hadoop.hbase.master.RegionPlan;
33 import org.apache.hadoop.hbase.master.RegionState;
34 import org.apache.hadoop.hbase.master.ServerManager;
35 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
36 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
37 import org.apache.hadoop.hbase.testclassification.MediumTests;
38 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
39 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40 import org.junit.AfterClass;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.mockito.Mockito;
45
46 import java.util.ArrayList;
47 import java.util.HashMap;
48 import java.util.HashSet;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.Map.Entry;
52 import java.util.Set;
53
54 import static org.junit.Assert.assertFalse;
55 import static org.junit.Assert.assertNotEquals;
56 import static org.junit.Assert.assertTrue;
57
58
59
60
61
62 @Category(MediumTests.class)
63 public class TestDrainingServer {
64 private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
65 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private Abortable abortable = new Abortable() {
67 @Override
68 public boolean isAborted() {
69 return false;
70 }
71
72 @Override
73 public void abort(String why, Throwable e) {
74 }
75 };
76
77 @AfterClass
78 public static void afterClass() throws Exception {
79 TEST_UTIL.shutdownMiniZKCluster();
80 }
81
82 @BeforeClass
83 public static void beforeClass() throws Exception {
84 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
85 TEST_UTIL.startMiniZKCluster();
86 }
87
88 @Test
89 public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
90 AssignmentManager am;
91 Configuration conf = TEST_UTIL.getConfiguration();
92 final HMaster master = Mockito.mock(HMaster.class);
93 final MasterServices server = Mockito.mock(MasterServices.class);
94 final ServerManager serverManager = Mockito.mock(ServerManager.class);
95 final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
96 final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
97 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
98 final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
99 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
100
101 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
102 "zkWatcher-Test", abortable, true);
103
104 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
105
106 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
107 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
108
109 Mockito.when(server.getConfiguration()).thenReturn(conf);
110 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
111 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
112 Mockito.when(server.getRegionServerVersion(Mockito.any(ServerName.class))).thenReturn("0.0.0");
113
114 CoordinatedStateManager cp = new ZkCoordinatedStateManager();
115 cp.initialize(server);
116 cp.start();
117
118 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
119
120 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
121 Mockito.when(serverManager.getOnlineServersList())
122 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
123
124 Mockito.when(serverManager.createDestinationServersList())
125 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
126 Mockito.when(serverManager.createDestinationServersList(null))
127 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
128 Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
129 new ArrayList<ServerName>(onlineServers.keySet()));
130
131 for (ServerName sn : onlineServers.keySet()) {
132 Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
133 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
134 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
135 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
136 .thenReturn(RegionOpeningState.OPENED);
137 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
138 .thenReturn(RegionOpeningState.OPENED);
139 Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
140 }
141
142 Mockito.when(master.getServerManager()).thenReturn(serverManager);
143
144 am = new AssignmentManager(server, serverManager,
145 balancer, startupMasterExecutor("mockExecutorService"), null, null);
146
147 Mockito.when(master.getAssignmentManager()).thenReturn(am);
148 Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
149
150 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
151
152 zkWatcher.registerListenerFirst(am);
153
154 addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
155
156 am.assign(REGIONINFO, true);
157
158 setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
159 setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
160
161 am.waitForAssignment(REGIONINFO);
162
163 assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
164 assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
165 }
166
167 @Test
168 public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
169 Configuration conf = TEST_UTIL.getConfiguration();
170 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
171 AssignmentManager am;
172 final HMaster master = Mockito.mock(HMaster.class);
173 final MasterServices server = Mockito.mock(MasterServices.class);
174 final ServerManager serverManager = Mockito.mock(ServerManager.class);
175 final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
176 final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
177 final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
178 final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
179 final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
180 final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
181
182 Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
183
184 HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
185 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
186 HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
187 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
188 HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
189 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
190 HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
191 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
192 HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
193 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
194
195 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
196 List<ServerName> drainedServers = new ArrayList<ServerName>();
197
198 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
199 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
200 onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
201 onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
202 onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
203
204 bulk.put(REGIONINFO_A, SERVERNAME_A);
205 bulk.put(REGIONINFO_B, SERVERNAME_B);
206 bulk.put(REGIONINFO_C, SERVERNAME_C);
207 bulk.put(REGIONINFO_D, SERVERNAME_D);
208 bulk.put(REGIONINFO_E, SERVERNAME_E);
209
210 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
211 "zkWatcher-BulkAssignTest", abortable, true);
212
213 Mockito.when(server.getConfiguration()).thenReturn(conf);
214 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
215 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
216
217 CoordinatedStateManager cp = new ZkCoordinatedStateManager();
218 cp.initialize(server);
219 cp.start();
220
221 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
222
223 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
224 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
225 new ArrayList<ServerName>(onlineServers.keySet()));
226
227 Mockito.when(serverManager.createDestinationServersList()).thenReturn(
228 new ArrayList<ServerName>(onlineServers.keySet()));
229 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
230 new ArrayList<ServerName>(onlineServers.keySet()));
231 Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
232 new ArrayList<ServerName>(onlineServers.keySet()));
233
234 for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
235 Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
236 Mockito.when(serverManager.sendRegionClose(entry.getValue(),
237 entry.getKey(), -1)).thenReturn(true);
238 Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
239 entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
240 Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
241 }
242
243 Mockito.when(master.getServerManager()).thenReturn(serverManager);
244
245 drainedServers.add(SERVERNAME_A);
246 drainedServers.add(SERVERNAME_B);
247 drainedServers.add(SERVERNAME_C);
248 drainedServers.add(SERVERNAME_D);
249
250 am = new AssignmentManager(server, serverManager,
251 balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
252
253 Mockito.when(master.getAssignmentManager()).thenReturn(am);
254
255 zkWatcher.registerListener(am);
256
257 for (ServerName drained : drainedServers) {
258 addServerToDrainedList(drained, onlineServers, serverManager);
259 }
260
261 am.assign(bulk);
262
263 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
264 for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
265 setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
266 entry.getValue().getRegion());
267 }
268
269 am.waitForAssignment(REGIONINFO_A);
270 am.waitForAssignment(REGIONINFO_B);
271 am.waitForAssignment(REGIONINFO_C);
272 am.waitForAssignment(REGIONINFO_D);
273 am.waitForAssignment(REGIONINFO_E);
274
275 Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
276 for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
277 LOG.info("Region Assignment: "
278 + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
279 bunchServersAssigned.add(entry.getValue());
280 }
281
282 for (ServerName sn : drainedServers) {
283 assertFalse(bunchServersAssigned.contains(sn));
284 }
285 }
286
287 private void addServerToDrainedList(ServerName serverName,
288 Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
289 onlineServers.remove(serverName);
290 List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
291 Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
292 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
293 Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
294 new ArrayList<ServerName>(onlineServers.keySet()));
295 }
296
297 private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
298 HRegionInfo hregionInfo) throws Exception {
299 int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
300 int versionTransition = ZKAssign.transitionNode(zkWatcher,
301 hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
302 EventType.RS_ZK_REGION_OPENING, version);
303 ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
304 }
305
306 private ExecutorService startupMasterExecutor(final String name) {
307 ExecutorService executor = new ExecutorService(name);
308 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
309 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
310 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
311 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
312 return executor;
313 }
314 }