View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
25  import org.apache.hadoop.hbase.executor.EventType;
26  import org.apache.hadoop.hbase.executor.ExecutorService;
27  import org.apache.hadoop.hbase.executor.ExecutorType;
28  import org.apache.hadoop.hbase.master.AssignmentManager;
29  import org.apache.hadoop.hbase.master.HMaster;
30  import org.apache.hadoop.hbase.master.LoadBalancer;
31  import org.apache.hadoop.hbase.master.MasterServices;
32  import org.apache.hadoop.hbase.master.RegionPlan;
33  import org.apache.hadoop.hbase.master.RegionState;
34  import org.apache.hadoop.hbase.master.ServerManager;
35  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
36  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
39  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.mockito.Mockito;
45  
46  import java.util.ArrayList;
47  import java.util.HashMap;
48  import java.util.HashSet;
49  import java.util.List;
50  import java.util.Map;
51  import java.util.Map.Entry;
52  import java.util.Set;
53  
54  import static org.junit.Assert.assertFalse;
55  import static org.junit.Assert.assertNotEquals;
56  import static org.junit.Assert.assertTrue;
57  
58  
59  /**
60   * Test the draining servers feature.
61   */
62  @Category(MediumTests.class)
63  public class TestDrainingServer {
64    private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
65    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private Abortable abortable = new Abortable() {
67      @Override
68      public boolean isAborted() {
69        return false;
70      }
71  
72      @Override
73      public void abort(String why, Throwable e) {
74      }
75    };
76  
77    @AfterClass
78    public static void afterClass() throws Exception {
79      TEST_UTIL.shutdownMiniZKCluster();
80    }
81  
82    @BeforeClass
83    public static void beforeClass() throws Exception {
84      TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
85      TEST_UTIL.startMiniZKCluster();
86    }
87  
88    @Test
89    public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
90      AssignmentManager am;
91      Configuration conf = TEST_UTIL.getConfiguration();
92      final HMaster master = Mockito.mock(HMaster.class);
93      final MasterServices server = Mockito.mock(MasterServices.class);
94      final ServerManager serverManager = Mockito.mock(ServerManager.class);
95      final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
96      final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
97      LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
98      final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
99          HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
100 
101     ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
102       "zkWatcher-Test", abortable, true);
103 
104     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
105 
106     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
107     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
108 
109     Mockito.when(server.getConfiguration()).thenReturn(conf);
110     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
111     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
112     Mockito.when(server.getRegionServerVersion(Mockito.any(ServerName.class))).thenReturn("0.0.0");
113 
114     CoordinatedStateManager cp = new ZkCoordinatedStateManager();
115     cp.initialize(server);
116     cp.start();
117 
118     Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
119 
120     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
121     Mockito.when(serverManager.getOnlineServersList())
122     .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
123 
124     Mockito.when(serverManager.createDestinationServersList())
125         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
126     Mockito.when(serverManager.createDestinationServersList(null))
127         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
128     Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
129         new ArrayList<ServerName>(onlineServers.keySet()));
130 
131     for (ServerName sn : onlineServers.keySet()) {
132       Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
133       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
134       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
135       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
136       .thenReturn(RegionOpeningState.OPENED);
137       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
138       .thenReturn(RegionOpeningState.OPENED);
139       Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
140     }
141 
142     Mockito.when(master.getServerManager()).thenReturn(serverManager);
143 
144     am = new AssignmentManager(server, serverManager,
145         balancer, startupMasterExecutor("mockExecutorService"), null, null);
146 
147     Mockito.when(master.getAssignmentManager()).thenReturn(am);
148     Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
149 
150     am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
151 
152     zkWatcher.registerListenerFirst(am);
153 
154     addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
155 
156     am.assign(REGIONINFO, true);
157 
158     setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
159     setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
160 
161     am.waitForAssignment(REGIONINFO);
162 
163     assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
164     assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
165   }
166 
167   @Test
168   public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
169     Configuration conf = TEST_UTIL.getConfiguration();
170     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
171     AssignmentManager am;
172     final HMaster master = Mockito.mock(HMaster.class);
173     final MasterServices server = Mockito.mock(MasterServices.class);
174     final ServerManager serverManager = Mockito.mock(ServerManager.class);
175     final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
176     final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
177     final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
178     final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
179     final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
180     final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
181 
182     Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
183 
184     HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
185         HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
186     HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
187       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
188     HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
189       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
190     HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
191       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
192     HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
193       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
194 
195     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
196     List<ServerName> drainedServers = new ArrayList<ServerName>();
197 
198     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
199     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
200     onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
201     onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
202     onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
203 
204     bulk.put(REGIONINFO_A, SERVERNAME_A);
205     bulk.put(REGIONINFO_B, SERVERNAME_B);
206     bulk.put(REGIONINFO_C, SERVERNAME_C);
207     bulk.put(REGIONINFO_D, SERVERNAME_D);
208     bulk.put(REGIONINFO_E, SERVERNAME_E);
209 
210     ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
211         "zkWatcher-BulkAssignTest", abortable, true);
212 
213     Mockito.when(server.getConfiguration()).thenReturn(conf);
214     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
215     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
216 
217     CoordinatedStateManager cp = new ZkCoordinatedStateManager();
218     cp.initialize(server);
219     cp.start();
220 
221     Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
222 
223     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
224     Mockito.when(serverManager.getOnlineServersList()).thenReturn(
225       new ArrayList<ServerName>(onlineServers.keySet()));
226 
227     Mockito.when(serverManager.createDestinationServersList()).thenReturn(
228       new ArrayList<ServerName>(onlineServers.keySet()));
229     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
230       new ArrayList<ServerName>(onlineServers.keySet()));
231     Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
232         new ArrayList<ServerName>(onlineServers.keySet()));
233 
234     for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
235       Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
236       Mockito.when(serverManager.sendRegionClose(entry.getValue(),
237         entry.getKey(), -1)).thenReturn(true);
238       Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
239         entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
240       Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
241     }
242 
243     Mockito.when(master.getServerManager()).thenReturn(serverManager);
244 
245     drainedServers.add(SERVERNAME_A);
246     drainedServers.add(SERVERNAME_B);
247     drainedServers.add(SERVERNAME_C);
248     drainedServers.add(SERVERNAME_D);
249 
250     am = new AssignmentManager(server, serverManager,
251       balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
252 
253     Mockito.when(master.getAssignmentManager()).thenReturn(am);
254 
255     zkWatcher.registerListener(am);
256 
257     for (ServerName drained : drainedServers) {
258       addServerToDrainedList(drained, onlineServers, serverManager);
259     }
260 
261     am.assign(bulk);
262 
263     Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
264     for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
265       setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
266         entry.getValue().getRegion());
267     }
268 
269     am.waitForAssignment(REGIONINFO_A);
270     am.waitForAssignment(REGIONINFO_B);
271     am.waitForAssignment(REGIONINFO_C);
272     am.waitForAssignment(REGIONINFO_D);
273     am.waitForAssignment(REGIONINFO_E);
274 
275     Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
276     for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
277       LOG.info("Region Assignment: "
278           + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
279       bunchServersAssigned.add(entry.getValue());
280     }
281 
282     for (ServerName sn : drainedServers) {
283       assertFalse(bunchServersAssigned.contains(sn));
284     }
285   }
286 
287   private void addServerToDrainedList(ServerName serverName,
288       Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
289     onlineServers.remove(serverName);
290     List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
291     Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
292     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
293     Mockito.when(serverManager.createDestinationServersList(Mockito.anyList())).thenReturn(
294         new ArrayList<ServerName>(onlineServers.keySet()));
295   }
296 
297   private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
298                                    HRegionInfo hregionInfo) throws Exception {
299     int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
300     int versionTransition = ZKAssign.transitionNode(zkWatcher,
301         hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
302         EventType.RS_ZK_REGION_OPENING, version);
303     ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
304   }
305 
306   private ExecutorService startupMasterExecutor(final String name) {
307     ExecutorService executor = new ExecutorService(name);
308     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
309     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
310     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
311     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
312     return executor;
313   }
314 }