View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.mockito.Mockito.mock;
23  import static org.mockito.Mockito.when;
24  
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.LinkedHashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeMap;
32  import java.util.TreeSet;
33  
34  import org.apache.commons.lang.ArrayUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HBaseIOException;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
45  import org.apache.hadoop.hbase.master.LoadBalancer;
46  import org.apache.hadoop.hbase.master.MasterServices;
47  import org.apache.hadoop.hbase.master.RackManager;
48  import org.apache.hadoop.hbase.master.RegionPlan;
49  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
50  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction;
51  import org.apache.hadoop.net.DNSToSwitchMapping;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  import org.mockito.Mockito;
56  
57  import com.google.common.collect.Lists;
58  
59  @Category(MediumTests.class)
60  public class TestBaseLoadBalancer extends BalancerTestBase {
61  
62    private static LoadBalancer loadBalancer;
63    private static final Log LOG = LogFactory.getLog(TestBaseLoadBalancer.class);
64    private static final ServerName master = ServerName.valueOf("fake-master", 0, 1L);
65    private static RackManager rackManager;
66    private static final int NUM_SERVERS = 15;
67    private static ServerName[] servers = new ServerName[NUM_SERVERS];
68  
69    int[][] regionsAndServersMocks = new int[][] {
70        // { num regions, num servers }
71        new int[] { 0, 0 }, new int[] { 0, 1 }, new int[] { 1, 1 }, new int[] { 2, 1 },
72        new int[] { 10, 1 }, new int[] { 1, 2 }, new int[] { 2, 2 }, new int[] { 3, 2 },
73        new int[] { 1, 3 }, new int[] { 2, 3 }, new int[] { 3, 3 }, new int[] { 25, 3 },
74        new int[] { 2, 10 }, new int[] { 2, 100 }, new int[] { 12, 10 }, new int[] { 12, 100 }, };
75  
76    @BeforeClass
77    public static void beforeAllTests() throws Exception {
78      Configuration conf = HBaseConfiguration.create();
79      conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
80      loadBalancer = new MockBalancer();
81      loadBalancer.setConf(conf);
82      MasterServices st = Mockito.mock(MasterServices.class);
83      Mockito.when(st.getServerName()).thenReturn(master);
84      loadBalancer.setMasterServices(st);
85  
86      // Set up the rack topologies (5 machines per rack)
87      rackManager = Mockito.mock(RackManager.class);
88      for (int i = 0; i < NUM_SERVERS; i++) {
89        servers[i] = ServerName.valueOf("foo"+i+":1234",-1);
90        if (i < 5) {
91          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack1");
92        }
93        if (i >= 5 && i < 10) {
94          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack2");
95        }
96        if (i >= 10) {
97          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack3");
98        }
99      }
100   }
101 
102   public static class MockBalancer extends BaseLoadBalancer {
103 
104     @Override
105     public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
106       return null;
107     }
108 
109     @Override
110     public List<RegionPlan> balanceCluster(TableName tableName,
111         Map<ServerName, List<HRegionInfo>> clusterState) throws HBaseIOException {
112       return null;
113     }
114 
115   }
116 
117   /**
118    * Tests immediate assignment.
119    *
120    * Invariant is that all regions have an assignment.
121    *
122    * @throws Exception
123    */
124   @Test (timeout=30000)
125   public void testImmediateAssignment() throws Exception {
126     for (int[] mock : regionsAndServersMocks) {
127       LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
128       List<HRegionInfo> regions = randomRegions(mock[0]);
129       List<ServerAndLoad> servers = randomServers(mock[1], 0);
130       List<ServerName> list = getListOfServerNames(servers);
131       Map<HRegionInfo, ServerName> assignments = loadBalancer.immediateAssignment(regions, list);
132       assertImmediateAssignment(regions, list, assignments);
133       returnRegions(regions);
134       returnServers(list);
135     }
136   }
137 
138   /**
139    * All regions have an assignment.
140    * @param regions
141    * @param servers
142    * @param assignments
143    */
144   private void assertImmediateAssignment(List<HRegionInfo> regions, List<ServerName> servers,
145       Map<HRegionInfo, ServerName> assignments) {
146     for (HRegionInfo region : regions) {
147       assertTrue(assignments.containsKey(region));
148     }
149   }
150 
151   /**
152    * Tests the bulk assignment used during cluster startup.
153    *
154    * Round-robin. Should yield a balanced cluster so same invariant as the load
155    * balancer holds, all servers holding either floor(avg) or ceiling(avg).
156    *
157    * @throws Exception
158    */
159   @Test (timeout=180000)
160   public void testBulkAssignment() throws Exception {
161     for (int[] mock : regionsAndServersMocks) {
162       LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
163       List<HRegionInfo> regions = randomRegions(mock[0]);
164       List<ServerAndLoad> servers = randomServers(mock[1], 0);
165       List<ServerName> list = getListOfServerNames(servers);
166       Map<ServerName, List<HRegionInfo>> assignments =
167           loadBalancer.roundRobinAssignment(regions, list);
168       float average = (float) regions.size() / servers.size();
169       int min = (int) Math.floor(average);
170       int max = (int) Math.ceil(average);
171       if (assignments != null && !assignments.isEmpty()) {
172         for (List<HRegionInfo> regionList : assignments.values()) {
173           assertTrue(regionList.size() == min || regionList.size() == max);
174         }
175       }
176       returnRegions(regions);
177       returnServers(list);
178     }
179   }
180 
181   /**
182    * Test the cluster startup bulk assignment which attempts to retain
183    * assignment info.
184    * @throws Exception
185    */
186   @Test (timeout=180000)
187   public void testRetainAssignment() throws Exception {
188     // Test simple case where all same servers are there
189     List<ServerAndLoad> servers = randomServers(10, 10);
190     List<HRegionInfo> regions = randomRegions(100);
191     Map<HRegionInfo, ServerName> existing = new TreeMap<HRegionInfo, ServerName>();
192     for (int i = 0; i < regions.size(); i++) {
193       ServerName sn = servers.get(i % servers.size()).getServerName();
194       // The old server would have had same host and port, but different
195       // start code!
196       ServerName snWithOldStartCode =
197           ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10);
198       existing.put(regions.get(i), snWithOldStartCode);
199     }
200     List<ServerName> listOfServerNames = getListOfServerNames(servers);
201     Map<ServerName, List<HRegionInfo>> assignment =
202         loadBalancer.retainAssignment(existing, listOfServerNames);
203     assertRetainedAssignment(existing, listOfServerNames, assignment);
204 
205     // Include two new servers that were not there before
206     List<ServerAndLoad> servers2 = new ArrayList<ServerAndLoad>(servers);
207     servers2.add(randomServer(10));
208     servers2.add(randomServer(10));
209     listOfServerNames = getListOfServerNames(servers2);
210     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
211     assertRetainedAssignment(existing, listOfServerNames, assignment);
212 
213     // Remove two of the servers that were previously there
214     List<ServerAndLoad> servers3 = new ArrayList<ServerAndLoad>(servers);
215     servers3.remove(0);
216     servers3.remove(0);
217     listOfServerNames = getListOfServerNames(servers3);
218     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
219     assertRetainedAssignment(existing, listOfServerNames, assignment);
220   }
221 
222   @Test (timeout=180000)
223   public void testRegionAvailability() throws Exception {
224     // Create a cluster with a few servers, assign them to specific racks
225     // then assign some regions. The tests should check whether moving a
226     // replica from one node to a specific other node or rack lowers the
227     // availability of the region or not
228 
229     List<HRegionInfo> list0 = new ArrayList<HRegionInfo>();
230     List<HRegionInfo> list1 = new ArrayList<HRegionInfo>();
231     List<HRegionInfo> list2 = new ArrayList<HRegionInfo>();
232     // create a region (region1)
233     HRegionInfo hri1 = new HRegionInfo(
234         TableName.valueOf("table"), "key1".getBytes(), "key2".getBytes(),
235         false, 100);
236     // create a replica of the region (replica_of_region1)
237     HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
238     // create a second region (region2)
239     HRegionInfo hri3 = new HRegionInfo(
240         TableName.valueOf("table"), "key2".getBytes(), "key3".getBytes(),
241         false, 101);
242     list0.add(hri1); //only region1
243     list1.add(hri2); //only replica_of_region1
244     list2.add(hri3); //only region2
245     Map<ServerName, List<HRegionInfo>> clusterState =
246         new LinkedHashMap<ServerName, List<HRegionInfo>>();
247     clusterState.put(servers[0], list0); //servers[0] hosts region1
248     clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
249     clusterState.put(servers[2], list2); //servers[2] hosts region2
250     // create a cluster with the above clusterState. The way in which the
251     // cluster is created (constructor code) would make sure the indices of
252     // the servers are in the order in which it is inserted in the clusterState
253     // map (linkedhashmap is important). A similar thing applies to the region lists
254     Cluster cluster = new Cluster(clusterState, null, null, rackManager);
255     // check whether a move of region1 from servers[0] to servers[1] would lower
256     // the availability of region1
257     assertTrue(cluster.wouldLowerAvailability(hri1, servers[1]));
258     // check whether a move of region1 from servers[0] to servers[2] would lower
259     // the availability of region1
260     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
261     // check whether a move of replica_of_region1 from servers[0] to servers[2] would lower
262     // the availability of replica_of_region1
263     assertTrue(!cluster.wouldLowerAvailability(hri2, servers[2]));
264     // check whether a move of region2 from servers[0] to servers[1] would lower
265     // the availability of region2
266     assertTrue(!cluster.wouldLowerAvailability(hri3, servers[1]));
267 
268     // now lets have servers[1] host replica_of_region2
269     list1.add(RegionReplicaUtil.getRegionInfoForReplica(hri3, 1));
270     // create a new clusterState with the above change
271     cluster = new Cluster(clusterState, null, null, rackManager);
272     // now check whether a move of a replica from servers[0] to servers[1] would lower
273     // the availability of region2
274     assertTrue(cluster.wouldLowerAvailability(hri3, servers[1]));
275 
276     // start over again
277     clusterState.clear();
278     clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
279     clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1 and replica_of_region2
280     clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
281     clusterState.put(servers[10], new ArrayList<HRegionInfo>()); //servers[10], rack3 hosts no region
282     // create a cluster with the above clusterState
283     cluster = new Cluster(clusterState, null, null, rackManager);
284     // check whether a move of region1 from servers[0],rack1 to servers[6],rack2 would
285     // lower the availability
286 
287     assertTrue(cluster.wouldLowerAvailability(hri1, servers[0]));
288 
289     // now create a cluster without the rack manager
290     cluster = new Cluster(clusterState, null, null, null);
291     // now repeat check whether a move of region1 from servers[0] to servers[6] would
292     // lower the availability
293     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[6]));
294   }
295 
296   @Test (timeout=180000)
297   public void testRegionAvailabilityWithRegionMoves() throws Exception {
298     List<HRegionInfo> list0 = new ArrayList<HRegionInfo>();
299     List<HRegionInfo> list1 = new ArrayList<HRegionInfo>();
300     List<HRegionInfo> list2 = new ArrayList<HRegionInfo>();
301     // create a region (region1)
302     HRegionInfo hri1 = new HRegionInfo(
303         TableName.valueOf("table"), "key1".getBytes(), "key2".getBytes(),
304         false, 100);
305     // create a replica of the region (replica_of_region1)
306     HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
307     // create a second region (region2)
308     HRegionInfo hri3 = new HRegionInfo(
309         TableName.valueOf("table"), "key2".getBytes(), "key3".getBytes(),
310         false, 101);
311     list0.add(hri1); //only region1
312     list1.add(hri2); //only replica_of_region1
313     list2.add(hri3); //only region2
314     Map<ServerName, List<HRegionInfo>> clusterState =
315         new LinkedHashMap<ServerName, List<HRegionInfo>>();
316     clusterState.put(servers[0], list0); //servers[0] hosts region1
317     clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
318     clusterState.put(servers[2], list2); //servers[2] hosts region2
319     // create a cluster with the above clusterState. The way in which the
320     // cluster is created (constructor code) would make sure the indices of
321     // the servers are in the order in which it is inserted in the clusterState
322     // map (linkedhashmap is important).
323     Cluster cluster = new Cluster(clusterState, null, null, rackManager);
324     // check whether moving region1 from servers[1] to servers[2] would lower availability
325     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
326 
327     // now move region1 from servers[0] to servers[2]
328     cluster.doAction(new MoveRegionAction(0, 0, 2));
329     // now repeat check whether moving region1 from servers[1] to servers[2]
330     // would lower availability
331     assertTrue(cluster.wouldLowerAvailability(hri1, servers[2]));
332 
333     // start over again
334     clusterState.clear();
335     List<HRegionInfo> list3 = new ArrayList<HRegionInfo>();
336     HRegionInfo hri4 = RegionReplicaUtil.getRegionInfoForReplica(hri3, 1);
337     list3.add(hri4);
338     clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
339     clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1
340     clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
341     clusterState.put(servers[12], list3); //servers[12], rack3 hosts replica_of_region2
342     // create a cluster with the above clusterState
343     cluster = new Cluster(clusterState, null, null, rackManager);
344     // check whether a move of replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
345     // lower the availability
346     assertTrue(!cluster.wouldLowerAvailability(hri4, servers[0]));
347     // now move region2 from servers[6],rack2 to servers[0],rack1
348     cluster.doAction(new MoveRegionAction(2, 2, 0));
349     // now repeat check if replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
350     // lower the availability
351     assertTrue(cluster.wouldLowerAvailability(hri3, servers[0]));
352   }
353 
354   private List<ServerName> getListOfServerNames(final List<ServerAndLoad> sals) {
355     List<ServerName> list = new ArrayList<ServerName>();
356     for (ServerAndLoad e : sals) {
357       list.add(e.getServerName());
358     }
359     return list;
360   }
361 
362   /**
363    * Asserts a valid retained assignment plan.
364    * <p>
365    * Must meet the following conditions:
366    * <ul>
367    * <li>Every input region has an assignment, and to an online server
368    * <li>If a region had an existing assignment to a server with the same
369    * address a a currently online server, it will be assigned to it
370    * </ul>
371    * @param existing
372    * @param servers
373    * @param assignment
374    */
375   private void assertRetainedAssignment(Map<HRegionInfo, ServerName> existing,
376       List<ServerName> servers, Map<ServerName, List<HRegionInfo>> assignment) {
377     // Verify condition 1, every region assigned, and to online server
378     Set<ServerName> onlineServerSet = new TreeSet<ServerName>(servers);
379     Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
380     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
381       assertTrue("Region assigned to server that was not listed as online",
382         onlineServerSet.contains(a.getKey()));
383       for (HRegionInfo r : a.getValue())
384         assignedRegions.add(r);
385     }
386     assertEquals(existing.size(), assignedRegions.size());
387 
388     // Verify condition 2, if server had existing assignment, must have same
389     Set<String> onlineHostNames = new TreeSet<String>();
390     for (ServerName s : servers) {
391       onlineHostNames.add(s.getHostname());
392     }
393 
394     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
395       ServerName assignedTo = a.getKey();
396       for (HRegionInfo r : a.getValue()) {
397         ServerName address = existing.get(r);
398         if (address != null && onlineHostNames.contains(address.getHostname())) {
399           // this region was prevously assigned somewhere, and that
400           // host is still around, then it should be re-assigned on the
401           // same host
402           assertEquals(address.getHostname(), assignedTo.getHostname());
403         }
404       }
405     }
406   }
407 
408   @Test (timeout=180000)
409   public void testClusterServersWithSameHostPort() {
410     // tests whether the BaseLoadBalancer.Cluster can be constructed with servers
411     // sharing same host and port
412     List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
413     List<HRegionInfo> regions = randomRegions(101);
414     Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName, List<HRegionInfo>>();
415 
416     assignRegions(regions, servers, clusterState);
417 
418     // construct another list of servers, but sharing same hosts and ports
419     List<ServerName> oldServers = new ArrayList<ServerName>(servers.size());
420     for (ServerName sn : servers) {
421       // The old server would have had same host and port, but different start code!
422       oldServers.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10));
423     }
424 
425     regions = randomRegions(9); // some more regions
426     assignRegions(regions, oldServers, clusterState);
427 
428     // should not throw exception:
429     BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, null, null);
430     assertEquals(101 + 9, cluster.numRegions);
431     assertEquals(10, cluster.numServers); // only 10 servers because they share the same host + port
432   }
433 
434   private void assignRegions(List<HRegionInfo> regions, List<ServerName> servers,
435       Map<ServerName, List<HRegionInfo>> clusterState) {
436     for (int i = 0; i < regions.size(); i++) {
437       ServerName sn = servers.get(i % servers.size());
438       List<HRegionInfo> regionsOfServer = clusterState.get(sn);
439       if (regionsOfServer == null) {
440         regionsOfServer = new ArrayList<HRegionInfo>(10);
441         clusterState.put(sn, regionsOfServer);
442       }
443 
444       regionsOfServer.add(regions.get(i));
445     }
446   }
447 
448   @Test (timeout=180000)
449   public void testClusterRegionLocations() {
450     // tests whether region locations are handled correctly in Cluster
451     List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
452     List<HRegionInfo> regions = randomRegions(101);
453     Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName, List<HRegionInfo>>();
454 
455     assignRegions(regions, servers, clusterState);
456 
457     // mock block locality for some regions
458     RegionLocationFinder locationFinder = mock(RegionLocationFinder.class);
459     // block locality: region:0   => {server:0}
460     //                 region:1   => {server:0, server:1}
461     //                 region:42 => {server:4, server:9, server:5}
462     when(locationFinder.getTopBlockLocations(regions.get(0))).thenReturn(
463         Lists.newArrayList(servers.get(0)));
464     when(locationFinder.getTopBlockLocations(regions.get(1))).thenReturn(
465         Lists.newArrayList(servers.get(0), servers.get(1)));
466     when(locationFinder.getTopBlockLocations(regions.get(42))).thenReturn(
467         Lists.newArrayList(servers.get(4), servers.get(9), servers.get(5)));
468     when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn(
469         Lists.newArrayList(ServerName.valueOf("foo", 0, 0))); // this server does not exists in clusterStatus
470 
471     BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, locationFinder, null);
472 
473     int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just a test
474     int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));
475     int r10 = ArrayUtils.indexOf(cluster.regions, regions.get(10));
476     int r42 = ArrayUtils.indexOf(cluster.regions, regions.get(42));
477     int r43 = ArrayUtils.indexOf(cluster.regions, regions.get(43));
478 
479     int s0 = cluster.serversToIndex.get(servers.get(0).getHostAndPort());
480     int s1 = cluster.serversToIndex.get(servers.get(1).getHostAndPort());
481     int s4 = cluster.serversToIndex.get(servers.get(4).getHostAndPort());
482     int s5 = cluster.serversToIndex.get(servers.get(5).getHostAndPort());
483     int s9 = cluster.serversToIndex.get(servers.get(9).getHostAndPort());
484 
485     // region 0 locations
486     assertEquals(1, cluster.regionLocations[r0].length);
487     assertEquals(s0, cluster.regionLocations[r0][0]);
488 
489     // region 1 locations
490     assertEquals(2, cluster.regionLocations[r1].length);
491     assertEquals(s0, cluster.regionLocations[r1][0]);
492     assertEquals(s1, cluster.regionLocations[r1][1]);
493 
494     // region 10 locations
495     assertEquals(0, cluster.regionLocations[r10].length);
496 
497     // region 42 locations
498     assertEquals(3, cluster.regionLocations[r42].length);
499     assertEquals(s4, cluster.regionLocations[r42][0]);
500     assertEquals(s9, cluster.regionLocations[r42][1]);
501     assertEquals(s5, cluster.regionLocations[r42][2]);
502 
503     // region 43 locations
504     assertEquals(1, cluster.regionLocations[r43].length);
505     assertEquals(-1, cluster.regionLocations[r43][0]);
506   }
507 }