View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.HashSet;
25  import java.util.Hashtable;
26  import java.util.Iterator;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Random;
30  import java.util.Set;
31  
32  import javax.management.MBeanAttributeInfo;
33  import javax.management.MBeanInfo;
34  import javax.management.MBeanServerConnection;
35  import javax.management.ObjectInstance;
36  import javax.management.ObjectName;
37  import javax.management.remote.JMXConnector;
38  import javax.management.remote.JMXConnectorFactory;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
44  import org.apache.hadoop.hbase.master.balancer.BalancerTestBase;
45  import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
46  import org.apache.hadoop.hbase.testclassification.MediumTests;
47  import org.apache.hadoop.hbase.util.Threads;
48  import org.apache.hadoop.net.DNSToSwitchMapping;
49  import org.junit.AfterClass;
50  import org.junit.BeforeClass;
51  import org.junit.FixMethodOrder;
52  import org.junit.Ignore;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  import org.junit.runners.MethodSorters;
56  
57  @Category({ MediumTests.class })
58  @FixMethodOrder(MethodSorters.NAME_ASCENDING)
59  @Ignore
60  public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
61    private static final Log LOG = LogFactory.getLog(TestStochasticBalancerJmxMetrics.class);
62    private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
63    private static int connectorPort = 61120;
64    private static StochasticLoadBalancer loadBalancer;
65    /**
66     * a simple cluster for testing JMX.
67     */
68    private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
69    private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
70    private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
71    private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
72  
73    private static final String TABLE_NAME_1 = "Table1";
74    private static final String TABLE_NAME_2 = "Table2";
75    private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
76  
77    private static Configuration conf = null;
78  
79    /**
80     * Setup the environment for the test.
81     */
82    @BeforeClass
83    public static void setupBeforeClass() throws Exception {
84  
85      conf = UTIL.getConfiguration();
86  
87      conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
88      conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
89      conf.setFloat("hbase.regions.slop", 0.0f);
90      conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
91  
92      Random rand = new Random();
93      for (int i = 0; i < 10; i++) {
94        do {
95          int sign = i % 2 == 0 ? 1 : -1;
96          connectorPort += sign * rand.nextInt(100);
97        } while (!HBaseTestingUtility.available(connectorPort));
98        try {
99          conf.setInt("regionserver.rmi.registry.port", connectorPort);
100         UTIL.startMiniCluster();
101         break;
102       } catch (Exception e) {
103         LOG.debug("Encountered exception when starting mini cluster. Trying port " + connectorPort,
104           e);
105         try {
106           // this is to avoid "IllegalStateException: A mini-cluster is already running"
107           UTIL.shutdownMiniCluster();
108         } catch (Exception ex) {
109           LOG.debug("Encountered exception shutting down cluster", ex);
110         }
111       }
112     }
113   }
114 
115   @AfterClass
116   public static void tearDownAfterClass() throws Exception {
117     UTIL.shutdownMiniCluster();
118   }
119 
120   /**
121    * In Ensemble mode, there should be only one ensemble table
122    */
123   @Test (timeout=60000)
124   public void testJmxMetrics_EnsembleMode() throws Exception {
125     loadBalancer = new StochasticLoadBalancer();
126 
127     conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
128     loadBalancer.setConf(conf);
129 
130     TableName tableName = TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME);
131     Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
132     loadBalancer.balanceCluster(tableName, clusterState);
133 
134     String[] tableNames = new String[] { tableName.getNameAsString() };
135     String[] functionNames = loadBalancer.getCostFunctionNames();
136     Set<String> jmxMetrics = readJmxMetricsWithRetry();
137     Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
138 
139     // printMetrics(jmxMetrics, "existing metrics in ensemble mode");
140     // printMetrics(expectedMetrics, "expected metrics in ensemble mode");
141 
142     // assert that every expected is in the JMX
143     for (String expected : expectedMetrics) {
144       assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
145         jmxMetrics.contains(expected));
146     }
147   }
148 
149   /**
150    * In per-table mode, each table has a set of metrics
151    */
152   @Test (timeout=60000)
153   public void testJmxMetrics_PerTableMode() throws Exception {
154     loadBalancer = new StochasticLoadBalancer();
155 
156     conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
157     loadBalancer.setConf(conf);
158 
159     // NOTE the size is normally set in setClusterStatus, for test purpose, we set it manually
160     // Tables: hbase:namespace, table1, table2
161     // Functions: costFunctions, overall
162     String[] functionNames = loadBalancer.getCostFunctionNames();
163     loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
164 
165     // table 1
166     TableName tableName = TableName.valueOf(TABLE_NAME_1);
167     Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
168     loadBalancer.balanceCluster(tableName, clusterState);
169 
170     // table 2
171     tableName = TableName.valueOf(TABLE_NAME_2);
172     clusterState = mockClusterServers(mockCluster_pertable_2);
173     loadBalancer.balanceCluster(tableName, clusterState);
174 
175     // table hbase:namespace
176     tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
177     clusterState = mockClusterServers(mockCluster_pertable_namespace);
178     loadBalancer.balanceCluster(tableName, clusterState);
179 
180     String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
181     Set<String> jmxMetrics = readJmxMetricsWithRetry();
182     Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
183 
184     // printMetrics(jmxMetrics, "existing metrics in per-table mode");
185     // printMetrics(expectedMetrics, "expected metrics in per-table mode");
186 
187     // assert that every expected is in the JMX
188     for (String expected : expectedMetrics) {
189       assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
190         jmxMetrics.contains(expected));
191     }
192   }
193 
194   private Set<String> readJmxMetricsWithRetry() throws IOException {
195     final int count = 0;
196     for (int i = 0; i < 10; i++) {
197       Set<String> metrics = readJmxMetrics();
198       if (metrics != null) return metrics;
199       LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
200       Threads.sleep(1000);
201     }
202     return null;
203   }
204 
205   /**
206    * Read the attributes from Hadoop->HBase->Master->Balancer in JMX
207    * @throws IOException 
208    */
209   private Set<String> readJmxMetrics() throws IOException {
210     JMXConnector connector = null;
211     ObjectName target = null;
212     MBeanServerConnection mb = null;
213     try {
214       connector =
215           JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
216       mb = connector.getMBeanServerConnection();
217 
218       Hashtable<String, String> pairs = new Hashtable<>();
219       pairs.put("service", "HBase");
220       pairs.put("name", "Master");
221       pairs.put("sub", "Balancer");
222       target = new ObjectName("Hadoop", pairs);
223       MBeanInfo beanInfo = mb.getMBeanInfo(target);
224 
225       Set<String> existingAttrs = new HashSet<String>();
226       for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
227         existingAttrs.add(attrInfo.getName());
228       }
229       return existingAttrs;
230     } catch (Exception e) {
231       LOG.warn("Failed to get bean!!! " + target, e);
232       if (mb != null) {
233         Set<ObjectInstance> instances = mb.queryMBeans(null, null);
234         Iterator<ObjectInstance> iterator = instances.iterator();
235         System.out.println("MBean Found:");
236         while (iterator.hasNext()) {
237           ObjectInstance instance = iterator.next();
238           System.out.println("Class Name: " + instance.getClassName());
239           System.out.println("Object Name: " + instance.getObjectName());
240         }
241       }
242     } finally {
243       if (connector != null) {
244         try {
245           connector.close();
246         } catch (Exception e) {
247           e.printStackTrace();
248         }
249       }
250     }
251     return null;
252   }
253 
254   /**
255    * Given the tables and functions, return metrics names that should exist in JMX
256    */
257   private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
258     Set<String> ret = new HashSet<String>();
259 
260     for (String tableName : tableNames) {
261       ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
262       for (String functionName : functionNames) {
263         String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
264         ret.add(metricsName);
265       }
266     }
267 
268     return ret;
269   }
270 
271   private static void printMetrics(Set<String> metrics, String info) {
272     if (null != info) LOG.info("++++ ------ " + info + " ------");
273 
274     LOG.info("++++ metrics count = " + metrics.size());
275     for (String str : metrics) {
276       LOG.info(" ++++ " + str);
277     }
278   }
279 }