1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.HashSet;
25 import java.util.Hashtable;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Random;
30 import java.util.Set;
31
32 import javax.management.MBeanAttributeInfo;
33 import javax.management.MBeanInfo;
34 import javax.management.MBeanServerConnection;
35 import javax.management.ObjectInstance;
36 import javax.management.ObjectName;
37 import javax.management.remote.JMXConnector;
38 import javax.management.remote.JMXConnectorFactory;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
44 import org.apache.hadoop.hbase.master.balancer.BalancerTestBase;
45 import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
46 import org.apache.hadoop.hbase.testclassification.MediumTests;
47 import org.apache.hadoop.hbase.util.Threads;
48 import org.apache.hadoop.net.DNSToSwitchMapping;
49 import org.junit.AfterClass;
50 import org.junit.BeforeClass;
51 import org.junit.FixMethodOrder;
52 import org.junit.Ignore;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55 import org.junit.runners.MethodSorters;
56
57 @Category({ MediumTests.class })
58 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
59 @Ignore
60 public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
61 private static final Log LOG = LogFactory.getLog(TestStochasticBalancerJmxMetrics.class);
62 private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
63 private static int connectorPort = 61120;
64 private static StochasticLoadBalancer loadBalancer;
65
66
67
68 private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
69 private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
70 private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
71 private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
72
73 private static final String TABLE_NAME_1 = "Table1";
74 private static final String TABLE_NAME_2 = "Table2";
75 private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
76
77 private static Configuration conf = null;
78
79
80
81
82 @BeforeClass
83 public static void setupBeforeClass() throws Exception {
84
85 conf = UTIL.getConfiguration();
86
87 conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
88 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
89 conf.setFloat("hbase.regions.slop", 0.0f);
90 conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
91
92 Random rand = new Random();
93 for (int i = 0; i < 10; i++) {
94 do {
95 int sign = i % 2 == 0 ? 1 : -1;
96 connectorPort += sign * rand.nextInt(100);
97 } while (!HBaseTestingUtility.available(connectorPort));
98 try {
99 conf.setInt("regionserver.rmi.registry.port", connectorPort);
100 UTIL.startMiniCluster();
101 break;
102 } catch (Exception e) {
103 LOG.debug("Encountered exception when starting mini cluster. Trying port " + connectorPort,
104 e);
105 try {
106
107 UTIL.shutdownMiniCluster();
108 } catch (Exception ex) {
109 LOG.debug("Encountered exception shutting down cluster", ex);
110 }
111 }
112 }
113 }
114
115 @AfterClass
116 public static void tearDownAfterClass() throws Exception {
117 UTIL.shutdownMiniCluster();
118 }
119
120
121
122
123 @Test (timeout=60000)
124 public void testJmxMetrics_EnsembleMode() throws Exception {
125 loadBalancer = new StochasticLoadBalancer();
126
127 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
128 loadBalancer.setConf(conf);
129
130 TableName tableName = TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME);
131 Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
132 loadBalancer.balanceCluster(tableName, clusterState);
133
134 String[] tableNames = new String[] { tableName.getNameAsString() };
135 String[] functionNames = loadBalancer.getCostFunctionNames();
136 Set<String> jmxMetrics = readJmxMetricsWithRetry();
137 Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
138
139
140
141
142
143 for (String expected : expectedMetrics) {
144 assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
145 jmxMetrics.contains(expected));
146 }
147 }
148
149
150
151
152 @Test (timeout=60000)
153 public void testJmxMetrics_PerTableMode() throws Exception {
154 loadBalancer = new StochasticLoadBalancer();
155
156 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
157 loadBalancer.setConf(conf);
158
159
160
161
162 String[] functionNames = loadBalancer.getCostFunctionNames();
163 loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
164
165
166 TableName tableName = TableName.valueOf(TABLE_NAME_1);
167 Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
168 loadBalancer.balanceCluster(tableName, clusterState);
169
170
171 tableName = TableName.valueOf(TABLE_NAME_2);
172 clusterState = mockClusterServers(mockCluster_pertable_2);
173 loadBalancer.balanceCluster(tableName, clusterState);
174
175
176 tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
177 clusterState = mockClusterServers(mockCluster_pertable_namespace);
178 loadBalancer.balanceCluster(tableName, clusterState);
179
180 String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
181 Set<String> jmxMetrics = readJmxMetricsWithRetry();
182 Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
183
184
185
186
187
188 for (String expected : expectedMetrics) {
189 assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
190 jmxMetrics.contains(expected));
191 }
192 }
193
194 private Set<String> readJmxMetricsWithRetry() throws IOException {
195 final int count = 0;
196 for (int i = 0; i < 10; i++) {
197 Set<String> metrics = readJmxMetrics();
198 if (metrics != null) return metrics;
199 LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
200 Threads.sleep(1000);
201 }
202 return null;
203 }
204
205
206
207
208
209 private Set<String> readJmxMetrics() throws IOException {
210 JMXConnector connector = null;
211 ObjectName target = null;
212 MBeanServerConnection mb = null;
213 try {
214 connector =
215 JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
216 mb = connector.getMBeanServerConnection();
217
218 Hashtable<String, String> pairs = new Hashtable<>();
219 pairs.put("service", "HBase");
220 pairs.put("name", "Master");
221 pairs.put("sub", "Balancer");
222 target = new ObjectName("Hadoop", pairs);
223 MBeanInfo beanInfo = mb.getMBeanInfo(target);
224
225 Set<String> existingAttrs = new HashSet<String>();
226 for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
227 existingAttrs.add(attrInfo.getName());
228 }
229 return existingAttrs;
230 } catch (Exception e) {
231 LOG.warn("Failed to get bean!!! " + target, e);
232 if (mb != null) {
233 Set<ObjectInstance> instances = mb.queryMBeans(null, null);
234 Iterator<ObjectInstance> iterator = instances.iterator();
235 System.out.println("MBean Found:");
236 while (iterator.hasNext()) {
237 ObjectInstance instance = iterator.next();
238 System.out.println("Class Name: " + instance.getClassName());
239 System.out.println("Object Name: " + instance.getObjectName());
240 }
241 }
242 } finally {
243 if (connector != null) {
244 try {
245 connector.close();
246 } catch (Exception e) {
247 e.printStackTrace();
248 }
249 }
250 }
251 return null;
252 }
253
254
255
256
257 private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
258 Set<String> ret = new HashSet<String>();
259
260 for (String tableName : tableNames) {
261 ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
262 for (String functionName : functionNames) {
263 String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
264 ret.add(metricsName);
265 }
266 }
267
268 return ret;
269 }
270
271 private static void printMetrics(Set<String> metrics, String info) {
272 if (null != info) LOG.info("++++ ------ " + info + " ------");
273
274 LOG.info("++++ metrics count = " + metrics.size());
275 for (String str : metrics) {
276 LOG.info(" ++++ " + str);
277 }
278 }
279 }