1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.*;
25 import org.apache.hadoop.hbase.client.Admin;
26 import org.apache.hadoop.hbase.regionserver.Region;
27 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
28 import org.apache.hadoop.hbase.testclassification.MediumTests;
29 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31 import org.apache.hadoop.hdfs.MiniDFSCluster;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.ServerLoad;
35 import org.apache.hadoop.hbase.RegionLoad;
36
37 import java.io.*;
38 import java.util.*;
39
40 import org.junit.*;
41 import org.junit.experimental.categories.Category;
42
43 import static org.junit.Assert.assertEquals;
44 import static org.junit.Assert.assertNotNull;
45 import static org.junit.Assert.assertTrue;
46 import static org.junit.Assert.assertFalse;
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestClassLoading {
53 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 private static MiniDFSCluster cluster;
57
58 static final TableName tableName = TableName.valueOf("TestClassLoading");
59 static final String cpName1 = "TestCP1";
60 static final String cpName2 = "TestCP2";
61 static final String cpName3 = "TestCP3";
62 static final String cpName4 = "TestCP4";
63 static final String cpName5 = "TestCP5";
64 static final String cpName6 = "TestCP6";
65
66 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
67
68 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
69 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
70 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
71
72 private static final String[] regionServerSystemCoprocessors =
73 new String[]{
74 regionServerCoprocessor.getSimpleName()
75 };
76
77 private static final String[] masterRegionServerSystemCoprocessors = new String[] {
78 regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
79 regionServerCoprocessor.getSimpleName() };
80
81 @BeforeClass
82 public static void setUpBeforeClass() throws Exception {
83 Configuration conf = TEST_UTIL.getConfiguration();
84
85
86
87 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
88 regionCoprocessor1.getName());
89
90
91
92
93 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
94 regionCoprocessor2.getName());
95
96 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
97 regionServerCoprocessor.getName());
98 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
99 masterCoprocessor.getName());
100 TEST_UTIL.startMiniCluster(1);
101 cluster = TEST_UTIL.getDFSCluster();
102 }
103
104 @AfterClass
105 public static void tearDownAfterClass() throws Exception {
106 TEST_UTIL.shutdownMiniCluster();
107 }
108
109 static File buildCoprocessorJar(String className) throws Exception {
110 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
111 "public class " + className + " extends BaseRegionObserver {}";
112 return ClassLoaderTestHelper.buildJar(
113 TEST_UTIL.getDataTestDir().toString(), className, code);
114 }
115
116 @Test
117
118 public void testClassLoadingFromHDFS() throws Exception {
119 FileSystem fs = cluster.getFileSystem();
120
121 File jarFile1 = buildCoprocessorJar(cpName1);
122 File jarFile2 = buildCoprocessorJar(cpName2);
123
124
125 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
126 new Path(fs.getUri().toString() + Path.SEPARATOR));
127 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
128 jarFile1.getName();
129 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
130 assertTrue("Copy jar file to HDFS failed.",
131 fs.exists(pathOnHDFS1));
132 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
133
134 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
135 new Path(fs.getUri().toString() + Path.SEPARATOR));
136 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
137 jarFile2.getName();
138 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
139 assertTrue("Copy jar file to HDFS failed.",
140 fs.exists(pathOnHDFS2));
141 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
142
143
144 HTableDescriptor htd = new HTableDescriptor(tableName);
145 htd.addFamily(new HColumnDescriptor("test"));
146
147 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
148 "|" + Coprocessor.PRIORITY_USER);
149
150 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
151 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
152 Admin admin = TEST_UTIL.getHBaseAdmin();
153 if (admin.tableExists(tableName)) {
154 if (admin.isTableEnabled(tableName)) {
155 admin.disableTable(tableName);
156 }
157 admin.deleteTable(tableName);
158 }
159 CoprocessorClassLoader.clearCache();
160 byte[] startKey = {10, 63};
161 byte[] endKey = {12, 43};
162 admin.createTable(htd, startKey, endKey, 4);
163 waitForTable(htd.getTableName());
164
165
166 boolean foundTableRegion=false;
167 boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
168 Map<Region, Set<ClassLoader>> regionsActiveClassLoaders =
169 new HashMap<Region, Set<ClassLoader>>();
170 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
171 for (Region region:
172 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
173 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
174 foundTableRegion = true;
175 CoprocessorEnvironment env;
176 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
177 found1 = found1 && (env != null);
178 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
179 found2 = found2 && (env != null);
180 if (env != null) {
181 Configuration conf = env.getConfiguration();
182 found2_k1 = found2_k1 && (conf.get("k1") != null);
183 found2_k2 = found2_k2 && (conf.get("k2") != null);
184 found2_k3 = found2_k3 && (conf.get("k3") != null);
185 } else {
186 found2_k1 = found2_k2 = found2_k3 = false;
187 }
188 regionsActiveClassLoaders
189 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
190 }
191 }
192
193 assertTrue("No region was found for table " + tableName, foundTableRegion);
194 assertTrue("Class " + cpName1 + " was missing on a region", found1);
195 assertTrue("Class " + cpName2 + " was missing on a region", found2);
196 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
197 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
198 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
199
200 assertNotNull(jarFileOnHDFS1 + " was not cached",
201 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
202 assertNotNull(jarFileOnHDFS2 + " was not cached",
203 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
204
205 assertEquals("The number of cached classloaders should be equal to the number" +
206 " of external jar files",
207 2, CoprocessorClassLoader.getAllCached().size());
208
209 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
210 CoprocessorClassLoader.getAllCached());
211 for (Map.Entry<Region, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
212 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
213 + " ClassLoader Cache:" + externalClassLoaders
214 + " Region ClassLoaders:" + regionCP.getValue(),
215 externalClassLoaders.containsAll(regionCP.getValue()));
216 }
217 }
218
219 private String getLocalPath(File file) {
220 return new Path(file.toURI()).toString();
221 }
222
223 @Test
224
225 public void testClassLoadingFromLocalFS() throws Exception {
226 File jarFile = buildCoprocessorJar(cpName3);
227
228
229 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
230 htd.addFamily(new HColumnDescriptor("test"));
231 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
232 Coprocessor.PRIORITY_USER);
233 Admin admin = TEST_UTIL.getHBaseAdmin();
234 admin.createTable(htd);
235 waitForTable(htd.getTableName());
236
237
238 boolean found = false;
239 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
240 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
241 if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
242 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
243 }
244 }
245 assertTrue("Class " + cpName3 + " was missing on a region", found);
246 }
247
248 @Test
249
250 public void testPrivateClassLoader() throws Exception {
251 File jarFile = buildCoprocessorJar(cpName4);
252
253
254 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
255 htd.addFamily(new HColumnDescriptor("test"));
256 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
257 Coprocessor.PRIORITY_USER);
258 Admin admin = TEST_UTIL.getHBaseAdmin();
259 admin.createTable(htd);
260 waitForTable(htd.getTableName());
261
262
263 boolean found = false;
264 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
265 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
266 if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) {
267 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
268 if (cp != null) {
269 found = true;
270 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
271 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
272 }
273 }
274 }
275 assertTrue("Class " + cpName4 + " was missing on a region", found);
276 }
277
278 @Test
279
280
281 public void testHBase3810() throws Exception {
282
283
284 File jarFile1 = buildCoprocessorJar(cpName1);
285 File jarFile2 = buildCoprocessorJar(cpName2);
286 File jarFile5 = buildCoprocessorJar(cpName5);
287 File jarFile6 = buildCoprocessorJar(cpName6);
288
289 String cpKey1 = "COPROCESSOR$1";
290 String cpKey2 = " Coprocessor$2 ";
291 String cpKey3 = " coprocessor$03 ";
292
293 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
294 Coprocessor.PRIORITY_USER;
295 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
296
297 String cpValue3 =
298 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
299
300
301 HTableDescriptor htd = new HTableDescriptor(tableName);
302 htd.addFamily(new HColumnDescriptor("test"));
303
304
305 htd.setValue(cpKey1, cpValue1);
306 htd.setValue(cpKey2, cpValue2);
307 htd.setValue(cpKey3, cpValue3);
308
309
310 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
311 Coprocessor.PRIORITY_USER, null);
312 Map<String, String> kvs = new HashMap<String, String>();
313 kvs.put("k1", "v1");
314 kvs.put("k2", "v2");
315 kvs.put("k3", "v3");
316 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
317 Coprocessor.PRIORITY_USER, kvs);
318
319 Admin admin = TEST_UTIL.getHBaseAdmin();
320 if (admin.tableExists(tableName)) {
321 if (admin.isTableEnabled(tableName)) {
322 admin.disableTable(tableName);
323 }
324 admin.deleteTable(tableName);
325 }
326 admin.createTable(htd);
327 waitForTable(htd.getTableName());
328
329
330 boolean found_2 = false, found_1 = false, found_3 = false,
331 found_5 = false, found_6 = false;
332 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
333 found6_k4 = false;
334
335 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
336 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
337 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
338 found_1 = found_1 ||
339 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
340 found_2 = found_2 ||
341 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
342 found_3 = found_3 ||
343 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
344 != null);
345 found_5 = found_5 ||
346 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
347
348 CoprocessorEnvironment env =
349 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
350 if (env != null) {
351 found_6 = true;
352 Configuration conf = env.getConfiguration();
353 found6_k1 = conf.get("k1") != null;
354 found6_k2 = conf.get("k2") != null;
355 found6_k3 = conf.get("k3") != null;
356 }
357 }
358 }
359
360 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
361 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
362 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
363 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
364 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
365
366 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
367 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
368 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
369 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
370 }
371
372 @Test
373 public void testClassLoadingFromLibDirInJar() throws Exception {
374 loadingClassFromLibDirInJar("/lib/");
375 }
376
377 @Test
378 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
379 loadingClassFromLibDirInJar("lib/");
380 }
381
382 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
383 FileSystem fs = cluster.getFileSystem();
384
385 File innerJarFile1 = buildCoprocessorJar(cpName1);
386 File innerJarFile2 = buildCoprocessorJar(cpName2);
387 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
388
389 ClassLoaderTestHelper.addJarFilesToJar(
390 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
391
392
393 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
394 new Path(fs.getUri().toString() + Path.SEPARATOR));
395 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
396 outerJarFile.getName();
397 assertTrue("Copy jar file to HDFS failed.",
398 fs.exists(new Path(jarFileOnHDFS)));
399 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
400
401
402 HTableDescriptor htd = new HTableDescriptor(tableName);
403 htd.addFamily(new HColumnDescriptor("test"));
404
405 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
406 "|" + Coprocessor.PRIORITY_USER);
407
408 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
409 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
410 Admin admin = TEST_UTIL.getHBaseAdmin();
411 if (admin.tableExists(tableName)) {
412 if (admin.isTableEnabled(tableName)) {
413 admin.disableTable(tableName);
414 }
415 admin.deleteTable(tableName);
416 }
417 admin.createTable(htd);
418 waitForTable(htd.getTableName());
419
420
421 boolean found1 = false, found2 = false, found2_k1 = false,
422 found2_k2 = false, found2_k3 = false;
423 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
424 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
425 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
426 CoprocessorEnvironment env;
427 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
428 if (env != null) {
429 found1 = true;
430 }
431 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
432 if (env != null) {
433 found2 = true;
434 Configuration conf = env.getConfiguration();
435 found2_k1 = conf.get("k1") != null;
436 found2_k2 = conf.get("k2") != null;
437 found2_k3 = conf.get("k3") != null;
438 }
439 }
440 }
441 assertTrue("Class " + cpName1 + " was missing on a region", found1);
442 assertTrue("Class " + cpName2 + " was missing on a region", found2);
443 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
444 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
445 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
446 }
447
448 @Test
449 public void testRegionServerCoprocessorsReported() throws Exception {
450
451
452
453 assertAllRegionServers(null);
454 }
455
456
457
458
459
460
461
462
463
464
465 Map<ServerName, ServerLoad> serversForTable(String tableName) {
466 Map<ServerName, ServerLoad> serverLoadHashMap =
467 new HashMap<ServerName, ServerLoad>();
468 for(Map.Entry<ServerName,ServerLoad> server:
469 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
470 getOnlineServers().entrySet()) {
471 for( Map.Entry<byte[], RegionLoad> region:
472 server.getValue().getRegionsLoad().entrySet()) {
473 if (region.getValue().getNameAsString().equals(tableName)) {
474
475 serverLoadHashMap.put(server.getKey(),server.getValue());
476
477 break;
478 }
479 }
480 }
481 return serverLoadHashMap;
482 }
483
484 void assertAllRegionServers(String tableName) throws InterruptedException {
485 Map<ServerName, ServerLoad> servers;
486 String[] actualCoprocessors = null;
487 boolean success = false;
488 String[] expectedCoprocessors = regionServerSystemCoprocessors;
489 if (tableName == null) {
490
491 servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
492 } else {
493 servers = serversForTable(tableName);
494 }
495 for (int i = 0; i < 5; i++) {
496 boolean any_failed = false;
497 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
498 actualCoprocessors = server.getValue().getRsCoprocessors();
499 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
500 LOG.debug("failed comparison: actual: " +
501 Arrays.toString(actualCoprocessors) +
502 " ; expected: " + Arrays.toString(expectedCoprocessors));
503 any_failed = true;
504 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
505 break;
506 }
507 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
508 }
509 if (any_failed == false) {
510 success = true;
511 break;
512 }
513 LOG.debug("retrying after failed comparison: " + i);
514 Thread.sleep(1000);
515 }
516 assertTrue(success);
517 }
518
519 private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
520 if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
521 expectedCoprocessors = masterRegionServerSystemCoprocessors;
522 } else {
523 expectedCoprocessors = regionServerSystemCoprocessors;
524 }
525 return expectedCoprocessors;
526 }
527
528 @Test
529 public void testMasterCoprocessorsReported() {
530
531
532
533 final String loadedMasterCoprocessorsVerify =
534 "["+ masterCoprocessor.getSimpleName() + "]";
535 String loadedMasterCoprocessors =
536 java.util.Arrays.toString(
537 TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessors());
538 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
539 }
540
541 @Test
542 public void testFindCoprocessors() {
543
544 CoprocessorHost masterCpHost =
545 TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost();
546
547 List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
548
549 assertTrue(masterObservers != null && masterObservers.size() > 0);
550 assertEquals(masterCoprocessor.getSimpleName(),
551 masterObservers.get(0).getClass().getSimpleName());
552 }
553
554 private void waitForTable(TableName name) throws InterruptedException, IOException {
555
556 TEST_UTIL.waitTableEnabled(name);
557
558 Thread.sleep(1000);
559 }
560
561 }
562