1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.http;
19
20 import static org.mockito.Mockito.doReturn;
21 import static org.mockito.Mockito.mock;
22
23 import java.io.IOException;
24 import java.io.PrintWriter;
25 import java.net.HttpURLConnection;
26 import java.net.URI;
27 import java.net.URL;
28 import java.util.Arrays;
29 import java.util.Enumeration;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.SortedSet;
34 import java.util.TreeSet;
35 import java.util.concurrent.CountDownLatch;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.Executors;
38
39 import javax.servlet.Filter;
40 import javax.servlet.FilterChain;
41 import javax.servlet.FilterConfig;
42 import javax.servlet.ServletContext;
43 import javax.servlet.ServletException;
44 import javax.servlet.ServletRequest;
45 import javax.servlet.ServletResponse;
46 import javax.servlet.http.HttpServlet;
47 import javax.servlet.http.HttpServletRequest;
48 import javax.servlet.http.HttpServletRequestWrapper;
49 import javax.servlet.http.HttpServletResponse;
50
51 import junit.framework.Assert;
52
53 import org.apache.commons.logging.Log;
54 import org.apache.commons.logging.LogFactory;
55 import org.apache.hadoop.conf.Configuration;
56 import org.apache.hadoop.fs.CommonConfigurationKeys;
57 import org.apache.hadoop.hbase.testclassification.SmallTests;
58 import org.apache.hadoop.hbase.http.HttpServer.QuotingInputFilter.RequestQuoter;
59 import org.apache.hadoop.hbase.http.resource.JerseyResource;
60 import org.apache.hadoop.net.NetUtils;
61 import org.apache.hadoop.security.Groups;
62 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
63 import org.apache.hadoop.security.UserGroupInformation;
64 import org.apache.hadoop.security.authorize.AccessControlList;
65 import org.junit.AfterClass;
66 import org.junit.BeforeClass;
67 import org.junit.Ignore;
68 import org.junit.Test;
69 import org.junit.experimental.categories.Category;
70 import org.mockito.Mockito;
71 import org.mockito.internal.util.reflection.Whitebox;
72 import org.mortbay.jetty.Connector;
73 import org.mortbay.util.ajax.JSON;
74
75 @Category(SmallTests.class)
76 public class TestHttpServer extends HttpServerFunctionalTest {
77 static final Log LOG = LogFactory.getLog(TestHttpServer.class);
78 private static HttpServer server;
79 private static URL baseUrl;
80 private static final int MAX_THREADS = 10;
81
82 @SuppressWarnings("serial")
83 public static class EchoMapServlet extends HttpServlet {
84 @SuppressWarnings("unchecked")
85 @Override
86 public void doGet(HttpServletRequest request,
87 HttpServletResponse response
88 ) throws ServletException, IOException {
89 PrintWriter out = response.getWriter();
90 Map<String, String[]> params = request.getParameterMap();
91 SortedSet<String> keys = new TreeSet<String>(params.keySet());
92 for(String key: keys) {
93 out.print(key);
94 out.print(':');
95 String[] values = params.get(key);
96 if (values.length > 0) {
97 out.print(values[0]);
98 for(int i=1; i < values.length; ++i) {
99 out.print(',');
100 out.print(values[i]);
101 }
102 }
103 out.print('\n');
104 }
105 out.close();
106 }
107 }
108
109 @SuppressWarnings("serial")
110 public static class EchoServlet extends HttpServlet {
111 @SuppressWarnings("unchecked")
112 @Override
113 public void doGet(HttpServletRequest request,
114 HttpServletResponse response
115 ) throws ServletException, IOException {
116 PrintWriter out = response.getWriter();
117 SortedSet<String> sortedKeys = new TreeSet<String>();
118 Enumeration<String> keys = request.getParameterNames();
119 while(keys.hasMoreElements()) {
120 sortedKeys.add(keys.nextElement());
121 }
122 for(String key: sortedKeys) {
123 out.print(key);
124 out.print(':');
125 out.print(request.getParameter(key));
126 out.print('\n');
127 }
128 out.close();
129 }
130 }
131
132 @SuppressWarnings("serial")
133 public static class LongHeaderServlet extends HttpServlet {
134 @Override
135 public void doGet(HttpServletRequest request,
136 HttpServletResponse response
137 ) throws ServletException, IOException {
138 Assert.assertEquals(63 * 1024, request.getHeader("longheader").length());
139 response.setStatus(HttpServletResponse.SC_OK);
140 }
141 }
142
143 @SuppressWarnings("serial")
144 public static class HtmlContentServlet extends HttpServlet {
145 @Override
146 public void doGet(HttpServletRequest request,
147 HttpServletResponse response
148 ) throws ServletException, IOException {
149 response.setContentType("text/html");
150 PrintWriter out = response.getWriter();
151 out.print("hello world");
152 out.close();
153 }
154 }
155
156 @BeforeClass public static void setup() throws Exception {
157 Configuration conf = new Configuration();
158 conf.setInt(HttpServer.HTTP_MAX_THREADS, 10);
159 server = createTestServer(conf);
160 server.addServlet("echo", "/echo", EchoServlet.class);
161 server.addServlet("echomap", "/echomap", EchoMapServlet.class);
162 server.addServlet("htmlcontent", "/htmlcontent", HtmlContentServlet.class);
163 server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
164 server.addJerseyResourcePackage(
165 JerseyResource.class.getPackage().getName(), "/jersey/*");
166 server.start();
167 baseUrl = getServerURL(server);
168 LOG.info("HTTP server started: "+ baseUrl);
169 }
170
171 @AfterClass public static void cleanup() throws Exception {
172 server.stop();
173 }
174
175
176 @Test public void testMaxThreads() throws Exception {
177 int clientThreads = MAX_THREADS * 10;
178 Executor executor = Executors.newFixedThreadPool(clientThreads);
179
180 final CountDownLatch ready = new CountDownLatch(clientThreads);
181 final CountDownLatch start = new CountDownLatch(1);
182 for (int i = 0; i < clientThreads; i++) {
183 executor.execute(new Runnable() {
184 @Override
185 public void run() {
186 ready.countDown();
187 try {
188 start.await();
189 assertEquals("a:b\nc:d\n",
190 readOutput(new URL(baseUrl, "/echo?a=b&c=d")));
191 int serverThreads = server.webServer.getThreadPool().getThreads();
192 assertTrue("More threads are started than expected, Server Threads count: "
193 + serverThreads, serverThreads <= MAX_THREADS);
194 System.out.println("Number of threads = " + serverThreads +
195 " which is less or equal than the max = " + MAX_THREADS);
196 } catch (Exception e) {
197
198 }
199 }
200 });
201 }
202
203 ready.await();
204 start.countDown();
205 }
206
207 @Test public void testEcho() throws Exception {
208 assertEquals("a:b\nc:d\n",
209 readOutput(new URL(baseUrl, "/echo?a=b&c=d")));
210 assertEquals("a:b\nc<:d\ne:>\n",
211 readOutput(new URL(baseUrl, "/echo?a=b&c<=d&e=>")));
212 }
213
214
215 @Test public void testEchoMap() throws Exception {
216 assertEquals("a:b\nc:d\n",
217 readOutput(new URL(baseUrl, "/echomap?a=b&c=d")));
218 assertEquals("a:b,>\nc<:d\n",
219 readOutput(new URL(baseUrl, "/echomap?a=b&c<=d&a=>")));
220 }
221
222
223
224
225
226
227 @Test public void testLongHeader() throws Exception {
228 URL url = new URL(baseUrl, "/longheader");
229 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
230 StringBuilder sb = new StringBuilder();
231 for (int i = 0 ; i < 63 * 1024; i++) {
232 sb.append("a");
233 }
234 conn.setRequestProperty("longheader", sb.toString());
235 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
236 }
237
238 @Test
239 @Ignore
240 public void testContentTypes() throws Exception {
241
242 URL cssUrl = new URL(baseUrl, "/static/test.css");
243 HttpURLConnection conn = (HttpURLConnection)cssUrl.openConnection();
244 conn.connect();
245 assertEquals(200, conn.getResponseCode());
246 assertEquals("text/css", conn.getContentType());
247
248
249 URL servletUrl = new URL(baseUrl, "/echo?a=b");
250 conn = (HttpURLConnection)servletUrl.openConnection();
251 conn.connect();
252 assertEquals(200, conn.getResponseCode());
253 assertEquals("text/plain; charset=utf-8", conn.getContentType());
254
255
256
257 servletUrl = new URL(baseUrl, "/echo?a=b.css");
258 conn = (HttpURLConnection)servletUrl.openConnection();
259 conn.connect();
260 assertEquals(200, conn.getResponseCode());
261 assertEquals("text/plain; charset=utf-8", conn.getContentType());
262
263
264 servletUrl = new URL(baseUrl, "/htmlcontent");
265 conn = (HttpURLConnection)servletUrl.openConnection();
266 conn.connect();
267 assertEquals(200, conn.getResponseCode());
268 assertEquals("text/html; charset=utf-8", conn.getContentType());
269
270
271 servletUrl = new URL(baseUrl, "/testjsp.jsp");
272 conn = (HttpURLConnection)servletUrl.openConnection();
273 conn.connect();
274 assertEquals(200, conn.getResponseCode());
275 assertEquals("text/html; charset=utf-8", conn.getContentType());
276 }
277
278
279
280
281
282
283
284 public static class DummyServletFilter implements Filter {
285 @Override
286 public void destroy() { }
287
288 @Override
289 public void doFilter(ServletRequest request, ServletResponse response,
290 FilterChain filterChain) throws IOException, ServletException {
291 final String userName = request.getParameter("user.name");
292 ServletRequest requestModified =
293 new HttpServletRequestWrapper((HttpServletRequest) request) {
294 @Override
295 public String getRemoteUser() {
296 return userName;
297 }
298 };
299 filterChain.doFilter(requestModified, response);
300 }
301
302 @Override
303 public void init(FilterConfig arg0) throws ServletException { }
304 }
305
306
307
308
309
310 public static class DummyFilterInitializer extends FilterInitializer {
311 public DummyFilterInitializer() {
312 }
313
314 @Override
315 public void initFilter(FilterContainer container, Configuration conf) {
316 container.addFilter("DummyFilter", DummyServletFilter.class.getName(), null);
317 }
318 }
319
320
321
322
323
324
325
326
327
328
329
330 static int getHttpStatusCode(String urlstring, String userName)
331 throws IOException {
332 URL url = new URL(urlstring + "?user.name=" + userName);
333 System.out.println("Accessing " + url + " as user " + userName);
334 HttpURLConnection connection = (HttpURLConnection)url.openConnection();
335 connection.connect();
336 return connection.getResponseCode();
337 }
338
339
340
341
342 public static class MyGroupsProvider extends ShellBasedUnixGroupsMapping {
343 static Map<String, List<String>> mapping = new HashMap<String, List<String>>();
344
345 static void clearMapping() {
346 mapping.clear();
347 }
348
349 @Override
350 public List<String> getGroups(String user) throws IOException {
351 return mapping.get(user);
352 }
353 }
354
355
356
357
358
359
360
361 @Test
362 @Ignore
363 public void testDisabledAuthorizationOfDefaultServlets() throws Exception {
364
365 Configuration conf = new Configuration();
366
367
368 conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY,
369 DummyFilterInitializer.class.getName());
370 conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
371 MyGroupsProvider.class.getName());
372 Groups.getUserToGroupsMappingService(conf);
373 MyGroupsProvider.clearMapping();
374 MyGroupsProvider.mapping.put("userA", Arrays.asList("groupA"));
375 MyGroupsProvider.mapping.put("userB", Arrays.asList("groupB"));
376
377 HttpServer myServer = new HttpServer.Builder().setName("test")
378 .addEndpoint(new URI("http://localhost:0")).setFindPort(true).build();
379 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
380 myServer.start();
381 String serverURL = "http://" + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
382 for (String servlet : new String[] { "conf", "logs", "stacks",
383 "logLevel", "metrics" }) {
384 for (String user : new String[] { "userA", "userB" }) {
385 assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(serverURL
386 + servlet, user));
387 }
388 }
389 myServer.stop();
390 }
391
392
393
394
395
396
397
398 @Test
399 @Ignore
400 public void testAuthorizationOfDefaultServlets() throws Exception {
401 Configuration conf = new Configuration();
402 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
403 true);
404 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN,
405 true);
406 conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY,
407 DummyFilterInitializer.class.getName());
408
409 conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
410 MyGroupsProvider.class.getName());
411 Groups.getUserToGroupsMappingService(conf);
412 MyGroupsProvider.clearMapping();
413 MyGroupsProvider.mapping.put("userA", Arrays.asList("groupA"));
414 MyGroupsProvider.mapping.put("userB", Arrays.asList("groupB"));
415 MyGroupsProvider.mapping.put("userC", Arrays.asList("groupC"));
416 MyGroupsProvider.mapping.put("userD", Arrays.asList("groupD"));
417 MyGroupsProvider.mapping.put("userE", Arrays.asList("groupE"));
418
419 HttpServer myServer = new HttpServer.Builder().setName("test")
420 .addEndpoint(new URI("http://localhost:0")).setFindPort(true).setConf(conf)
421 .setACL(new AccessControlList("userA,userB groupC,groupD")).build();
422 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
423 myServer.start();
424
425 String serverURL = "http://"
426 + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
427 for (String servlet : new String[] { "conf", "logs", "stacks",
428 "logLevel", "metrics" }) {
429 for (String user : new String[] { "userA", "userB", "userC", "userD" }) {
430 assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(serverURL
431 + servlet, user));
432 }
433 assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(
434 serverURL + servlet, "userE"));
435 }
436 myServer.stop();
437 }
438
439 @Test
440 public void testRequestQuoterWithNull() throws Exception {
441 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
442 Mockito.doReturn(null).when(request).getParameterValues("dummy");
443 RequestQuoter requestQuoter = new RequestQuoter(request);
444 String[] parameterValues = requestQuoter.getParameterValues("dummy");
445 Assert.assertEquals("It should return null "
446 + "when there are no values for the parameter", null, parameterValues);
447 }
448
449 @Test
450 public void testRequestQuoterWithNotNull() throws Exception {
451 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
452 String[] values = new String[] { "abc", "def" };
453 Mockito.doReturn(values).when(request).getParameterValues("dummy");
454 RequestQuoter requestQuoter = new RequestQuoter(request);
455 String[] parameterValues = requestQuoter.getParameterValues("dummy");
456 Assert.assertTrue("It should return Parameter Values", Arrays.equals(
457 values, parameterValues));
458 }
459
460 @SuppressWarnings("unchecked")
461 private static Map<String, Object> parse(String jsonString) {
462 return (Map<String, Object>)JSON.parse(jsonString);
463 }
464
465 @Test public void testJersey() throws Exception {
466 LOG.info("BEGIN testJersey()");
467 final String js = readOutput(new URL(baseUrl, "/jersey/foo?op=bar"));
468 final Map<String, Object> m = parse(js);
469 LOG.info("m=" + m);
470 assertEquals("foo", m.get(JerseyResource.PATH));
471 assertEquals("bar", m.get(JerseyResource.OP));
472 LOG.info("END testJersey()");
473 }
474
475 @Test
476 public void testHasAdministratorAccess() throws Exception {
477 Configuration conf = new Configuration();
478 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
479 ServletContext context = Mockito.mock(ServletContext.class);
480 Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
481 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(null);
482 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
483 Mockito.when(request.getRemoteUser()).thenReturn(null);
484 HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
485
486
487 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
488
489
490 response = Mockito.mock(HttpServletResponse.class);
491 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
492 Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
493 Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
494
495
496 response = Mockito.mock(HttpServletResponse.class);
497 Mockito.when(request.getRemoteUser()).thenReturn("foo");
498 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
499
500
501 response = Mockito.mock(HttpServletResponse.class);
502 AccessControlList acls = Mockito.mock(AccessControlList.class);
503 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
504 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
505 Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
506 Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
507
508
509 response = Mockito.mock(HttpServletResponse.class);
510 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(true);
511 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
512 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
513
514 }
515
516 @Test
517 public void testRequiresAuthorizationAccess() throws Exception {
518 Configuration conf = new Configuration();
519 ServletContext context = Mockito.mock(ServletContext.class);
520 Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
521 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
522 HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
523
524
525 Assert.assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response));
526
527
528 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, true);
529 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
530 AccessControlList acls = Mockito.mock(AccessControlList.class);
531 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
532 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
533 Assert.assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response));
534 }
535
536 @Test public void testBindAddress() throws Exception {
537 checkBindAddress("localhost", 0, false).stop();
538
539 HttpServer myServer = checkBindAddress("localhost", 0, false);
540 HttpServer myServer2 = null;
541 try {
542 int port = myServer.getConnectorAddress(0).getPort();
543
544 myServer2 = checkBindAddress("localhost", port, true);
545
546 port = myServer2.getConnectorAddress(0).getPort();
547 myServer2.stop();
548 assertNull(myServer2.getConnectorAddress(0));
549 myServer2.openListeners();
550 assertEquals(port, myServer2.getConnectorAddress(0).getPort());
551 } finally {
552 myServer.stop();
553 if (myServer2 != null) {
554 myServer2.stop();
555 }
556 }
557 }
558
559 private HttpServer checkBindAddress(String host, int port, boolean findPort)
560 throws Exception {
561 HttpServer server = createServer(host, port);
562 try {
563
564 List<?> listeners = (List<?>) Whitebox.getInternalState(server,
565 "listeners");
566 Connector listener = (Connector) Whitebox.getInternalState(
567 listeners.get(0), "listener");
568
569 assertEquals(port, listener.getPort());
570
571 server.openListeners();
572 assertEquals(host, server.getConnectorAddress(0).getHostName());
573
574 int boundPort = server.getConnectorAddress(0).getPort();
575 if (port == 0) {
576 assertTrue(boundPort != 0);
577 } else if (findPort) {
578 assertTrue(boundPort > port);
579
580
581 assertTrue(boundPort - port < 8);
582 }
583 } catch (Exception e) {
584 server.stop();
585 throw e;
586 }
587 return server;
588 }
589
590 @Test
591 public void testXFrameHeaderSameOrigin() throws Exception {
592 Configuration conf = new Configuration();
593 conf.set("hbase.http.filter.xframeoptions.mode", "SAMEORIGIN");
594
595 HttpServer myServer = new HttpServer.Builder().setName("test")
596 .addEndpoint(new URI("http://localhost:0"))
597 .setFindPort(true).setConf(conf).build();
598 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
599 myServer.addServlet("echo", "/echo", EchoServlet.class);
600 myServer.start();
601
602 String serverURL = "http://"
603 + NetUtils.getHostPortString(myServer.getConnectorAddress(0));
604 URL url = new URL(new URL(serverURL), "/echo?a=b&c=d");
605 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
606 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
607 assertEquals("SAMEORIGIN", conn.getHeaderField("X-Frame-Options"));
608 myServer.stop();
609 }
610
611
612
613 @Test
614 public void testNoCacheHeader() throws Exception {
615 URL url = new URL(baseUrl, "/echo?a=b&c=d");
616 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
617 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
618 assertEquals("no-cache", conn.getHeaderField("Cache-Control"));
619 assertEquals("no-cache", conn.getHeaderField("Pragma"));
620 assertNotNull(conn.getHeaderField("Expires"));
621 assertNotNull(conn.getHeaderField("Date"));
622 assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
623 assertEquals("DENY", conn.getHeaderField("X-Frame-Options"));
624 }
625
626
627
628
629 @Test
630 public void testHttpServerBuilderWithExternalConnector() throws Exception {
631 Connector c = mock(Connector.class);
632 doReturn("localhost").when(c).getHost();
633 HttpServer s = new HttpServer.Builder().setName("test").setConnector(c)
634 .build();
635 s.stop();
636 }
637 }