1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.rest;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.ByteArrayInputStream;
28 import java.io.IOException;
29 import java.io.StringWriter;
30 import java.util.ArrayList;
31 import java.util.Iterator;
32 import java.util.List;
33 import java.util.Random;
34
35 import javax.xml.bind.JAXBContext;
36 import javax.xml.bind.JAXBException;
37 import javax.xml.bind.Marshaller;
38 import javax.xml.bind.Unmarshaller;
39
40 import org.apache.http.Header;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.KeyValue;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.client.Admin;
48 import org.apache.hadoop.hbase.client.Connection;
49 import org.apache.hadoop.hbase.client.ConnectionFactory;
50 import org.apache.hadoop.hbase.client.Durability;
51 import org.apache.hadoop.hbase.client.Put;
52 import org.apache.hadoop.hbase.client.Table;
53 import org.apache.hadoop.hbase.rest.client.Client;
54 import org.apache.hadoop.hbase.rest.client.Cluster;
55 import org.apache.hadoop.hbase.rest.client.Response;
56 import org.apache.hadoop.hbase.rest.model.CellModel;
57 import org.apache.hadoop.hbase.rest.model.CellSetModel;
58 import org.apache.hadoop.hbase.rest.model.RowModel;
59 import org.apache.hadoop.hbase.rest.model.ScannerModel;
60 import org.apache.hadoop.hbase.testclassification.MediumTests;
61 import org.apache.hadoop.hbase.util.Bytes;
62 import org.junit.AfterClass;
63 import org.junit.BeforeClass;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66
67 @Category(MediumTests.class)
68 public class TestScannerResource {
69 private static final TableName TABLE = TableName.valueOf("TestScannerResource");
70 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
71 private static final String CFA = "a";
72 private static final String CFB = "b";
73 private static final String COLUMN_1 = CFA + ":1";
74 private static final String COLUMN_2 = CFB + ":2";
75
76 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
77 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
78 new HBaseRESTTestingUtility();
79 private static Client client;
80 private static JAXBContext context;
81 private static Marshaller marshaller;
82 private static Unmarshaller unmarshaller;
83 private static int expectedRows1;
84 private static int expectedRows2;
85 private static Configuration conf;
86
87 static int insertData(Configuration conf, TableName tableName, String column, double prob)
88 throws IOException {
89 Random rng = new Random();
90 byte[] k = new byte[3];
91 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
92 List<Put> puts = new ArrayList<>();
93 for (byte b1 = 'a'; b1 < 'z'; b1++) {
94 for (byte b2 = 'a'; b2 < 'z'; b2++) {
95 for (byte b3 = 'a'; b3 < 'z'; b3++) {
96 if (rng.nextDouble() < prob) {
97 k[0] = b1;
98 k[1] = b2;
99 k[2] = b3;
100 Put put = new Put(k);
101 put.setDurability(Durability.SKIP_WAL);
102 put.add(famAndQf[0], famAndQf[1], k);
103 puts.add(put);
104 }
105 }
106 }
107 }
108 try (Connection conn = ConnectionFactory.createConnection(conf);
109 Table table = conn.getTable(tableName)) {
110 table.put(puts);
111 }
112 return puts.size();
113 }
114
115 static int countCellSet(CellSetModel model) {
116 int count = 0;
117 Iterator<RowModel> rows = model.getRows().iterator();
118 while (rows.hasNext()) {
119 RowModel row = rows.next();
120 Iterator<CellModel> cells = row.getCells().iterator();
121 while (cells.hasNext()) {
122 cells.next();
123 count++;
124 }
125 }
126 return count;
127 }
128
129 private static int fullTableScan(ScannerModel model) throws IOException {
130 model.setBatch(100);
131 Response response = client.put("/" + TABLE + "/scanner",
132 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
133 assertEquals(response.getCode(), 201);
134 String scannerURI = response.getLocation();
135 assertNotNull(scannerURI);
136 int count = 0;
137 while (true) {
138 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
139 assertTrue(response.getCode() == 200 || response.getCode() == 204);
140 if (response.getCode() == 200) {
141 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
142 CellSetModel cellSet = new CellSetModel();
143 cellSet.getObjectFromMessage(response.getBody());
144 Iterator<RowModel> rows = cellSet.getRows().iterator();
145 while (rows.hasNext()) {
146 RowModel row = rows.next();
147 Iterator<CellModel> cells = row.getCells().iterator();
148 while (cells.hasNext()) {
149 cells.next();
150 count++;
151 }
152 }
153 } else {
154 break;
155 }
156 }
157
158 response = client.delete(scannerURI);
159 assertEquals(response.getCode(), 200);
160 return count;
161 }
162
163 @BeforeClass
164 public static void setUpBeforeClass() throws Exception {
165 conf = TEST_UTIL.getConfiguration();
166 TEST_UTIL.startMiniCluster();
167 REST_TEST_UTIL.startServletContainer(conf);
168 client = new Client(new Cluster().add("localhost",
169 REST_TEST_UTIL.getServletPort()));
170 context = JAXBContext.newInstance(
171 CellModel.class,
172 CellSetModel.class,
173 RowModel.class,
174 ScannerModel.class);
175 marshaller = context.createMarshaller();
176 unmarshaller = context.createUnmarshaller();
177 Admin admin = TEST_UTIL.getHBaseAdmin();
178 if (admin.tableExists(TABLE)) {
179 return;
180 }
181 HTableDescriptor htd = new HTableDescriptor(TABLE);
182 htd.addFamily(new HColumnDescriptor(CFA));
183 htd.addFamily(new HColumnDescriptor(CFB));
184 admin.createTable(htd);
185 expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
186 expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
187 }
188
189 @AfterClass
190 public static void tearDownAfterClass() throws Exception {
191 REST_TEST_UTIL.shutdownServletContainer();
192 TEST_UTIL.shutdownMiniCluster();
193 }
194
195 @Test
196 public void testSimpleScannerXML() throws IOException, JAXBException {
197 final int BATCH_SIZE = 5;
198
199 ScannerModel model = new ScannerModel();
200 model.setBatch(BATCH_SIZE);
201 model.addColumn(Bytes.toBytes(COLUMN_1));
202 StringWriter writer = new StringWriter();
203 marshaller.marshal(model, writer);
204 byte[] body = Bytes.toBytes(writer.toString());
205
206
207 conf.set("hbase.rest.readonly", "true");
208 Response response = client.put("/" + TABLE + "/scanner",
209 Constants.MIMETYPE_XML, body);
210 assertEquals(response.getCode(), 403);
211 String scannerURI = response.getLocation();
212 assertNull(scannerURI);
213
214
215 conf.set("hbase.rest.readonly", "false");
216 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
217 body);
218 assertEquals(response.getCode(), 201);
219 scannerURI = response.getLocation();
220 assertNotNull(scannerURI);
221
222
223 response = client.get(scannerURI, Constants.MIMETYPE_XML);
224 assertEquals(response.getCode(), 200);
225 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
226 CellSetModel cellSet = (CellSetModel)
227 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
228
229 assertEquals(countCellSet(cellSet), BATCH_SIZE);
230
231
232 conf.set("hbase.rest.readonly", "true");
233 response = client.delete(scannerURI);
234 assertEquals(response.getCode(), 403);
235
236
237 conf.set("hbase.rest.readonly", "false");
238 response = client.delete(scannerURI);
239 assertEquals(response.getCode(), 200);
240 }
241
242 @Test
243 public void testSimpleScannerPB() throws IOException {
244 final int BATCH_SIZE = 10;
245
246 ScannerModel model = new ScannerModel();
247 model.setBatch(BATCH_SIZE);
248 model.addColumn(Bytes.toBytes(COLUMN_1));
249
250
251 conf.set("hbase.rest.readonly", "true");
252 Response response = client.put("/" + TABLE + "/scanner",
253 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
254 assertEquals(response.getCode(), 403);
255 String scannerURI = response.getLocation();
256 assertNull(scannerURI);
257
258
259 conf.set("hbase.rest.readonly", "false");
260 response = client.put("/" + TABLE + "/scanner",
261 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
262 assertEquals(response.getCode(), 201);
263 scannerURI = response.getLocation();
264 assertNotNull(scannerURI);
265
266
267 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
268 assertEquals(response.getCode(), 200);
269 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
270 CellSetModel cellSet = new CellSetModel();
271 cellSet.getObjectFromMessage(response.getBody());
272
273 assertEquals(countCellSet(cellSet), BATCH_SIZE);
274
275
276 conf.set("hbase.rest.readonly", "true");
277 response = client.delete(scannerURI);
278 assertEquals(response.getCode(), 403);
279
280
281 conf.set("hbase.rest.readonly", "false");
282 response = client.delete(scannerURI);
283 assertEquals(response.getCode(), 200);
284 }
285
286 @Test
287 public void testSimpleScannerBinary() throws IOException {
288
289 ScannerModel model = new ScannerModel();
290 model.setBatch(1);
291 model.addColumn(Bytes.toBytes(COLUMN_1));
292
293
294 conf.set("hbase.rest.readonly", "true");
295 Response response = client.put("/" + TABLE + "/scanner",
296 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
297 assertEquals(response.getCode(), 403);
298 String scannerURI = response.getLocation();
299 assertNull(scannerURI);
300
301
302 conf.set("hbase.rest.readonly", "false");
303 response = client.put("/" + TABLE + "/scanner",
304 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
305 assertEquals(response.getCode(), 201);
306 scannerURI = response.getLocation();
307 assertNotNull(scannerURI);
308
309
310 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
311 assertEquals(response.getCode(), 200);
312 assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
313
314 assertTrue(response.getBody().length > 0);
315
316 boolean foundRowHeader = false, foundColumnHeader = false,
317 foundTimestampHeader = false;
318 for (Header header: response.getHeaders()) {
319 if (header.getName().equals("X-Row")) {
320 foundRowHeader = true;
321 } else if (header.getName().equals("X-Column")) {
322 foundColumnHeader = true;
323 } else if (header.getName().equals("X-Timestamp")) {
324 foundTimestampHeader = true;
325 }
326 }
327 assertTrue(foundRowHeader);
328 assertTrue(foundColumnHeader);
329 assertTrue(foundTimestampHeader);
330
331
332 conf.set("hbase.rest.readonly", "true");
333 response = client.delete(scannerURI);
334 assertEquals(response.getCode(), 403);
335
336
337 conf.set("hbase.rest.readonly", "false");
338 response = client.delete(scannerURI);
339 assertEquals(response.getCode(), 200);
340 }
341
342 @Test
343 public void testFullTableScan() throws IOException {
344 ScannerModel model = new ScannerModel();
345 model.addColumn(Bytes.toBytes(COLUMN_1));
346 assertEquals(fullTableScan(model), expectedRows1);
347
348 model = new ScannerModel();
349 model.addColumn(Bytes.toBytes(COLUMN_2));
350 assertEquals(fullTableScan(model), expectedRows2);
351 }
352
353 @Test
354 public void testTableDoesNotExist() throws IOException, JAXBException {
355 ScannerModel model = new ScannerModel();
356 StringWriter writer = new StringWriter();
357 marshaller.marshal(model, writer);
358 byte[] body = Bytes.toBytes(writer.toString());
359 Response response = client.put("/" + NONEXISTENT_TABLE +
360 "/scanner", Constants.MIMETYPE_XML, body);
361 assertEquals(response.getCode(), 404);
362 }
363
364 }
365