1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import org.apache.commons.io.IOUtils;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FileUtil;
24 import org.apache.hadoop.fs.LocalFileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.testclassification.LargeTests;
28 import org.apache.hadoop.hbase.client.Put;
29 import org.apache.hadoop.hbase.client.Table;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32 import org.apache.hadoop.mapreduce.Job;
33 import org.apache.hadoop.util.GenericOptionsParser;
34 import org.junit.AfterClass;
35 import org.junit.BeforeClass;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39 import java.io.*;
40
41 import static org.junit.Assert.assertTrue;
42 import static org.junit.Assert.assertEquals;
43 import static org.junit.Assert.fail;
44
45 @Category(LargeTests.class)
46 public class TestCellCounter {
47 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48 private static final byte[] ROW1 = Bytes.toBytes("row1");
49 private static final byte[] ROW2 = Bytes.toBytes("row2");
50 private static final String FAMILY_A_STRING = "a";
51 private static final String FAMILY_B_STRING = "b";
52 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54 private static final byte[] QUALIFIER = Bytes.toBytes("q");
55
56 private static Path FQ_OUTPUT_DIR;
57 private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58 + "output";
59 private static long now = System.currentTimeMillis();
60
61 @BeforeClass
62 public static void beforeClass() throws Exception {
63 UTIL.startMiniCluster();
64 UTIL.startMiniMapReduceCluster();
65 FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66 FileUtil.fullyDelete(new File(OUTPUT_DIR));
67 }
68
69 @AfterClass
70 public static void afterClass() throws Exception {
71 UTIL.shutdownMiniMapReduceCluster();
72 UTIL.shutdownMiniCluster();
73 }
74
75
76
77
78
79 @Test (timeout=300000)
80 public void testCellCounter() throws Exception {
81 String sourceTable = "sourceTable";
82 byte[][] families = { FAMILY_A, FAMILY_B };
83 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
84 try{
85 Put p = new Put(ROW1);
86 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
87 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
88 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
89 t.put(p);
90 p = new Put(ROW2);
91 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
92 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
93 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
94 t.put(p);
95 String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
96 runCount(args);
97 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
98 "part-r-00000");
99 String data = IOUtils.toString(inputStream);
100 inputStream.close();
101 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
102 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
103 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
104 assertTrue(data.contains("b;q" + "\t" + "1"));
105 assertTrue(data.contains("a;q" + "\t" + "1"));
106 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
107 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
108 }finally{
109 t.close();
110 FileUtil.fullyDelete(new File(OUTPUT_DIR));
111 }
112
113 }
114
115
116
117
118 @Test (timeout=300000)
119 public void testCellCounterStartTimeRange() throws Exception {
120 String sourceTable = "testCellCounterStartTimeRange";
121 byte[][] families = { FAMILY_A, FAMILY_B };
122 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
123 try{
124 Put p = new Put(ROW1);
125 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
126 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
127 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
128 t.put(p);
129 p = new Put(ROW2);
130 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
131 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
132 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
133 t.put(p);
134 String[] args = {
135 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--starttime=" + now,
136 "--endtime=" + now + 2 };
137 runCount(args);
138 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
139 "part-r-00000");
140 String data = IOUtils.toString(inputStream);
141 inputStream.close();
142 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
143 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
144 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
145 assertTrue(data.contains("b;q" + "\t" + "1"));
146 assertTrue(data.contains("a;q" + "\t" + "1"));
147 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
148 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
149 }finally{
150 t.close();
151 FileUtil.fullyDelete(new File(OUTPUT_DIR));
152 }
153 }
154
155
156
157
158 @Test (timeout=300000)
159 public void testCellCounteEndTimeRange() throws Exception {
160 String sourceTable = "testCellCounterEndTimeRange";
161 byte[][] families = { FAMILY_A, FAMILY_B };
162 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
163 try{
164 Put p = new Put(ROW1);
165 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
166 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
167 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
168 t.put(p);
169 p = new Put(ROW2);
170 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
171 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
172 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
173 t.put(p);
174 String[] args = {
175 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--endtime=" + now + 1 };
176 runCount(args);
177 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
178 "part-r-00000");
179 String data = IOUtils.toString(inputStream);
180 inputStream.close();
181 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
182 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
183 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
184 assertTrue(data.contains("b;q" + "\t" + "1"));
185 assertTrue(data.contains("a;q" + "\t" + "1"));
186 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
187 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
188 }finally{
189 t.close();
190 FileUtil.fullyDelete(new File(OUTPUT_DIR));
191 }
192 }
193
194
195
196
197 @Test (timeout=300000)
198 public void testCellCounteOutOfTimeRange() throws Exception {
199 String sourceTable = "testCellCounterOutTimeRange";
200 byte[][] families = { FAMILY_A, FAMILY_B };
201 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
202 try{
203 Put p = new Put(ROW1);
204 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
205 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
206 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
207 t.put(p);
208 p = new Put(ROW2);
209 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
210 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
211 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
212 t.put(p);
213 String[] args = {
214 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "--starttime=" + now + 1,
215 "--endtime=" + now + 2 };
216
217 runCount(args);
218 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
219 "part-r-00000");
220 String data = IOUtils.toString(inputStream);
221 inputStream.close();
222
223 assertTrue(data.isEmpty());
224 }finally{
225 t.close();
226 FileUtil.fullyDelete(new File(OUTPUT_DIR));
227 }
228 }
229
230
231 private boolean runCount(String[] args) throws IOException, InterruptedException,
232 ClassNotFoundException {
233
234
235 GenericOptionsParser opts = new GenericOptionsParser(
236 new Configuration(UTIL.getConfiguration()), args);
237 Configuration configuration = opts.getConfiguration();
238 args = opts.getRemainingArgs();
239 Job job = CellCounter.createSubmittableJob(configuration, args);
240 job.waitForCompletion(false);
241 return job.isSuccessful();
242 }
243
244
245
246
247 @Test (timeout=300000)
248 public void testCellCounterMain() throws Exception {
249
250 PrintStream oldPrintStream = System.err;
251 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
252 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
253 System.setSecurityManager(newSecurityManager);
254 ByteArrayOutputStream data = new ByteArrayOutputStream();
255 String[] args = {};
256 System.setErr(new PrintStream(data));
257 try {
258 System.setErr(new PrintStream(data));
259
260 try {
261 CellCounter.main(args);
262 fail("should be SecurityException");
263 } catch (SecurityException e) {
264 assertEquals(-1, newSecurityManager.getExitCode());
265 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
266
267 assertTrue(data.toString().contains("Usage:"));
268 }
269
270 } finally {
271 System.setErr(oldPrintStream);
272 System.setSecurityManager(SECURITY_MANAGER);
273 }
274 }
275
276
277
278
279 @Test(timeout = 300000)
280 public void testCellCounterForCompleteTable() throws Exception {
281 String sourceTable = "testCellCounterForCompleteTable";
282 String outputPath = OUTPUT_DIR + sourceTable;
283 LocalFileSystem localFileSystem = new LocalFileSystem();
284 Path outputDir =
285 new Path(outputPath).makeQualified(localFileSystem.getUri(),
286 localFileSystem.getWorkingDirectory());
287 byte[][] families = { FAMILY_A, FAMILY_B };
288 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
289 try {
290 Put p = new Put(ROW1);
291 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
292 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
293 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
294 t.put(p);
295 p = new Put(ROW2);
296 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
297 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
298 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
299 t.put(p);
300 String[] args = { sourceTable, outputDir.toString(), ";" };
301 runCount(args);
302 FileInputStream inputStream =
303 new FileInputStream(outputPath + File.separator + "part-r-00000");
304 String data = IOUtils.toString(inputStream);
305 inputStream.close();
306 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
307 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
308 assertTrue(data.contains("Total ROWS" + "\t" + "2"));
309 assertTrue(data.contains("b;q" + "\t" + "2"));
310 assertTrue(data.contains("a;q" + "\t" + "2"));
311 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
312 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
313 assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
314 assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
315 } finally {
316 t.close();
317 FileUtil.fullyDelete(new File(outputPath));
318 }
319 }
320
321 @Test (timeout=300000)
322 public void TestCellCounterWithoutOutputDir() throws Exception {
323 PrintStream oldPrintStream = System.err;
324 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
325 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
326 System.setSecurityManager(newSecurityManager);
327 ByteArrayOutputStream data = new ByteArrayOutputStream();
328 String[] args = {"tableName"};
329 System.setErr(new PrintStream(data));
330 try {
331 System.setErr(new PrintStream(data));
332 try {
333 CellCounter.main(args);
334 fail("should be SecurityException");
335 } catch (SecurityException e) {
336 assertEquals(-1, newSecurityManager.getExitCode());
337 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
338
339 assertTrue(data.toString().contains("Usage:"));
340 }
341
342 } finally {
343 System.setErr(oldPrintStream);
344 System.setSecurityManager(SECURITY_MANAGER);
345 }
346 }
347 }