View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import org.apache.commons.io.IOUtils;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.fs.FileUtil;
24  import org.apache.hadoop.fs.LocalFileSystem;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.testclassification.LargeTests;
28  import org.apache.hadoop.hbase.client.Put;
29  import org.apache.hadoop.hbase.client.Table;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32  import org.apache.hadoop.mapreduce.Job;
33  import org.apache.hadoop.util.GenericOptionsParser;
34  import org.junit.AfterClass;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  import java.io.*;
40  
41  import static org.junit.Assert.assertTrue;
42  import static org.junit.Assert.assertEquals;
43  import static org.junit.Assert.fail;
44  
45  @Category(LargeTests.class)
46  public class TestCellCounter {
47    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48    private static final byte[] ROW1 = Bytes.toBytes("row1");
49    private static final byte[] ROW2 = Bytes.toBytes("row2");
50    private static final String FAMILY_A_STRING = "a";
51    private static final String FAMILY_B_STRING = "b";
52    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54    private static final byte[] QUALIFIER = Bytes.toBytes("q");
55  
56    private static Path FQ_OUTPUT_DIR;
57    private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58        + "output";
59    private static long now = System.currentTimeMillis();
60  
61    @BeforeClass
62    public static void beforeClass() throws Exception {
63      UTIL.startMiniCluster();
64      UTIL.startMiniMapReduceCluster();
65      FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66      FileUtil.fullyDelete(new File(OUTPUT_DIR));
67    }
68  
69    @AfterClass
70    public static void afterClass() throws Exception {
71      UTIL.shutdownMiniMapReduceCluster();
72      UTIL.shutdownMiniCluster();
73    }
74  
75    /**
76     * Test CellCounter all data should print to output
77     *
78     */
79    @Test (timeout=300000)
80    public void testCellCounter() throws Exception {
81      String sourceTable = "sourceTable";
82      byte[][] families = { FAMILY_A, FAMILY_B };
83      Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
84      try{
85      Put p = new Put(ROW1);
86      p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
87      p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
88      p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
89      t.put(p);
90      p = new Put(ROW2);
91      p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
92      p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
93      p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
94      t.put(p);
95      String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
96      runCount(args);
97      FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
98          "part-r-00000");
99      String data = IOUtils.toString(inputStream);
100     inputStream.close();
101     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
102     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
103     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
104     assertTrue(data.contains("b;q" + "\t" + "1"));
105     assertTrue(data.contains("a;q" + "\t" + "1"));
106     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
107     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
108     }finally{
109       t.close();
110       FileUtil.fullyDelete(new File(OUTPUT_DIR));
111     }
112 
113   }
114 
115   /**
116    * Test CellCounter with time range all data should print to output
117    */
118   @Test (timeout=300000)
119   public void testCellCounterStartTimeRange() throws Exception {
120     String sourceTable = "testCellCounterStartTimeRange";
121     byte[][] families = { FAMILY_A, FAMILY_B };
122     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
123     try{
124     Put p = new Put(ROW1);
125     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
126     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
127     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
128     t.put(p);
129     p = new Put(ROW2);
130     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
131     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
132     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
133     t.put(p);
134     String[] args = {
135       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--starttime=" + now,
136       "--endtime=" + now + 2 };
137     runCount(args);
138     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
139         "part-r-00000");
140     String data = IOUtils.toString(inputStream);
141     inputStream.close();
142     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
143     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
144     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
145     assertTrue(data.contains("b;q" + "\t" + "1"));
146     assertTrue(data.contains("a;q" + "\t" + "1"));
147     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
148     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
149     }finally{
150       t.close();
151       FileUtil.fullyDelete(new File(OUTPUT_DIR));
152     }
153   }
154 
155   /**
156    * Test CellCounter with time range all data should print to output
157    */
158   @Test (timeout=300000)
159   public void testCellCounteEndTimeRange() throws Exception {
160     String sourceTable = "testCellCounterEndTimeRange";
161     byte[][] families = { FAMILY_A, FAMILY_B };
162     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
163     try{
164     Put p = new Put(ROW1);
165     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
166     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
167     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
168     t.put(p);
169     p = new Put(ROW2);
170     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
171     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
172     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
173     t.put(p);
174     String[] args = {
175       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--endtime=" + now + 1 };
176     runCount(args);
177     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
178         "part-r-00000");
179     String data = IOUtils.toString(inputStream);
180     inputStream.close();
181     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
182     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
183     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
184     assertTrue(data.contains("b;q" + "\t" + "1"));
185     assertTrue(data.contains("a;q" + "\t" + "1"));
186     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
187     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
188     }finally{
189       t.close();
190       FileUtil.fullyDelete(new File(OUTPUT_DIR));
191     }
192   }
193 
194    /**
195    * Test CellCounter with time range all data should print to output
196    */
197   @Test (timeout=300000)
198   public void testCellCounteOutOfTimeRange() throws Exception {
199     String sourceTable = "testCellCounterOutTimeRange";
200     byte[][] families = { FAMILY_A, FAMILY_B };
201     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
202     try{
203     Put p = new Put(ROW1);
204     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
205     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
206     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
207     t.put(p);
208     p = new Put(ROW2);
209     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
210     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
211     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
212     t.put(p);
213     String[] args = {
214       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "--starttime=" + now + 1,
215       "--endtime=" + now + 2 };
216 
217     runCount(args);
218     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
219         "part-r-00000");
220     String data = IOUtils.toString(inputStream);
221     inputStream.close();
222     // nothing should hace been emitted to the reducer
223     assertTrue(data.isEmpty());
224     }finally{
225       t.close();
226       FileUtil.fullyDelete(new File(OUTPUT_DIR));
227     }
228   }
229 
230 
231   private boolean runCount(String[] args) throws IOException, InterruptedException,
232       ClassNotFoundException {
233     // need to make a copy of the configuration because to make sure
234     // different temp dirs are used.
235     GenericOptionsParser opts = new GenericOptionsParser(
236         new Configuration(UTIL.getConfiguration()), args);
237     Configuration configuration = opts.getConfiguration();
238     args = opts.getRemainingArgs();
239     Job job = CellCounter.createSubmittableJob(configuration, args);
240     job.waitForCompletion(false);
241     return job.isSuccessful();
242   }
243 
244   /**
245    * Test main method of CellCounter
246    */
247   @Test (timeout=300000)
248   public void testCellCounterMain() throws Exception {
249 
250     PrintStream oldPrintStream = System.err;
251     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
252     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
253     System.setSecurityManager(newSecurityManager);
254     ByteArrayOutputStream data = new ByteArrayOutputStream();
255     String[] args = {};
256     System.setErr(new PrintStream(data));
257     try {
258       System.setErr(new PrintStream(data));
259 
260       try {
261         CellCounter.main(args);
262         fail("should be SecurityException");
263       } catch (SecurityException e) {
264         assertEquals(-1, newSecurityManager.getExitCode());
265         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
266         // should be information about usage
267         assertTrue(data.toString().contains("Usage:"));
268       }
269 
270     } finally {
271       System.setErr(oldPrintStream);
272       System.setSecurityManager(SECURITY_MANAGER);
273     }
274   }
275 
276   /**
277    * Test CellCounter for complete table all data should print to output
278    */
279   @Test(timeout = 300000)
280   public void testCellCounterForCompleteTable() throws Exception {
281     String sourceTable = "testCellCounterForCompleteTable";
282     String outputPath = OUTPUT_DIR + sourceTable;
283     LocalFileSystem localFileSystem = new LocalFileSystem();
284     Path outputDir =
285         new Path(outputPath).makeQualified(localFileSystem.getUri(),
286           localFileSystem.getWorkingDirectory());
287     byte[][] families = { FAMILY_A, FAMILY_B };
288     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
289     try {
290       Put p = new Put(ROW1);
291       p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
292       p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
293       p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
294       t.put(p);
295       p = new Put(ROW2);
296       p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
297       p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
298       p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
299       t.put(p);
300       String[] args = { sourceTable, outputDir.toString(), ";" };
301       runCount(args);
302       FileInputStream inputStream =
303           new FileInputStream(outputPath + File.separator + "part-r-00000");
304       String data = IOUtils.toString(inputStream);
305       inputStream.close();
306       assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
307       assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
308       assertTrue(data.contains("Total ROWS" + "\t" + "2"));
309       assertTrue(data.contains("b;q" + "\t" + "2"));
310       assertTrue(data.contains("a;q" + "\t" + "2"));
311       assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
312       assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
313       assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
314       assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
315     } finally {
316       t.close();
317       FileUtil.fullyDelete(new File(outputPath));
318     }
319   }
320 
321   @Test (timeout=300000)
322   public void TestCellCounterWithoutOutputDir() throws Exception {
323     PrintStream oldPrintStream = System.err;
324     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
325     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
326     System.setSecurityManager(newSecurityManager);
327     ByteArrayOutputStream data = new ByteArrayOutputStream();
328     String[] args = {"tableName"};
329     System.setErr(new PrintStream(data));
330     try {
331       System.setErr(new PrintStream(data));
332       try {
333         CellCounter.main(args);
334         fail("should be SecurityException");
335       } catch (SecurityException e) {
336         assertEquals(-1, newSecurityManager.getExitCode());
337         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
338         // should be information about usage
339         assertTrue(data.toString().contains("Usage:"));
340       }
341 
342     } finally {
343       System.setErr(oldPrintStream);
344       System.setSecurityManager(SECURITY_MANAGER);
345     }
346   }
347 }