View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.ByteArrayOutputStream;
27  import java.io.IOException;
28  import java.io.PrintStream;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.testclassification.LargeTests;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Table;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
41  import org.apache.hadoop.mapreduce.Job;
42  import org.apache.hadoop.util.GenericOptionsParser;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Basic test for the CopyTable M/R tool
50   */
51  @Category(LargeTests.class)
52  public class TestCopyTable {
53    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54    private static final byte[] ROW1 = Bytes.toBytes("row1");
55    private static final byte[] ROW2 = Bytes.toBytes("row2");
56    private static final String FAMILY_A_STRING = "a";
57    private static final String FAMILY_B_STRING = "b";
58    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
59    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
60    private static final byte[] QUALIFIER = Bytes.toBytes("q");
61  
62  
63    @BeforeClass
64    public static void beforeClass() throws Exception {
65      TEST_UTIL.startMiniCluster(3);
66      TEST_UTIL.startMiniMapReduceCluster();
67    }
68  
69    @AfterClass
70    public static void afterClass() throws Exception {
71      TEST_UTIL.shutdownMiniMapReduceCluster();
72      TEST_UTIL.shutdownMiniCluster();
73    }
74  
75    private void doCopyTableTest(boolean bulkload) throws Exception {
76      final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
77      final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
78      final byte[] FAMILY = Bytes.toBytes("family");
79      final byte[] COLUMN1 = Bytes.toBytes("c1");
80  
81      Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
82      Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
83  
84      // put rows into the first table
85      for (int i = 0; i < 10; i++) {
86        Put p = new Put(Bytes.toBytes("row" + i));
87        p.add(FAMILY, COLUMN1, COLUMN1);
88        t1.put(p);
89      }
90  
91      CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
92  
93      int code;
94      if (bulkload) {
95        code = copy.run(new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
96            "--bulkload", TABLENAME1.getNameAsString() });
97      } else {
98        code = copy.run(new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
99            TABLENAME1.getNameAsString() });
100     }
101     assertEquals("copy job failed", 0, code);
102 
103     // verify the data was copied into table 2
104     for (int i = 0; i < 10; i++) {
105       Get g = new Get(Bytes.toBytes("row" + i));
106       Result r = t2.get(g);
107       assertEquals(1, r.size());
108       assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
109     }
110     
111     t1.close();
112     t2.close();
113     TEST_UTIL.deleteTable(TABLENAME1);
114     TEST_UTIL.deleteTable(TABLENAME2);
115   }
116 
117   /**
118    * Simple end-to-end test
119    * @throws Exception
120    */
121   @Test
122   public void testCopyTable() throws Exception {
123     doCopyTableTest(false);
124   }
125   
126   /**
127    * Simple end-to-end test with bulkload.
128    */
129   @Test
130   public void testCopyTableWithBulkload() throws Exception {
131     doCopyTableTest(true);
132   }
133   
134   @Test
135   public void testStartStopRow() throws Exception {
136     final TableName TABLENAME1 = TableName.valueOf("testStartStopRow1");
137     final TableName TABLENAME2 = TableName.valueOf("testStartStopRow2");
138     final byte[] FAMILY = Bytes.toBytes("family");
139     final byte[] COLUMN1 = Bytes.toBytes("c1");
140     final byte[] ROW0 = Bytes.toBytes("row0");
141     final byte[] ROW1 = Bytes.toBytes("row1");
142     final byte[] ROW2 = Bytes.toBytes("row2");
143 
144     Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
145     Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
146 
147     // put rows into the first table
148     Put p = new Put(ROW0);
149     p.add(FAMILY, COLUMN1, COLUMN1);
150     t1.put(p);
151     p = new Put(ROW1);
152     p.add(FAMILY, COLUMN1, COLUMN1);
153     t1.put(p);
154     p = new Put(ROW2);
155     p.add(FAMILY, COLUMN1, COLUMN1);
156     t1.put(p);
157 
158     CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
159     assertEquals(
160       0,
161       copy.run(new String[] { "--new.name=" + TABLENAME2, "--startrow=row1",
162           "--stoprow=row2", TABLENAME1.getNameAsString() }));
163 
164     // verify the data was copied into table 2
165     // row1 exist, row0, row2 do not exist
166     Get g = new Get(ROW1);
167     Result r = t2.get(g);
168     assertEquals(1, r.size());
169     assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
170 
171     g = new Get(ROW0);
172     r = t2.get(g);
173     assertEquals(0, r.size());
174     
175     g = new Get(ROW2);
176     r = t2.get(g);
177     assertEquals(0, r.size());
178     
179     t1.close();
180     t2.close();
181     TEST_UTIL.deleteTable(TABLENAME1);
182     TEST_UTIL.deleteTable(TABLENAME2);
183   }
184 
185   /**
186    * Test copy of table from sourceTable to targetTable all rows from family a
187    */
188   @Test
189   public void testRenameFamily() throws Exception {
190     String sourceTable = "sourceTable";
191     String targetTable = "targetTable";
192 
193     byte[][] families = { FAMILY_A, FAMILY_B };
194 
195     Table t = TEST_UTIL.createTable(Bytes.toBytes(sourceTable), families);
196     Table t2 = TEST_UTIL.createTable(Bytes.toBytes(targetTable), families);
197     Put p = new Put(ROW1);
198     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data11"));
199     p.add(FAMILY_B, QUALIFIER,  Bytes.toBytes("Data12"));
200     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data13"));
201     t.put(p);
202     p = new Put(ROW2);
203     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
204     p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
205     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
206     t.put(p);
207 
208     long currentTime = System.currentTimeMillis();
209     String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
210         "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
211         "--versions=1", sourceTable };
212     assertNull(t2.get(new Get(ROW1)).getRow());
213 
214     assertTrue(runCopy(args));
215 
216     assertNotNull(t2.get(new Get(ROW1)).getRow());
217     Result res = t2.get(new Get(ROW1));
218     byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
219     assertEquals("Data13", new String(b1));
220     assertNotNull(t2.get(new Get(ROW2)).getRow());
221     res = t2.get(new Get(ROW2));
222     b1 = res.getValue(FAMILY_A, QUALIFIER);
223     // Data from the family of B is not copied
224     assertNull(b1);
225 
226   }
227 
228   /**
229    * Test main method of CopyTable.
230    */
231   @Test
232   public void testMainMethod() throws Exception {
233     String[] emptyArgs = { "-h" };
234     PrintStream oldWriter = System.err;
235     ByteArrayOutputStream data = new ByteArrayOutputStream();
236     PrintStream writer = new PrintStream(data);
237     System.setErr(writer);
238     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
239     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
240     System.setSecurityManager(newSecurityManager);
241     try {
242       CopyTable.main(emptyArgs);
243       fail("should be exit");
244     } catch (SecurityException e) {
245       assertEquals(1, newSecurityManager.getExitCode());
246     } finally {
247       System.setErr(oldWriter);
248       System.setSecurityManager(SECURITY_MANAGER);
249     }
250     assertTrue(data.toString().contains("rs.class"));
251     // should print usage information
252     assertTrue(data.toString().contains("Usage:"));
253   }
254 
255   private boolean runCopy(String[] args) throws IOException, InterruptedException,
256       ClassNotFoundException {
257     GenericOptionsParser opts = new GenericOptionsParser(
258         new Configuration(TEST_UTIL.getConfiguration()), args);
259     Configuration configuration = opts.getConfiguration();
260     args = opts.getRemainingArgs();
261     Job job = new CopyTable(configuration).createSubmittableJob(args);
262     job.waitForCompletion(false);
263     return job.isSuccessful();
264   }
265 }