View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.Arrays;
23  import java.util.List;
24  
25  import org.apache.commons.cli.CommandLine;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.client.Admin;
28  import org.apache.hadoop.hbase.client.ConnectionFactory;
29  import org.apache.hadoop.hbase.testclassification.IntegrationTests;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
32  import org.apache.hadoop.hbase.util.LoadTestTool;
33  import org.apache.hadoop.util.ToolRunner;
34  import org.junit.Test;
35  import org.junit.experimental.categories.Category;
36  
37  /**
38   * Integration Test for MOB ingest.
39   */
40  @Category(IntegrationTests.class)
41  public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
42    private static final char COLON = ':';
43  
44    private byte[] mobColumnFamily = LoadTestTool.DEFAULT_COLUMN_FAMILY;
45    public static final String THRESHOLD = "threshold";
46    public static final String MIN_MOB_DATA_SIZE = "minMobDataSize";
47    public static final String MAX_MOB_DATA_SIZE = "maxMobDataSize";
48    private int threshold = 1024; // 1KB
49    private int minMobDataSize = 512; // 512B
50    private int maxMobDataSize = threshold * 5; // 5KB
51    private static final long JUNIT_RUN_TIME = 2 * 60 * 1000; // 2 minutes
52  
53    //similar to LOAD_TEST_TOOL_INIT_ARGS except OPT_IN_MEMORY is removed
54    protected String[] LOAD_TEST_TOOL_MOB_INIT_ARGS = {
55        LoadTestTool.OPT_COMPRESSION,
56        LoadTestTool.OPT_DATA_BLOCK_ENCODING,
57        LoadTestTool.OPT_ENCRYPTION,
58        LoadTestTool.OPT_NUM_REGIONS_PER_SERVER,
59        LoadTestTool.OPT_REGION_REPLICATION,
60    };
61  
62    @Override
63    protected String[] getArgsForLoadTestToolInitTable() {
64      List<String> args = new ArrayList<String>();
65      args.add("-tn");
66      args.add(getTablename().getNameAsString());
67      // pass all remaining args from conf with keys <test class name>.<load test tool arg>
68      String clazz = this.getClass().getSimpleName();
69      for (String arg : LOAD_TEST_TOOL_MOB_INIT_ARGS) {
70        String val = conf.get(String.format("%s.%s", clazz, arg));
71        if (val != null) {
72          args.add("-" + arg);
73          args.add(val);
74        }
75      }
76      args.add("-init_only");
77      return args.toArray(new String[args.size()]);
78    }
79  
80    @Override
81    protected void addOptions() {
82      super.addOptions();
83      super.addOptWithArg(THRESHOLD, "The threshold to classify cells to mob data");
84      super.addOptWithArg(MIN_MOB_DATA_SIZE, "Minimum value size for mob data");
85      super.addOptWithArg(MAX_MOB_DATA_SIZE, "Maximum value size for mob data");
86    }
87  
88    @Override
89    protected void processOptions(CommandLine cmd) {
90      super.processOptions(cmd);
91      if (cmd.hasOption(THRESHOLD)) {
92        threshold = Integer.parseInt(cmd.getOptionValue(THRESHOLD));
93      }
94      if (cmd.hasOption(MIN_MOB_DATA_SIZE)) {
95        minMobDataSize = Integer.parseInt(cmd.getOptionValue(MIN_MOB_DATA_SIZE));
96      }
97      if (cmd.hasOption(MAX_MOB_DATA_SIZE)) {
98        maxMobDataSize = Integer.parseInt(cmd.getOptionValue(MAX_MOB_DATA_SIZE));
99      }
100     if (minMobDataSize > maxMobDataSize) {
101       throw new IllegalArgumentException(
102           "The minMobDataSize should not be larger than minMobDataSize");
103     }
104   }
105 
106   @Test
107   public void testIngest() throws Exception {
108     runIngestTest(JUNIT_RUN_TIME, 100, 10, 1024, 10, 20);
109   };
110 
111   @Override
112   protected void initTable() throws IOException {
113     super.initTable();
114 
115     TableName tableName = getTablename();
116     Admin admin = ConnectionFactory.createConnection().getAdmin();
117     HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
118     LOG.info("Disabling table " + getTablename());
119     admin.disableTable(tableName);
120     for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
121       if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
122         columnDescriptor.setMobEnabled(true);
123         columnDescriptor.setMobThreshold((long) threshold);
124         admin.modifyColumn(tableName, columnDescriptor);
125       }
126     }
127     LOG.info("Enabling table " + getTablename());
128     admin.enableTable(tableName);
129     admin.close();
130   }
131 
132   @Override
133   protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
134       long numKeys) {
135     String[] args = super.getArgsForLoadTestTool(mode, modeSpecificArg, startKey, numKeys);
136     List<String> tmp = new ArrayList<String>(Arrays.asList(args));
137     // LoadTestDataGeneratorMOB:mobColumnFamily:minMobDataSize:maxMobDataSize
138     tmp.add(HIPHEN + LoadTestTool.OPT_GENERATOR);
139     StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithMOB.class.getName());
140     sb.append(COLON);
141     sb.append(Bytes.toString(mobColumnFamily));
142     sb.append(COLON);
143     sb.append(minMobDataSize);
144     sb.append(COLON);
145     sb.append(maxMobDataSize);
146     tmp.add(sb.toString());
147     return tmp.toArray(new String[tmp.size()]);
148   }
149 
150   public static void main(String[] args) throws Exception {
151     Configuration conf = HBaseConfiguration.create();
152     IntegrationTestingUtility.setUseDistributedCluster(conf);
153     int ret = ToolRunner.run(conf, new IntegrationTestIngestWithMOB(), args);
154     System.exit(ret);
155   }
156 }