1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.List;
24
25 import org.apache.commons.cli.CommandLine;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.hbase.client.Admin;
28 import org.apache.hadoop.hbase.client.ConnectionFactory;
29 import org.apache.hadoop.hbase.testclassification.IntegrationTests;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
32 import org.apache.hadoop.hbase.util.LoadTestTool;
33 import org.apache.hadoop.util.ToolRunner;
34 import org.junit.Test;
35 import org.junit.experimental.categories.Category;
36
37
38
39
40 @Category(IntegrationTests.class)
41 public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
42 private static final char COLON = ':';
43
44 private byte[] mobColumnFamily = LoadTestTool.DEFAULT_COLUMN_FAMILY;
45 public static final String THRESHOLD = "threshold";
46 public static final String MIN_MOB_DATA_SIZE = "minMobDataSize";
47 public static final String MAX_MOB_DATA_SIZE = "maxMobDataSize";
48 private int threshold = 1024;
49 private int minMobDataSize = 512;
50 private int maxMobDataSize = threshold * 5;
51 private static final long JUNIT_RUN_TIME = 2 * 60 * 1000;
52
53
54 protected String[] LOAD_TEST_TOOL_MOB_INIT_ARGS = {
55 LoadTestTool.OPT_COMPRESSION,
56 LoadTestTool.OPT_DATA_BLOCK_ENCODING,
57 LoadTestTool.OPT_ENCRYPTION,
58 LoadTestTool.OPT_NUM_REGIONS_PER_SERVER,
59 LoadTestTool.OPT_REGION_REPLICATION,
60 };
61
62 @Override
63 protected String[] getArgsForLoadTestToolInitTable() {
64 List<String> args = new ArrayList<String>();
65 args.add("-tn");
66 args.add(getTablename().getNameAsString());
67
68 String clazz = this.getClass().getSimpleName();
69 for (String arg : LOAD_TEST_TOOL_MOB_INIT_ARGS) {
70 String val = conf.get(String.format("%s.%s", clazz, arg));
71 if (val != null) {
72 args.add("-" + arg);
73 args.add(val);
74 }
75 }
76 args.add("-init_only");
77 return args.toArray(new String[args.size()]);
78 }
79
80 @Override
81 protected void addOptions() {
82 super.addOptions();
83 super.addOptWithArg(THRESHOLD, "The threshold to classify cells to mob data");
84 super.addOptWithArg(MIN_MOB_DATA_SIZE, "Minimum value size for mob data");
85 super.addOptWithArg(MAX_MOB_DATA_SIZE, "Maximum value size for mob data");
86 }
87
88 @Override
89 protected void processOptions(CommandLine cmd) {
90 super.processOptions(cmd);
91 if (cmd.hasOption(THRESHOLD)) {
92 threshold = Integer.parseInt(cmd.getOptionValue(THRESHOLD));
93 }
94 if (cmd.hasOption(MIN_MOB_DATA_SIZE)) {
95 minMobDataSize = Integer.parseInt(cmd.getOptionValue(MIN_MOB_DATA_SIZE));
96 }
97 if (cmd.hasOption(MAX_MOB_DATA_SIZE)) {
98 maxMobDataSize = Integer.parseInt(cmd.getOptionValue(MAX_MOB_DATA_SIZE));
99 }
100 if (minMobDataSize > maxMobDataSize) {
101 throw new IllegalArgumentException(
102 "The minMobDataSize should not be larger than minMobDataSize");
103 }
104 }
105
106 @Test
107 public void testIngest() throws Exception {
108 runIngestTest(JUNIT_RUN_TIME, 100, 10, 1024, 10, 20);
109 };
110
111 @Override
112 protected void initTable() throws IOException {
113 super.initTable();
114
115 TableName tableName = getTablename();
116 Admin admin = ConnectionFactory.createConnection().getAdmin();
117 HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
118 LOG.info("Disabling table " + getTablename());
119 admin.disableTable(tableName);
120 for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
121 if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
122 columnDescriptor.setMobEnabled(true);
123 columnDescriptor.setMobThreshold((long) threshold);
124 admin.modifyColumn(tableName, columnDescriptor);
125 }
126 }
127 LOG.info("Enabling table " + getTablename());
128 admin.enableTable(tableName);
129 admin.close();
130 }
131
132 @Override
133 protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
134 long numKeys) {
135 String[] args = super.getArgsForLoadTestTool(mode, modeSpecificArg, startKey, numKeys);
136 List<String> tmp = new ArrayList<String>(Arrays.asList(args));
137
138 tmp.add(HIPHEN + LoadTestTool.OPT_GENERATOR);
139 StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithMOB.class.getName());
140 sb.append(COLON);
141 sb.append(Bytes.toString(mobColumnFamily));
142 sb.append(COLON);
143 sb.append(minMobDataSize);
144 sb.append(COLON);
145 sb.append(maxMobDataSize);
146 tmp.add(sb.toString());
147 return tmp.toArray(new String[tmp.size()]);
148 }
149
150 public static void main(String[] args) throws Exception {
151 Configuration conf = HBaseConfiguration.create();
152 IntegrationTestingUtility.setUseDistributedCluster(conf);
153 int ret = ToolRunner.run(conf, new IntegrationTestIngestWithMOB(), args);
154 System.exit(ret);
155 }
156 }