1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.normalizer;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.hbase.HBaseTestingUtility;
24 import org.apache.hadoop.hbase.HConstants;
25 import org.apache.hadoop.hbase.HTableDescriptor;
26 import org.apache.hadoop.hbase.MetaTableAccessor;
27 import org.apache.hadoop.hbase.MiniHBaseCluster;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.client.Admin;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.master.HMaster;
33 import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor;
34 import org.apache.hadoop.hbase.quotas.QuotaUtil;
35 import org.apache.hadoop.hbase.regionserver.HRegion;
36 import org.apache.hadoop.hbase.regionserver.Region;
37 import org.apache.hadoop.hbase.testclassification.MediumTests;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
40 import org.junit.AfterClass;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45 import java.io.IOException;
46 import java.util.Collections;
47 import java.util.Comparator;
48 import java.util.List;
49
50 import static org.junit.Assert.assertEquals;
51
52
53
54
55 @Category(MediumTests.class)
56 public class TestSimpleRegionNormalizerOnCluster {
57 private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizerOnCluster.class);
58 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59 private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
60 private static Admin admin;
61
62 @BeforeClass
63 public static void beforeAllTests() throws Exception {
64
65 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
66 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
67
68
69 TEST_UTIL.startMiniCluster(1);
70 TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL);
71 admin = TEST_UTIL.getHBaseAdmin();
72 }
73
74 @AfterClass
75 public static void afterAllTests() throws Exception {
76 TEST_UTIL.shutdownMiniCluster();
77 }
78
79 @Test(timeout = 60000)
80 @SuppressWarnings("deprecation")
81 public void testRegionNormalizationSplitOnCluster() throws Exception {
82 final TableName TABLENAME =
83 TableName.valueOf("testRegionNormalizationSplitOnCluster");
84 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
85 HMaster m = cluster.getMaster();
86
87 try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
88
89 List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
90 Collections.sort(generatedRegions, new Comparator<HRegion>() {
91 @Override
92 public int compare(HRegion o1, HRegion o2) {
93 return o1.getRegionInfo().compareTo(o2.getRegionInfo());
94 }
95 });
96
97 HRegion region = generatedRegions.get(0);
98 generateTestData(region, 1);
99 region.flush(true);
100
101 region = generatedRegions.get(1);
102 generateTestData(region, 1);
103 region.flush(true);
104
105 region = generatedRegions.get(2);
106 generateTestData(region, 2);
107 region.flush(true);
108
109 region = generatedRegions.get(3);
110 generateTestData(region, 2);
111 region.flush(true);
112
113 region = generatedRegions.get(4);
114 generateTestData(region, 5);
115 region.flush(true);
116 }
117
118 HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
119 htd.setNormalizationEnabled(true);
120 admin.modifyTable(TABLENAME, htd);
121
122 admin.flush(TABLENAME);
123
124 System.out.println(admin.getTableDescriptor(TABLENAME));
125
126 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
127
128
129 Thread.sleep(5000);
130 m.normalizeRegions();
131
132 while (true) {
133 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
134 int cnt = 0;
135 for (HRegion region : regions) {
136 String regionName = region.getRegionInfo().getRegionNameAsString();
137 if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
138 cnt++;
139 }
140 }
141 if (cnt >= 2) {
142 break;
143 }
144 }
145 admin.disableTable(TABLENAME);
146 admin.deleteTable(TABLENAME);
147 }
148
149 @Test(timeout = 60000)
150 @SuppressWarnings("deprecation")
151 public void testRegionNormalizationMergeOnCluster() throws Exception {
152 final TableName TABLENAME =
153 TableName.valueOf("testRegionNormalizationMergeOnCluster");
154 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
155 HMaster m = cluster.getMaster();
156
157
158 try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
159
160 List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
161 Collections.sort(generatedRegions, new Comparator<HRegion>() {
162 @Override
163 public int compare(HRegion o1, HRegion o2) {
164 return o1.getRegionInfo().compareTo(o2.getRegionInfo());
165 }
166 });
167
168 HRegion region = generatedRegions.get(0);
169 generateTestData(region, 1);
170 region.flush(true);
171
172 region = generatedRegions.get(1);
173 generateTestData(region, 1);
174 region.flush(true);
175
176 region = generatedRegions.get(2);
177 generateTestData(region, 3);
178 region.flush(true);
179
180 region = generatedRegions.get(3);
181 generateTestData(region, 3);
182 region.flush(true);
183
184 region = generatedRegions.get(4);
185 generateTestData(region, 5);
186 region.flush(true);
187 }
188
189 HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
190 htd.setNormalizationEnabled(true);
191 admin.modifyTable(TABLENAME, htd);
192
193 admin.flush(TABLENAME);
194
195 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
196
197
198 Thread.sleep(5000);
199 m.normalizeRegions();
200
201 while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) > 4) {
202 LOG.info("Waiting for normalization merge to complete");
203 Thread.sleep(100);
204 }
205
206 assertEquals(4, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
207
208 admin.disableTable(TABLENAME);
209 admin.deleteTable(TABLENAME);
210 }
211
212 private void generateTestData(Region region, int numRows) throws IOException {
213
214 LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024);
215 for (int i = 0; i < numRows; ++i) {
216 byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i));
217 for (int j = 0; j < 1; ++j) {
218 Put put = new Put(key);
219 byte[] col = Bytes.toBytes(String.valueOf(j));
220 byte[] value = dataGenerator.generateRandomSizeValue(key, col);
221 put.add(FAMILYNAME, col, value);
222 region.put(put);
223 }
224 }
225 }
226 }