View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master.normalizer;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.HBaseTestingUtility;
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.HTableDescriptor;
26  import org.apache.hadoop.hbase.MetaTableAccessor;
27  import org.apache.hadoop.hbase.MiniHBaseCluster;
28  import org.apache.hadoop.hbase.TableName;
29  import org.apache.hadoop.hbase.client.Admin;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.master.HMaster;
33  import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor;
34  import org.apache.hadoop.hbase.quotas.QuotaUtil;
35  import org.apache.hadoop.hbase.regionserver.HRegion;
36  import org.apache.hadoop.hbase.regionserver.Region;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  
45  import java.io.IOException;
46  import java.util.Collections;
47  import java.util.Comparator;
48  import java.util.List;
49  
50  import static org.junit.Assert.assertEquals;
51  
52  /**
53   * Testing {@link SimpleRegionNormalizer} on minicluster.
54   */
55  @Category(MediumTests.class)
56  public class TestSimpleRegionNormalizerOnCluster {
57    private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizerOnCluster.class);
58    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59    private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
60    private static Admin admin;
61  
62    @BeforeClass
63    public static void beforeAllTests() throws Exception {
64      // we will retry operations when PleaseHoldException is thrown
65      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
66      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
67  
68      // Start a cluster of two regionservers.
69      TEST_UTIL.startMiniCluster(1);
70      TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL);
71      admin = TEST_UTIL.getHBaseAdmin();
72    }
73  
74    @AfterClass
75    public static void afterAllTests() throws Exception {
76      TEST_UTIL.shutdownMiniCluster();
77    }
78  
79    @Test(timeout = 60000)
80    @SuppressWarnings("deprecation")
81    public void testRegionNormalizationSplitOnCluster() throws Exception {
82      final TableName TABLENAME =
83        TableName.valueOf("testRegionNormalizationSplitOnCluster");
84      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
85      HMaster m = cluster.getMaster();
86  
87      try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
88        // Need to get sorted list of regions here
89        List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
90        Collections.sort(generatedRegions, new Comparator<HRegion>() {
91          @Override
92          public int compare(HRegion o1, HRegion o2) {
93            return o1.getRegionInfo().compareTo(o2.getRegionInfo());
94          }
95        });
96  
97        HRegion region = generatedRegions.get(0);
98        generateTestData(region, 1);
99        region.flush(true);
100 
101       region = generatedRegions.get(1);
102       generateTestData(region, 1);
103       region.flush(true);
104 
105       region = generatedRegions.get(2);
106       generateTestData(region, 2);
107       region.flush(true);
108 
109       region = generatedRegions.get(3);
110       generateTestData(region, 2);
111       region.flush(true);
112 
113       region = generatedRegions.get(4);
114       generateTestData(region, 5);
115       region.flush(true);
116     }
117 
118     HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
119     htd.setNormalizationEnabled(true);
120     admin.modifyTable(TABLENAME, htd);
121 
122     admin.flush(TABLENAME);
123 
124     System.out.println(admin.getTableDescriptor(TABLENAME));
125 
126     assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
127 
128     // Now trigger a split and stop when the split is in progress
129     Thread.sleep(5000); // to let region load to update
130     m.normalizeRegions();
131 
132     while (true) {
133       List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
134       int cnt = 0;
135       for (HRegion region : regions) {
136         String regionName = region.getRegionInfo().getRegionNameAsString();
137         if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
138           cnt++;
139         }
140       }
141       if (cnt >= 2) {
142         break;
143       }
144     }
145     admin.disableTable(TABLENAME);
146     admin.deleteTable(TABLENAME);
147   }
148 
149   @Test(timeout = 60000)
150   @SuppressWarnings("deprecation")
151   public void testRegionNormalizationMergeOnCluster() throws Exception {
152     final TableName TABLENAME =
153       TableName.valueOf("testRegionNormalizationMergeOnCluster");
154     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
155     HMaster m = cluster.getMaster();
156 
157     // create 5 regions with sizes to trigger merge of small regions
158     try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
159       // Need to get sorted list of regions here
160       List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
161       Collections.sort(generatedRegions, new Comparator<HRegion>() {
162         @Override
163         public int compare(HRegion o1, HRegion o2) {
164           return o1.getRegionInfo().compareTo(o2.getRegionInfo());
165         }
166       });
167 
168       HRegion region = generatedRegions.get(0);
169       generateTestData(region, 1);
170       region.flush(true);
171 
172       region = generatedRegions.get(1);
173       generateTestData(region, 1);
174       region.flush(true);
175 
176       region = generatedRegions.get(2);
177       generateTestData(region, 3);
178       region.flush(true);
179 
180       region = generatedRegions.get(3);
181       generateTestData(region, 3);
182       region.flush(true);
183 
184       region = generatedRegions.get(4);
185       generateTestData(region, 5);
186       region.flush(true);
187     }
188 
189     HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
190     htd.setNormalizationEnabled(true);
191     admin.modifyTable(TABLENAME, htd);
192 
193     admin.flush(TABLENAME);
194 
195     assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
196 
197     // Now trigger a merge and stop when the merge is in progress
198     Thread.sleep(5000); // to let region load to update
199     m.normalizeRegions();
200 
201     while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) > 4) {
202       LOG.info("Waiting for normalization merge to complete");
203       Thread.sleep(100);
204     }
205 
206     assertEquals(4, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
207 
208     admin.disableTable(TABLENAME);
209     admin.deleteTable(TABLENAME);
210   }
211 
212   private void generateTestData(Region region, int numRows) throws IOException {
213     // generating 1Mb values
214     LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024);
215     for (int i = 0; i < numRows; ++i) {
216       byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i));
217       for (int j = 0; j < 1; ++j) {
218         Put put = new Put(key);
219         byte[] col = Bytes.toBytes(String.valueOf(j));
220         byte[] value = dataGenerator.generateRandomSizeValue(key, col);
221         put.add(FAMILYNAME, col, value);
222         region.put(put);
223       }
224     }
225   }
226 }