View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.*;
32  import org.apache.hadoop.hbase.client.Admin;
33  import org.apache.hadoop.hbase.client.Connection;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.client.HConnectionManager;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Durability;
38  import org.apache.hadoop.hbase.regionserver.HRegion;
39  import org.apache.hadoop.hbase.testclassification.MediumTests;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  /**
44   * Tests merging a normal table's regions
45   */
46  @Category(MediumTests.class)
47  public class TestMergeTable {
48    private static final Log LOG = LogFactory.getLog(TestMergeTable.class);
49    private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
50    private static final byte [] COLUMN_NAME = Bytes.toBytes("contents");
51    private static final byte [] VALUE;
52    static {
53      // We will use the same value for the rows as that is not really important here
54      String partialValue = String.valueOf(System.currentTimeMillis());
55      StringBuilder val = new StringBuilder();
56      while (val.length() < 1024) {
57        val.append(partialValue);
58      }
59      VALUE = Bytes.toBytes(val.toString());
60    }
61  
62    /**
63     * Test merge.
64     * Hand-makes regions of a mergeable size and adds the hand-made regions to
65     * hand-made meta.  The hand-made regions are created offline.  We then start
66     * up mini cluster, disables the hand-made table and starts in on merging.
67     * @throws Exception
68     */
69    @Test (timeout=300000) public void testMergeTable() throws Exception {
70      // Table we are manually creating offline.
71      HTableDescriptor desc = new HTableDescriptor(org.apache.hadoop.hbase.TableName.valueOf(Bytes.toBytes("test")));
72      desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
73  
74      // Set maximum regionsize down.
75      UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, 64L * 1024L * 1024L);
76      // Make it so we don't split.
77      UTIL.getConfiguration().setInt("hbase.regionserver.regionSplitLimit", 0);
78      // Startup hdfs.  Its in here we'll be putting our manually made regions.
79      UTIL.startMiniDFSCluster(1);
80      // Create hdfs hbase rootdir.
81      Path rootdir = UTIL.createRootDir();
82      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
83      if (fs.exists(rootdir)) {
84        if (fs.delete(rootdir, true)) {
85          LOG.info("Cleaned up existing " + rootdir);
86        }
87      }
88  
89      // Now create three data regions: The first is too large to merge since it
90      // will be > 64 MB in size. The second two will be smaller and will be
91      // selected for merging.
92  
93      // To ensure that the first region is larger than 64MB we need to write at
94      // least 65536 rows. We will make certain by writing 70000
95      byte [] row_70001 = Bytes.toBytes("row_70001");
96      byte [] row_80001 = Bytes.toBytes("row_80001");
97  
98      // Create regions and populate them at same time.  Create the tabledir
99      // for them first.
100     new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir).createTableDescriptor(desc);
101     HRegion [] regions = {
102       createRegion(desc, null, row_70001, 1, 70000, rootdir),
103       createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir),
104       createRegion(desc, row_80001, null, 80001, 11000, rootdir)
105     };
106 
107     // Now create the root and meta regions and insert the data regions
108     // created above into hbase:meta
109     setupMeta(rootdir, regions);
110     try {
111       LOG.info("Starting mini zk cluster");
112       UTIL.startMiniZKCluster();
113       LOG.info("Starting mini hbase cluster");
114       UTIL.startMiniHBaseCluster(1, 1);
115       Configuration c = new Configuration(UTIL.getConfiguration());
116       Connection connection = HConnectionManager.getConnection(c);
117 
118       List<HRegionInfo> originalTableRegions =
119         MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(), connection,
120           desc.getTableName());
121       LOG.info("originalTableRegions size=" + originalTableRegions.size() +
122         "; " + originalTableRegions);
123       Admin admin = new HBaseAdmin(c);
124       admin.disableTable(desc.getTableName());
125       HMerge.merge(c, FileSystem.get(c), desc.getTableName());
126       List<HRegionInfo> postMergeTableRegions =
127         MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(), connection,
128           desc.getTableName());
129       LOG.info("postMergeTableRegions size=" + postMergeTableRegions.size() +
130         "; " + postMergeTableRegions);
131       assertTrue("originalTableRegions=" + originalTableRegions.size() +
132         ", postMergeTableRegions=" + postMergeTableRegions.size(),
133         postMergeTableRegions.size() < originalTableRegions.size());
134       LOG.info("Done with merge");
135     } finally {
136       UTIL.shutdownMiniCluster();
137       LOG.info("After cluster shutdown");
138     }
139   }
140 
141   private HRegion createRegion(final HTableDescriptor desc,
142       byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)
143   throws IOException {
144     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
145     HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration(), desc);
146     LOG.info("Created region " + region.getRegionInfo().getRegionNameAsString());
147     for(int i = firstRow; i < firstRow + nrows; i++) {
148       Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i)));
149       put.setDurability(Durability.SKIP_WAL);
150       put.add(COLUMN_NAME, null,  VALUE);
151       region.put(put);
152       if (i % 10000 == 0) {
153         LOG.info("Flushing write #" + i);
154         region.flush(true);
155       }
156     }
157     HRegion.closeHRegion(region);
158     return region;
159   }
160 
161   protected void setupMeta(Path rootdir, final HRegion [] regions)
162   throws IOException {
163     HRegion meta =
164       HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, rootdir,
165       UTIL.getConfiguration(), UTIL.getMetaTableDescriptor());
166     for (HRegion r: regions) {
167       HRegion.addRegionToMETA(meta, r);
168     }
169     HRegion.closeHRegion(meta);
170   }
171 
172 }
173