View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to you under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.hadoop.hbase.quotas;
18  
19  import static org.junit.Assert.assertTrue;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Map.Entry;
26  import java.util.Random;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.MiniHBaseCluster;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.client.Admin;
38  import org.apache.hadoop.hbase.client.Connection;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.Table;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.testclassification.MediumTests;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.After;
45  import org.junit.Before;
46  import org.junit.Rule;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  import org.junit.rules.TestName;
50  
51  /**
52   * Test class which verifies that region sizes are reported to the master.
53   */
54  @Category(MediumTests.class)
55  public class TestRegionSizeUse {
56    private static final Log LOG = LogFactory.getLog(TestRegionSizeUse.class);
57    private static final int SIZE_PER_VALUE = 256;
58    private static final int NUM_SPLITS = 10;
59    private static final String F1 = "f1";
60    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61  
62    private MiniHBaseCluster cluster;
63  
64    @Rule
65    public TestName testName = new TestName();
66  
67    @Before
68    public void setUp() throws Exception {
69      Configuration conf = TEST_UTIL.getConfiguration();
70      conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
71      conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
72      conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
73      cluster = TEST_UTIL.startMiniCluster(2);
74    }
75  
76    @After
77    public void tearDown() throws Exception {
78      TEST_UTIL.shutdownMiniCluster();
79    }
80  
81    @Test
82    public void testBasicRegionSizeReports() throws Exception {
83      final long bytesWritten = 5L * 1024L * 1024L; // 5MB
84      final TableName tn = writeData(bytesWritten);
85      LOG.debug("Data was written to HBase");
86      final Admin admin = TEST_UTIL.getHBaseAdmin();
87      // Push the data to disk.
88      admin.flush(tn);
89      LOG.debug("Data flushed to disk");
90      // Get the final region distribution
91      final List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(tn);
92  
93      HMaster master = cluster.getMaster();
94      MasterQuotaManager quotaManager = master.getMasterQuotaManager();
95      Map<HRegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes();
96      // Wait until we get all of the region reports for our table
97      // The table may split, so make sure we have at least as many as expected right after we
98      // finished writing the data.
99      int observedRegions = numRegionsForTable(tn, regionSizes);
100     while (observedRegions < regions.size()) {
101       LOG.debug("Expecting more regions. Saw " + observedRegions
102           + " region sizes reported, expected at least " + regions.size());
103       Thread.sleep(1000);
104       regionSizes = quotaManager.snapshotRegionSizes();
105       observedRegions = numRegionsForTable(tn, regionSizes);
106     }
107 
108     LOG.debug("Observed region sizes by the HMaster: " + regionSizes);
109     long totalRegionSize = 0L;
110     for (Long regionSize : regionSizes.values()) {
111       totalRegionSize += regionSize;
112     }
113     assertTrue("Expected region size report to exceed " + bytesWritten + ", but was "
114         + totalRegionSize + ". RegionSizes=" + regionSizes, bytesWritten < totalRegionSize);
115   }
116 
117   /**
118    * Writes at least {@code sizeInBytes} bytes of data to HBase and returns the TableName used.
119    *
120    * @param sizeInBytes The amount of data to write in bytes.
121    * @return The table the data was written to
122    */
123   private TableName writeData(long sizeInBytes) throws IOException {
124     final Connection conn = TEST_UTIL.getConnection();
125     final Admin admin = TEST_UTIL.getHBaseAdmin();
126     final TableName tn = TableName.valueOf(testName.getMethodName());
127 
128     // Delete the old table
129     if (admin.tableExists(tn)) {
130       admin.disableTable(tn);
131       admin.deleteTable(tn);
132     }
133 
134     // Create the table
135     HTableDescriptor tableDesc = new HTableDescriptor(tn);
136     tableDesc.addFamily(new HColumnDescriptor(F1));
137     admin.createTable(tableDesc, Bytes.toBytes("1"), Bytes.toBytes("9"), NUM_SPLITS);
138 
139     final Table table = conn.getTable(tn);
140     try {
141       List<Put> updates = new ArrayList<>();
142       long bytesToWrite = sizeInBytes;
143       long rowKeyId = 0L;
144       final StringBuilder sb = new StringBuilder();
145       final Random r = new Random();
146       while (bytesToWrite > 0L) {
147         sb.setLength(0);
148         sb.append(Long.toString(rowKeyId));
149         // Use the reverse counter as the rowKey to get even spread across all regions
150         Put p = new Put(Bytes.toBytes(sb.reverse().toString()));
151         byte[] value = new byte[SIZE_PER_VALUE];
152         r.nextBytes(value);
153         p.addColumn(Bytes.toBytes(F1), Bytes.toBytes("q1"), value);
154         updates.add(p);
155 
156         // Batch 50K worth of updates
157         if (updates.size() > 50) {
158           table.put(updates);
159           updates.clear();
160         }
161 
162         // Just count the value size, ignore the size of rowkey + column
163         bytesToWrite -= SIZE_PER_VALUE;
164         rowKeyId++;
165       }
166 
167       // Write the final batch
168       if (!updates.isEmpty()) {
169         table.put(updates);
170       }
171 
172       return tn;
173     } finally {
174       table.close();
175     }
176   }
177 
178   /**
179    * Computes the number of regions for the given table that have a positive size.
180    *
181    * @param tn The TableName in question
182    * @param regions A collection of region sizes
183    * @return The number of regions for the given table.
184    */
185   private int numRegionsForTable(TableName tn, Map<HRegionInfo,Long> regions) {
186     int sum = 0;
187     for (Entry<HRegionInfo,Long> entry : regions.entrySet()) {
188       if (tn.equals(entry.getKey().getTable()) && 0 < entry.getValue()) {
189         sum++;
190       }
191     }
192     return sum;
193   }
194 }