1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.quotas;
18
19 import static org.junit.Assert.assertTrue;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Map.Entry;
26 import java.util.Random;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.MiniHBaseCluster;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.client.Admin;
38 import org.apache.hadoop.hbase.client.Connection;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Table;
41 import org.apache.hadoop.hbase.master.HMaster;
42 import org.apache.hadoop.hbase.testclassification.MediumTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.After;
45 import org.junit.Before;
46 import org.junit.Rule;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49 import org.junit.rules.TestName;
50
51
52
53
54 @Category(MediumTests.class)
55 public class TestRegionSizeUse {
56 private static final Log LOG = LogFactory.getLog(TestRegionSizeUse.class);
57 private static final int SIZE_PER_VALUE = 256;
58 private static final int NUM_SPLITS = 10;
59 private static final String F1 = "f1";
60 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61
62 private MiniHBaseCluster cluster;
63
64 @Rule
65 public TestName testName = new TestName();
66
67 @Before
68 public void setUp() throws Exception {
69 Configuration conf = TEST_UTIL.getConfiguration();
70 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
71 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
72 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
73 cluster = TEST_UTIL.startMiniCluster(2);
74 }
75
76 @After
77 public void tearDown() throws Exception {
78 TEST_UTIL.shutdownMiniCluster();
79 }
80
81 @Test
82 public void testBasicRegionSizeReports() throws Exception {
83 final long bytesWritten = 5L * 1024L * 1024L;
84 final TableName tn = writeData(bytesWritten);
85 LOG.debug("Data was written to HBase");
86 final Admin admin = TEST_UTIL.getHBaseAdmin();
87
88 admin.flush(tn);
89 LOG.debug("Data flushed to disk");
90
91 final List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(tn);
92
93 HMaster master = cluster.getMaster();
94 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
95 Map<HRegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes();
96
97
98
99 int observedRegions = numRegionsForTable(tn, regionSizes);
100 while (observedRegions < regions.size()) {
101 LOG.debug("Expecting more regions. Saw " + observedRegions
102 + " region sizes reported, expected at least " + regions.size());
103 Thread.sleep(1000);
104 regionSizes = quotaManager.snapshotRegionSizes();
105 observedRegions = numRegionsForTable(tn, regionSizes);
106 }
107
108 LOG.debug("Observed region sizes by the HMaster: " + regionSizes);
109 long totalRegionSize = 0L;
110 for (Long regionSize : regionSizes.values()) {
111 totalRegionSize += regionSize;
112 }
113 assertTrue("Expected region size report to exceed " + bytesWritten + ", but was "
114 + totalRegionSize + ". RegionSizes=" + regionSizes, bytesWritten < totalRegionSize);
115 }
116
117
118
119
120
121
122
123 private TableName writeData(long sizeInBytes) throws IOException {
124 final Connection conn = TEST_UTIL.getConnection();
125 final Admin admin = TEST_UTIL.getHBaseAdmin();
126 final TableName tn = TableName.valueOf(testName.getMethodName());
127
128
129 if (admin.tableExists(tn)) {
130 admin.disableTable(tn);
131 admin.deleteTable(tn);
132 }
133
134
135 HTableDescriptor tableDesc = new HTableDescriptor(tn);
136 tableDesc.addFamily(new HColumnDescriptor(F1));
137 admin.createTable(tableDesc, Bytes.toBytes("1"), Bytes.toBytes("9"), NUM_SPLITS);
138
139 final Table table = conn.getTable(tn);
140 try {
141 List<Put> updates = new ArrayList<>();
142 long bytesToWrite = sizeInBytes;
143 long rowKeyId = 0L;
144 final StringBuilder sb = new StringBuilder();
145 final Random r = new Random();
146 while (bytesToWrite > 0L) {
147 sb.setLength(0);
148 sb.append(Long.toString(rowKeyId));
149
150 Put p = new Put(Bytes.toBytes(sb.reverse().toString()));
151 byte[] value = new byte[SIZE_PER_VALUE];
152 r.nextBytes(value);
153 p.addColumn(Bytes.toBytes(F1), Bytes.toBytes("q1"), value);
154 updates.add(p);
155
156
157 if (updates.size() > 50) {
158 table.put(updates);
159 updates.clear();
160 }
161
162
163 bytesToWrite -= SIZE_PER_VALUE;
164 rowKeyId++;
165 }
166
167
168 if (!updates.isEmpty()) {
169 table.put(updates);
170 }
171
172 return tn;
173 } finally {
174 table.close();
175 }
176 }
177
178
179
180
181
182
183
184
185 private int numRegionsForTable(TableName tn, Map<HRegionInfo,Long> regions) {
186 int sum = 0;
187 for (Entry<HRegionInfo,Long> entry : regions.entrySet()) {
188 if (tn.equals(entry.getKey().getTable()) && 0 < entry.getValue()) {
189 sum++;
190 }
191 }
192 return sum;
193 }
194 }