1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.quotas;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertNotNull;
21 import static org.mockito.Matchers.any;
22 import static org.mockito.Mockito.doAnswer;
23 import static org.mockito.Mockito.mock;
24 import static org.mockito.Mockito.when;
25
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.Collections;
30 import java.util.Iterator;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.concurrent.TimeUnit;
34
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.HBaseConfiguration;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.regionserver.HRegionServer;
39 import org.apache.hadoop.hbase.regionserver.Region;
40 import org.apache.hadoop.hbase.regionserver.Store;
41 import org.apache.hadoop.hbase.testclassification.SmallTests;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.mockito.invocation.InvocationOnMock;
45 import org.mockito.stubbing.Answer;
46
47
48
49
50 @Category(SmallTests.class)
51 public class TestFileSystemUtilizationChore {
52
53 @SuppressWarnings("unchecked")
54 @Test
55 public void testNoOnlineRegions() {
56
57 final List<Long> regionSizes = Collections.emptyList();
58 final Configuration conf = getDefaultHBaseConfiguration();
59 final HRegionServer rs = mockRegionServer(conf);
60 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
61 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(regionSizes)))
62 .when(rs)
63 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
64
65 final Region region = mockRegionWithSize(regionSizes);
66 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(region));
67 chore.chore();
68 }
69
70 @SuppressWarnings("unchecked")
71 @Test
72 public void testRegionSizes() {
73
74 final List<Long> regionSizes = Arrays.asList(1024L);
75 final Configuration conf = getDefaultHBaseConfiguration();
76 final HRegionServer rs = mockRegionServer(conf);
77 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
78 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(regionSizes)))
79 .when(rs)
80 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
81
82 final Region region = mockRegionWithSize(regionSizes);
83 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(region));
84 chore.chore();
85 }
86
87 @SuppressWarnings("unchecked")
88 @Test
89 public void testMultipleRegionSizes() {
90 final Configuration conf = getDefaultHBaseConfiguration();
91 final HRegionServer rs = mockRegionServer(conf);
92
93
94 final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
95 final long r1Sum = sum(r1Sizes);
96 final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
97 final long r2Sum = sum(r2Sizes);
98 final List<Long> r3Sizes = Arrays.asList(10L * 1024L * 1024L);
99 final long r3Sum = sum(r3Sizes);
100
101 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
102 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum, r2Sum, r3Sum))))
103 .when(rs)
104 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
105
106 final Region r1 = mockRegionWithSize(r1Sizes);
107 final Region r2 = mockRegionWithSize(r2Sizes);
108 final Region r3 = mockRegionWithSize(r3Sizes);
109 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3));
110 chore.chore();
111 }
112
113 @Test
114 public void testDefaultConfigurationProperties() {
115 final Configuration conf = getDefaultHBaseConfiguration();
116 final HRegionServer rs = mockRegionServer(conf);
117 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
118
119 assertEquals(
120 FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_DEFAULT, chore.getPeriod());
121 assertEquals(
122 FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_DEFAULT, chore.getInitialDelay());
123 assertEquals(
124 TimeUnit.valueOf(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_TIMEUNIT_DEFAULT),
125 chore.getTimeUnit());
126 }
127
128 @Test
129 public void testNonDefaultConfigurationProperties() {
130 final Configuration conf = getDefaultHBaseConfiguration();
131
132 final int period = 60 * 10;
133 final long delay = 30L;
134 final TimeUnit timeUnit = TimeUnit.SECONDS;
135 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, period);
136 conf.setLong(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, delay);
137 conf.set(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_TIMEUNIT_KEY, timeUnit.name());
138
139
140 final HRegionServer rs = mockRegionServer(conf);
141 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
142 assertEquals(period, chore.getPeriod());
143 assertEquals(delay, chore.getInitialDelay());
144 assertEquals(timeUnit, chore.getTimeUnit());
145 }
146
147 @SuppressWarnings("unchecked")
148 @Test
149 public void testProcessingLeftoverRegions() {
150 final Configuration conf = getDefaultHBaseConfiguration();
151 final HRegionServer rs = mockRegionServer(conf);
152
153
154 final List<Long> leftover1Sizes = Arrays.asList(1024L, 4096L);
155 final long leftover1Sum = sum(leftover1Sizes);
156 final List<Long> leftover2Sizes = Arrays.asList(2048L);
157 final long leftover2Sum = sum(leftover2Sizes);
158
159 final Region lr1 = mockRegionWithSize(leftover1Sizes);
160 final Region lr2 = mockRegionWithSize(leftover2Sizes);
161 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs) {
162 @Override
163 Iterator<Region> getLeftoverRegions() {
164 return Arrays.asList(lr1, lr2).iterator();
165 }
166 };
167 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(leftover1Sum, leftover2Sum))))
168 .when(rs)
169 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
170
171
172 final Region r1 = mockRegionWithSize(Arrays.asList(1024L, 2048L));
173 final Region r2 = mockRegionWithSize(Arrays.asList(1024L * 1024L));
174 final Region r3 = mockRegionWithSize(Arrays.asList(10L * 1024L * 1024L));
175 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3, lr1, lr2));
176
177 chore.chore();
178 }
179
180 @SuppressWarnings("unchecked")
181 @Test
182 public void testProcessingNowOfflineLeftoversAreIgnored() {
183 final Configuration conf = getDefaultHBaseConfiguration();
184 final HRegionServer rs = mockRegionServer(conf);
185
186
187 final List<Long> leftover1Sizes = Arrays.asList(1024L, 4096L);
188 final long leftover1Sum = sum(leftover1Sizes);
189 final List<Long> leftover2Sizes = Arrays.asList(2048L);
190
191 final Region lr1 = mockRegionWithSize(leftover1Sizes);
192 final Region lr2 = mockRegionWithSize(leftover2Sizes);
193 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs) {
194 @Override
195 Iterator<Region> getLeftoverRegions() {
196 return Arrays.asList(lr1, lr2).iterator();
197 }
198 };
199 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(leftover1Sum))))
200 .when(rs)
201 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
202
203
204 final Region r1 = mockRegionWithSize(Arrays.asList(1024L, 2048L));
205 final Region r2 = mockRegionWithSize(Arrays.asList(1024L * 1024L));
206 final Region r3 = mockRegionWithSize(Arrays.asList(10L * 1024L * 1024L));
207
208 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3, lr1));
209
210 chore.chore();
211 }
212
213 @SuppressWarnings("unchecked")
214 @Test
215 public void testIgnoreSplitParents() {
216 final Configuration conf = getDefaultHBaseConfiguration();
217 final HRegionServer rs = mockRegionServer(conf);
218
219
220 final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
221 final long r1Sum = sum(r1Sizes);
222 final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
223
224 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
225 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum))))
226 .when(rs)
227 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
228
229 final Region r1 = mockRegionWithSize(r1Sizes);
230 final Region r2 = mockSplitParentRegionWithSize(r2Sizes);
231 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2));
232 chore.chore();
233 }
234
235 @SuppressWarnings("unchecked")
236 @Test
237 public void testIgnoreRegionReplicas() {
238 final Configuration conf = getDefaultHBaseConfiguration();
239 final HRegionServer rs = mockRegionServer(conf);
240
241
242 final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
243 final long r1Sum = sum(r1Sizes);
244 final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
245
246 final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
247 doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum))))
248 .when(rs)
249 .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
250
251 final Region r1 = mockRegionWithSize(r1Sizes);
252 final Region r2 = mockRegionReplicaWithSize(r2Sizes);
253 when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2));
254 chore.chore();
255 }
256
257
258
259
260 private Configuration getDefaultHBaseConfiguration() {
261 final Configuration conf = HBaseConfiguration.create();
262 conf.addResource("hbase-default.xml");
263 return conf;
264 }
265
266
267
268
269 private HRegionServer mockRegionServer(Configuration conf) {
270 final HRegionServer rs = mock(HRegionServer.class);
271 when(rs.getConfiguration()).thenReturn(conf);
272 return rs;
273 }
274
275
276
277
278 private long sum(Collection<Long> values) {
279 long sum = 0L;
280 for (Long value : values) {
281 assertNotNull(value);
282 sum += value;
283 }
284 return sum;
285 }
286
287
288
289
290
291
292
293
294 private Region mockRegionWithSize(Collection<Long> storeSizes) {
295 final Region r = mock(Region.class);
296 final HRegionInfo info = mock(HRegionInfo.class);
297 when(r.getRegionInfo()).thenReturn(info);
298 List<Store> stores = new ArrayList<>();
299 when(r.getStores()).thenReturn(stores);
300 for (Long storeSize : storeSizes) {
301 final Store s = mock(Store.class);
302 stores.add(s);
303 when(s.getStorefilesSize()).thenReturn(storeSize);
304 }
305 return r;
306 }
307
308
309
310
311
312
313
314 private Region mockSplitParentRegionWithSize(Collection<Long> storeSizes) {
315 final Region r = mockRegionWithSize(storeSizes);
316 final HRegionInfo info = r.getRegionInfo();
317 when(info.isSplitParent()).thenReturn(true);
318 return r;
319 }
320
321
322
323
324
325
326
327 private Region mockRegionReplicaWithSize(Collection<Long> storeSizes) {
328 final Region r = mockRegionWithSize(storeSizes);
329 final HRegionInfo info = r.getRegionInfo();
330 when(info.getReplicaId()).thenReturn(1);
331 return r;
332 }
333
334
335
336
337 private static class ExpectedRegionSizeSummationAnswer implements Answer<Void> {
338 private final long expectedSize;
339
340 public ExpectedRegionSizeSummationAnswer(long expectedSize) {
341 this.expectedSize = expectedSize;
342 }
343
344 @Override
345 public Void answer(InvocationOnMock invocation) throws Throwable {
346 Object[] args = invocation.getArguments();
347 assertEquals(1, args.length);
348 @SuppressWarnings("unchecked")
349 Map<HRegionInfo,Long> regionSizes = (Map<HRegionInfo,Long>) args[0];
350 long sum = 0L;
351 for (Long regionSize : regionSizes.values()) {
352 sum += regionSize;
353 }
354 assertEquals(expectedSize, sum);
355 return null;
356 }
357 }
358 }