View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to you under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.hadoop.hbase.quotas;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertNotNull;
21  import static org.mockito.Matchers.any;
22  import static org.mockito.Mockito.doAnswer;
23  import static org.mockito.Mockito.mock;
24  import static org.mockito.Mockito.when;
25  
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.concurrent.TimeUnit;
34  
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.HBaseConfiguration;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.regionserver.HRegionServer;
39  import org.apache.hadoop.hbase.regionserver.Region;
40  import org.apache.hadoop.hbase.regionserver.Store;
41  import org.apache.hadoop.hbase.testclassification.SmallTests;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.mockito.invocation.InvocationOnMock;
45  import org.mockito.stubbing.Answer;
46  
47  /**
48   * Test class for {@link FileSystemUtilizationChore}.
49   */
50  @Category(SmallTests.class)
51  public class TestFileSystemUtilizationChore {
52  
53    @SuppressWarnings("unchecked")
54    @Test
55    public void testNoOnlineRegions() {
56      // One region with a store size of one.
57      final List<Long> regionSizes = Collections.emptyList();
58      final Configuration conf = getDefaultHBaseConfiguration();
59      final HRegionServer rs = mockRegionServer(conf);
60      final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
61      doAnswer(new ExpectedRegionSizeSummationAnswer(sum(regionSizes)))
62          .when(rs)
63          .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
64  
65      final Region region = mockRegionWithSize(regionSizes);
66      when(rs.getOnlineRegions()).thenReturn(Arrays.asList(region));
67      chore.chore();
68    }
69  
70    @SuppressWarnings("unchecked")
71    @Test
72    public void testRegionSizes() {
73      // One region with a store size of one.
74      final List<Long> regionSizes = Arrays.asList(1024L);
75      final Configuration conf = getDefaultHBaseConfiguration();
76      final HRegionServer rs = mockRegionServer(conf);
77      final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
78      doAnswer(new ExpectedRegionSizeSummationAnswer(sum(regionSizes)))
79          .when(rs)
80          .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
81  
82      final Region region = mockRegionWithSize(regionSizes);
83      when(rs.getOnlineRegions()).thenReturn(Arrays.asList(region));
84      chore.chore();
85    }
86  
87    @SuppressWarnings("unchecked")
88    @Test
89    public void testMultipleRegionSizes() {
90      final Configuration conf = getDefaultHBaseConfiguration();
91      final HRegionServer rs = mockRegionServer(conf);
92  
93      // Three regions with multiple store sizes
94      final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
95      final long r1Sum = sum(r1Sizes);
96      final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
97      final long r2Sum = sum(r2Sizes);
98      final List<Long> r3Sizes = Arrays.asList(10L * 1024L * 1024L);
99      final long r3Sum = sum(r3Sizes);
100 
101     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
102     doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum, r2Sum, r3Sum))))
103         .when(rs)
104         .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
105 
106     final Region r1 = mockRegionWithSize(r1Sizes);
107     final Region r2 = mockRegionWithSize(r2Sizes);
108     final Region r3 = mockRegionWithSize(r3Sizes);
109     when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3));
110     chore.chore();
111   }
112 
113   @Test
114   public void testDefaultConfigurationProperties() {
115     final Configuration conf = getDefaultHBaseConfiguration();
116     final HRegionServer rs = mockRegionServer(conf);
117     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
118     // Verify that the expected default values are actually represented.
119     assertEquals(
120         FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_DEFAULT, chore.getPeriod());
121     assertEquals(
122         FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_DEFAULT, chore.getInitialDelay());
123     assertEquals(
124         TimeUnit.valueOf(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_TIMEUNIT_DEFAULT),
125         chore.getTimeUnit());
126   }
127 
128   @Test
129   public void testNonDefaultConfigurationProperties() {
130     final Configuration conf = getDefaultHBaseConfiguration();
131     // Override the default values
132     final int period = 60 * 10;
133     final long delay = 30L;
134     final TimeUnit timeUnit = TimeUnit.SECONDS;
135     conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, period);
136     conf.setLong(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, delay);
137     conf.set(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_TIMEUNIT_KEY, timeUnit.name());
138 
139     // Verify that the chore reports these non-default values
140     final HRegionServer rs = mockRegionServer(conf);
141     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
142     assertEquals(period, chore.getPeriod());
143     assertEquals(delay, chore.getInitialDelay());
144     assertEquals(timeUnit, chore.getTimeUnit());
145   }
146 
147   @SuppressWarnings("unchecked")
148   @Test
149   public void testProcessingLeftoverRegions() {
150     final Configuration conf = getDefaultHBaseConfiguration();
151     final HRegionServer rs = mockRegionServer(conf);
152 
153     // Some leftover regions from a previous chore()
154     final List<Long> leftover1Sizes = Arrays.asList(1024L, 4096L);
155     final long leftover1Sum = sum(leftover1Sizes);
156     final List<Long> leftover2Sizes = Arrays.asList(2048L);
157     final long leftover2Sum = sum(leftover2Sizes);
158 
159     final Region lr1 = mockRegionWithSize(leftover1Sizes);
160     final Region lr2 = mockRegionWithSize(leftover2Sizes);
161     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs) {
162       @Override
163       Iterator<Region> getLeftoverRegions() {
164         return Arrays.asList(lr1, lr2).iterator();
165       }
166     };
167     doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(leftover1Sum, leftover2Sum))))
168         .when(rs)
169         .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
170 
171     // We shouldn't compute all of these region sizes, just the leftovers
172     final Region r1 = mockRegionWithSize(Arrays.asList(1024L, 2048L));
173     final Region r2 = mockRegionWithSize(Arrays.asList(1024L * 1024L));
174     final Region r3 = mockRegionWithSize(Arrays.asList(10L * 1024L * 1024L));
175     when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3, lr1, lr2));
176 
177     chore.chore();
178   }
179 
180   @SuppressWarnings("unchecked")
181   @Test
182   public void testProcessingNowOfflineLeftoversAreIgnored() {
183     final Configuration conf = getDefaultHBaseConfiguration();
184     final HRegionServer rs = mockRegionServer(conf);
185 
186     // Some leftover regions from a previous chore()
187     final List<Long> leftover1Sizes = Arrays.asList(1024L, 4096L);
188     final long leftover1Sum = sum(leftover1Sizes);
189     final List<Long> leftover2Sizes = Arrays.asList(2048L);
190 
191     final Region lr1 = mockRegionWithSize(leftover1Sizes);
192     final Region lr2 = mockRegionWithSize(leftover2Sizes);
193     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs) {
194       @Override
195       Iterator<Region> getLeftoverRegions() {
196         return Arrays.asList(lr1, lr2).iterator();
197       }
198     };
199     doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(leftover1Sum))))
200         .when(rs)
201         .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
202 
203     // We shouldn't compute all of these region sizes, just the leftovers
204     final Region r1 = mockRegionWithSize(Arrays.asList(1024L, 2048L));
205     final Region r2 = mockRegionWithSize(Arrays.asList(1024L * 1024L));
206     final Region r3 = mockRegionWithSize(Arrays.asList(10L * 1024L * 1024L));
207     // lr2 is no longer online, so it should be ignored
208     when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2, r3, lr1));
209 
210     chore.chore();
211   }
212 
213   @SuppressWarnings("unchecked")
214   @Test
215   public void testIgnoreSplitParents() {
216     final Configuration conf = getDefaultHBaseConfiguration();
217     final HRegionServer rs = mockRegionServer(conf);
218 
219     // Three regions with multiple store sizes
220     final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
221     final long r1Sum = sum(r1Sizes);
222     final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
223 
224     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
225     doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum))))
226         .when(rs)
227         .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
228 
229     final Region r1 = mockRegionWithSize(r1Sizes);
230     final Region r2 = mockSplitParentRegionWithSize(r2Sizes);
231     when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2));
232     chore.chore();
233   }
234 
235   @SuppressWarnings("unchecked")
236   @Test
237   public void testIgnoreRegionReplicas() {
238     final Configuration conf = getDefaultHBaseConfiguration();
239     final HRegionServer rs = mockRegionServer(conf);
240 
241     // Three regions with multiple store sizes
242     final List<Long> r1Sizes = Arrays.asList(1024L, 2048L);
243     final long r1Sum = sum(r1Sizes);
244     final List<Long> r2Sizes = Arrays.asList(1024L * 1024L);
245 
246     final FileSystemUtilizationChore chore = new FileSystemUtilizationChore(rs);
247     doAnswer(new ExpectedRegionSizeSummationAnswer(sum(Arrays.asList(r1Sum))))
248         .when(rs)
249         .reportRegionSizesForQuotas((Map<HRegionInfo,Long>) any(Map.class));
250 
251     final Region r1 = mockRegionWithSize(r1Sizes);
252     final Region r2 = mockRegionReplicaWithSize(r2Sizes);
253     when(rs.getOnlineRegions()).thenReturn(Arrays.asList(r1, r2));
254     chore.chore();
255   }
256 
257   /**
258    * Creates an HBase Configuration object for the default values.
259    */
260   private Configuration getDefaultHBaseConfiguration() {
261     final Configuration conf = HBaseConfiguration.create();
262     conf.addResource("hbase-default.xml");
263     return conf;
264   }
265 
266   /**
267    * Creates an HRegionServer using the given Configuration.
268    */
269   private HRegionServer mockRegionServer(Configuration conf) {
270     final HRegionServer rs = mock(HRegionServer.class);
271     when(rs.getConfiguration()).thenReturn(conf);
272     return rs;
273   }
274 
275   /**
276    * Sums the collection of non-null numbers.
277    */
278   private long sum(Collection<Long> values) {
279     long sum = 0L;
280     for (Long value : values) {
281       assertNotNull(value);
282       sum += value;
283     }
284     return sum;
285   }
286 
287   /**
288    * Creates a region with a number of Stores equal to the length of {@code storeSizes}. Each
289    * {@link Store} will have a reported size corresponding to the element in {@code storeSizes}.
290    *
291    * @param storeSizes A list of sizes for each Store.
292    * @return A mocked Region.
293    */
294   private Region mockRegionWithSize(Collection<Long> storeSizes) {
295     final Region r = mock(Region.class);
296     final HRegionInfo info = mock(HRegionInfo.class);
297     when(r.getRegionInfo()).thenReturn(info);
298     List<Store> stores = new ArrayList<>();
299     when(r.getStores()).thenReturn(stores);
300     for (Long storeSize : storeSizes) {
301       final Store s = mock(Store.class);
302       stores.add(s);
303       when(s.getStorefilesSize()).thenReturn(storeSize);
304     }
305     return r;
306   }
307 
308   /**
309    * Creates a region which is the parent of a split.
310    *
311    * @param storeSizes A list of sizes for each Store.
312    * @return A mocked Region.
313    */
314   private Region mockSplitParentRegionWithSize(Collection<Long> storeSizes) {
315     final Region r = mockRegionWithSize(storeSizes);
316     final HRegionInfo info = r.getRegionInfo();
317     when(info.isSplitParent()).thenReturn(true);
318     return r;
319   }
320 
321   /**
322    * Creates a region who has a replicaId of <code>1</code>.
323    *
324    * @param storeSizes A list of sizes for each Store.
325    * @return A mocked Region.
326    */
327   private Region mockRegionReplicaWithSize(Collection<Long> storeSizes) {
328     final Region r = mockRegionWithSize(storeSizes);
329     final HRegionInfo info = r.getRegionInfo();
330     when(info.getReplicaId()).thenReturn(1);
331     return r;
332   }
333 
334   /**
335    * An Answer implementation which verifies the sum of the Region sizes to report is as expected.
336    */
337   private static class ExpectedRegionSizeSummationAnswer implements Answer<Void> {
338     private final long expectedSize;
339 
340     public ExpectedRegionSizeSummationAnswer(long expectedSize) {
341       this.expectedSize = expectedSize;
342     }
343 
344     @Override
345     public Void answer(InvocationOnMock invocation) throws Throwable {
346       Object[] args = invocation.getArguments();
347       assertEquals(1, args.length);
348       @SuppressWarnings("unchecked")
349       Map<HRegionInfo,Long> regionSizes = (Map<HRegionInfo,Long>) args[0];
350       long sum = 0L;
351       for (Long regionSize : regionSizes.values()) {
352         sum += regionSize;
353       }
354       assertEquals(expectedSize, sum);
355       return null;
356     }
357   }
358 }