View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Random;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.testclassification.LargeTests;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Table;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /** Unit tests to test retrieving table/region compaction state*/
45  @Category(LargeTests.class)
46  public class TestCompactionState {
47    final static Log LOG = LogFactory.getLog(TestCompactionState.class);
48    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49    private final static Random random = new Random();
50  
51    @BeforeClass
52    public static void setUpBeforeClass() throws Exception {
53      TEST_UTIL.startMiniCluster();
54    }
55  
56    @AfterClass
57    public static void tearDownAfterClass() throws Exception {
58      TEST_UTIL.shutdownMiniCluster();
59    }
60  
61    @Test(timeout=600000)
62    public void testMajorCompaction() throws IOException, InterruptedException {
63      compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
64    }
65  
66    @Test(timeout=600000)
67    public void testMinorCompaction() throws IOException, InterruptedException {
68      compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
69    }
70  
71    @Test(timeout=600000)
72    public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
73      compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
74    }
75  
76    @Test(timeout=600000)
77    public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
78      compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
79    }
80  
81    @Test
82    public void testInvalidColumnFamily() throws IOException, InterruptedException {
83      TableName table = TableName.valueOf("testInvalidColumnFamily");
84      byte [] family = Bytes.toBytes("family");
85      byte [] fakecf = Bytes.toBytes("fakecf");
86      boolean caughtMinorCompact = false;
87      boolean caughtMajorCompact = false;
88      Table ht = null;
89      try {
90        ht = TEST_UTIL.createTable(table, family);
91        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
92        try {
93          admin.compact(table, fakecf);
94        } catch (IOException ioe) {
95          caughtMinorCompact = true;
96        }
97        try {
98          admin.majorCompact(table, fakecf);
99        } catch (IOException ioe) {
100         caughtMajorCompact = true;
101       }
102     } finally {
103       if (ht != null) {
104         TEST_UTIL.deleteTable(table);
105       }
106       assertTrue(caughtMinorCompact);
107       assertTrue(caughtMajorCompact);
108     }
109   }
110 
111   /**
112    * Load data to a table, flush it to disk, trigger compaction,
113    * confirm the compaction state is right and wait till it is done.
114    *
115    * @param tableName
116    * @param flushes
117    * @param expectedState
118    * @param singleFamily otherwise, run compaction on all cfs
119    * @throws IOException
120    * @throws InterruptedException
121    */
122   private void compaction(final String tableName, final int flushes,
123       final CompactionState expectedState, boolean singleFamily)
124       throws IOException, InterruptedException {
125     // Create a table with regions
126     TableName table = TableName.valueOf(tableName);
127     byte [] family = Bytes.toBytes("family");
128     byte [][] families =
129       {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
130     Table ht = null;
131     try {
132       ht = TEST_UTIL.createTable(table, families);
133       loadData(ht, families, 3000, flushes);
134       HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
135       List<Region> regions = rs.getOnlineRegions(table);
136       int countBefore = countStoreFilesInFamilies(regions, families);
137       int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
138       assertTrue(countBefore > 0); // there should be some data files
139       HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
140       if (expectedState == CompactionState.MINOR) {
141         if (singleFamily) {
142           admin.compact(table.getName(), family);
143         } else {
144           admin.compact(table.getName());
145         }
146       } else {
147         if (singleFamily) {
148           admin.majorCompact(table.getName(), family);
149         } else {
150           admin.majorCompact(table.getName());
151         }
152       }
153       long curt = System.currentTimeMillis();
154       long waitTime = 5000;
155       long endt = curt + waitTime;
156       CompactionState state = admin.getCompactionState(table.getName());
157       while (state == CompactionState.NONE && curt < endt) {
158         Thread.sleep(10);
159         state = admin.getCompactionState(table.getName());
160         curt = System.currentTimeMillis();
161       }
162       // Now, should have the right compaction state,
163       // otherwise, the compaction should have already been done
164       if (expectedState != state) {
165         for (Region region: regions) {
166           state = region.getCompactionState();
167           assertEquals(CompactionState.NONE, state);
168         }
169       } else {
170         // Wait until the compaction is done
171         state = admin.getCompactionState(table.getName());
172         while (state != CompactionState.NONE && curt < endt) {
173           Thread.sleep(10);
174           state = admin.getCompactionState(table.getName());
175         }
176         // Now, compaction should be done.
177         assertEquals(CompactionState.NONE, state);
178       }
179       int countAfter = countStoreFilesInFamilies(regions, families);
180       int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
181       assertTrue(countAfter < countBefore);
182       if (!singleFamily) {
183         if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
184         else assertTrue(families.length < countAfter);
185       } else {
186         int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
187         // assert only change was to single column family
188         assertTrue(singleFamDiff == (countBefore - countAfter));
189         if (expectedState == CompactionState.MAJOR) {
190           assertTrue(1 == countAfterSingleFamily);
191         } else {
192           assertTrue(1 < countAfterSingleFamily);
193         }
194       }
195     } finally {
196       if (ht != null) {
197         TEST_UTIL.deleteTable(table);
198       }
199     }
200   }
201 
202   private static int countStoreFilesInFamily(
203       List<Region> regions, final byte[] family) {
204     return countStoreFilesInFamilies(regions, new byte[][]{family});
205   }
206 
207   private static int countStoreFilesInFamilies(List<Region> regions, final byte[][] families) {
208     int count = 0;
209     for (Region region: regions) {
210       count += region.getStoreFileList(families).size();
211     }
212     return count;
213   }
214 
215   private static void loadData(final Table ht, final byte[][] families,
216       final int rows, final int flushes) throws IOException {
217     List<Put> puts = new ArrayList<Put>(rows);
218     byte[] qualifier = Bytes.toBytes("val");
219     for (int i = 0; i < flushes; i++) {
220       for (int k = 0; k < rows; k++) {
221         byte[] row = Bytes.toBytes(random.nextLong());
222         Put p = new Put(row);
223         for (int j = 0; j < families.length; ++j) {
224           p.add(families[ j ], qualifier, row);
225         }
226         puts.add(p);
227       }
228       ht.put(puts);
229       TEST_UTIL.flush();
230       puts.clear();
231     }
232   } 
233 }