View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mob;
20  
21  import java.io.IOException;
22  import java.util.Date;
23  
24  import junit.framework.TestCase;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.regionserver.HMobStore;
39  import org.apache.hadoop.hbase.regionserver.HRegion;
40  import org.apache.hadoop.hbase.regionserver.StoreFile;
41  import org.apache.hadoop.hbase.testclassification.SmallTests;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  @Category(SmallTests.class)
47  public class TestMobFileCache extends TestCase {
48    static final Log LOG = LogFactory.getLog(TestMobFileCache.class);
49    private HBaseTestingUtility UTIL;
50    private HRegion region;
51    private Configuration conf;
52    private MobCacheConfig mobCacheConf;
53    private MobFileCache mobFileCache;
54    private Date currentDate = new Date();
55    private static final String TEST_CACHE_SIZE = "2";
56    private static final int EXPECTED_CACHE_SIZE_ZERO = 0;
57    private static final int EXPECTED_CACHE_SIZE_ONE = 1;
58    private static final int EXPECTED_CACHE_SIZE_TWO = 2;
59    private static final int EXPECTED_CACHE_SIZE_THREE = 3;
60    private static final long EXPECTED_REFERENCE_ONE = 1;
61    private static final long EXPECTED_REFERENCE_TWO = 2;
62  
63    private static final String TABLE = "tableName";
64    private static final String FAMILY1 = "family1";
65    private static final String FAMILY2 = "family2";
66    private static final String FAMILY3 = "family3";
67  
68    private static final byte[] ROW = Bytes.toBytes("row");
69    private static final byte[] ROW2 = Bytes.toBytes("row2");
70    private static final byte[] VALUE = Bytes.toBytes("value");
71    private static final byte[] VALUE2 = Bytes.toBytes("value2");
72    private static final byte[] QF1 = Bytes.toBytes("qf1");
73    private static final byte[] QF2 = Bytes.toBytes("qf2");
74    private static final byte[] QF3 = Bytes.toBytes("qf3");
75  
76    @Override
77    public void setUp() throws Exception {
78      UTIL = HBaseTestingUtility.createLocalHTU();
79      conf = UTIL.getConfiguration();
80      HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache");
81      HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1);
82      hcd1.setMobEnabled(true);
83      hcd1.setMobThreshold(0);
84      HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2);
85      hcd2.setMobEnabled(true);
86      hcd2.setMobThreshold(0);
87      HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3);
88      hcd3.setMobEnabled(true);
89      hcd3.setMobThreshold(0);
90      htd.addFamily(hcd1);
91      htd.addFamily(hcd2);
92      htd.addFamily(hcd3);
93      region = UTIL.createLocalHRegion(htd, null, null);
94    }
95  
96    @Override
97    protected void tearDown() throws Exception {
98      region.close();
99      region.getFilesystem().delete(UTIL.getDataTestDir(), true);
100   }
101 
102   /**
103    * Create the mob store file.
104    */
105   private Path createMobStoreFile(String family) throws IOException {
106     return createMobStoreFile(HBaseConfiguration.create(), family);
107   }
108 
109   /**
110    * Create the mob store file
111    */
112   private Path createMobStoreFile(Configuration conf, String family) throws IOException {
113     HColumnDescriptor hcd = new HColumnDescriptor(family);
114     hcd.setMaxVersions(4);
115     hcd.setMobEnabled(true);
116     mobCacheConf = new MobCacheConfig(conf, hcd);
117     return createMobStoreFile(hcd);
118   }
119 
120   /**
121    * Create the mob store file
122    */
123   private Path createMobStoreFile(HColumnDescriptor hcd)
124       throws IOException {
125     // Setting up a Store
126     TableName tn = TableName.valueOf(TABLE);
127     HTableDescriptor htd = new HTableDescriptor(tn);
128     htd.addFamily(hcd);
129     HMobStore mobStore = (HMobStore) region.getStore(hcd.getName());
130     KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE);
131     KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE);
132     KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2);
133     KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
134     int maxKeyCount = keys.length;
135     HRegionInfo regionInfo = new HRegionInfo(tn);
136     StoreFile.Writer mobWriter = mobStore.createWriterInTmp(currentDate,
137         maxKeyCount, hcd.getCompactionCompression(), regionInfo.getStartKey());
138     Path mobFilePath = mobWriter.getPath();
139     String fileName = mobFilePath.getName();
140     mobWriter.append(key1);
141     mobWriter.append(key2);
142     mobWriter.append(key3);
143     mobWriter.close();
144     String targetPathName = MobUtils.formatDate(currentDate);
145     Path targetPath = new Path(mobStore.getPath(), targetPathName);
146     mobStore.commitFile(mobFilePath, targetPath);
147     return new Path(targetPath, fileName);
148   }
149 
150   @Test
151   public void testMobFileCache() throws Exception {
152     FileSystem fs = FileSystem.get(conf);
153     conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
154     mobFileCache = new MobFileCache(conf);
155     Path file1Path = createMobStoreFile(FAMILY1);
156     Path file2Path = createMobStoreFile(FAMILY2);
157     Path file3Path = createMobStoreFile(FAMILY3);
158 
159     // Before open one file by the MobFileCache
160     assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
161     // Open one file by the MobFileCache
162     CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
163         fs, file1Path, mobCacheConf);
164     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
165     assertNotNull(cachedMobFile1);
166     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
167 
168     // The evict is also managed by a schedule thread pool.
169     // And its check period is set as 3600 seconds by default.
170     // This evict should get the lock at the most time
171     mobFileCache.evict();  // Cache not full, evict it
172     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
173     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
174 
175     mobFileCache.evictFile(file1Path.getName());  // Evict one file
176     assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
177     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
178 
179     cachedMobFile1.close();  // Close the cached mob file
180 
181     // Reopen three cached file
182     cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
183         fs, file1Path, mobCacheConf);
184     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
185     CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(
186         fs, file2Path, mobCacheConf);
187     assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize());
188     CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(
189         fs, file3Path, mobCacheConf);
190     // Before the evict
191     // Evict the cache, should close the first file 1
192     assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize());
193     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
194     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount());
195     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
196     mobFileCache.evict();
197     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
198     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
199     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount());
200     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
201   }
202 }