View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.List;
26  import java.util.Random;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.ChoreService;
36  import org.apache.hadoop.hbase.CoordinatedStateManager;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.testclassification.MediumTests;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.client.ClusterConnection;
43  import org.apache.hadoop.hbase.util.EnvironmentEdge;
44  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
46  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  @Category(MediumTests.class)
54  public class TestHFileCleaner {
55    private static final Log LOG = LogFactory.getLog(TestHFileCleaner.class);
56  
57    private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
58  
59    @BeforeClass
60    public static void setupCluster() throws Exception {
61      // have to use a minidfs cluster because the localfs doesn't modify file times correctly
62      UTIL.startMiniDFSCluster(1);
63    }
64  
65    @AfterClass
66    public static void shutdownCluster() throws IOException {
67      UTIL.shutdownMiniDFSCluster();
68    }
69  
70    @Test
71    public void testTTLCleaner() throws IOException, InterruptedException {
72      FileSystem fs = UTIL.getDFSCluster().getFileSystem();
73      Path root = UTIL.getDataTestDirOnTestFS();
74      Path file = new Path(root, "file");
75      fs.createNewFile(file);
76      long createTime = System.currentTimeMillis();
77      assertTrue("Test file not created!", fs.exists(file));
78      TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner();
79      // update the time info for the file, so the cleaner removes it
80      fs.setTimes(file, createTime - 100, -1);
81      Configuration conf = UTIL.getConfiguration();
82      conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
83      cleaner.setConf(conf);
84      assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
85          + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file)));
86    }
87  
88    /**
89     * @param file to check
90     * @return loggable information about the file
91     */
92    private String getFileStats(Path file, FileSystem fs) throws IOException {
93      FileStatus status = fs.getFileStatus(file);
94      return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:"
95          + status.getAccessTime();
96    }
97  
98    @Test(timeout = 60 *1000)
99    public void testHFileCleaning() throws Exception {
100     final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
101     String prefix = "someHFileThatWouldBeAUUID";
102     Configuration conf = UTIL.getConfiguration();
103     // set TTL
104     long ttl = 2000;
105     conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
106       "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner");
107     conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
108     Server server = new DummyServer();
109     Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
110     FileSystem fs = FileSystem.get(conf);
111     HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
112 
113     // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
114     final long createTime = System.currentTimeMillis();
115     fs.delete(archivedHfileDir, true);
116     fs.mkdirs(archivedHfileDir);
117     // Case 1: 1 invalid file, which should be deleted directly
118     fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
119     // Case 2: 1 "recent" file, not even deletable for the first log cleaner
120     // (TimeToLiveLogCleaner), so we are not going down the chain
121     LOG.debug("Now is: " + createTime);
122     for (int i = 1; i < 32; i++) {
123       // Case 3: old files which would be deletable for the first log cleaner
124       // (TimeToLiveHFileCleaner),
125       Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i)));
126       fs.createNewFile(fileName);
127       // set the creation time past ttl to ensure that it gets removed
128       fs.setTimes(fileName, createTime - ttl - 1, -1);
129       LOG.debug("Creating " + getFileStats(fileName, fs));
130     }
131 
132     // Case 2: 1 newer file, not even deletable for the first log cleaner
133     // (TimeToLiveLogCleaner), so we are not going down the chain
134     Path saved = new Path(archivedHfileDir, prefix + ".00000000000");
135     fs.createNewFile(saved);
136     // set creation time within the ttl
137     fs.setTimes(saved, createTime - ttl / 2, -1);
138     LOG.debug("Creating " + getFileStats(saved, fs));
139     for (FileStatus stat : fs.listStatus(archivedHfileDir)) {
140       LOG.debug(stat.getPath().toString());
141     }
142 
143     assertEquals(33, fs.listStatus(archivedHfileDir).length);
144 
145     // set a custom edge manager to handle time checking
146     EnvironmentEdge setTime = new EnvironmentEdge() {
147       @Override
148       public long currentTime() {
149         return createTime;
150       }
151     };
152     EnvironmentEdgeManager.injectEdge(setTime);
153 
154     // run the chore
155     cleaner.chore();
156 
157     // ensure we only end up with the saved file
158     assertEquals(1, fs.listStatus(archivedHfileDir).length);
159 
160     for (FileStatus file : fs.listStatus(archivedHfileDir)) {
161       LOG.debug("Kept hfiles: " + file.getPath().getName());
162     }
163 
164     // reset the edge back to the original edge
165     EnvironmentEdgeManager.injectEdge(originalEdge);
166   }
167 
168   @Test
169   public void testRemovesEmptyDirectories() throws Exception {
170     Configuration conf = UTIL.getConfiguration();
171     // no cleaner policies = delete all files
172     conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
173     Server server = new DummyServer();
174     Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
175 
176     // setup the cleaner
177     FileSystem fs = UTIL.getDFSCluster().getFileSystem();
178     HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
179 
180     // make all the directories for archiving files
181     Path table = new Path(archivedHfileDir, "table");
182     Path region = new Path(table, "regionsomthing");
183     Path family = new Path(region, "fam");
184     Path file = new Path(family, "file12345");
185     fs.mkdirs(family);
186     if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
187     fs.create(file).close();
188     if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);
189 
190     // run the chore to cleanup the files (and the directories above it)
191     cleaner.chore();
192 
193     // make sure all the parent directories get removed
194     assertFalse("family directory not removed for empty directory", fs.exists(family));
195     assertFalse("region directory not removed for empty directory", fs.exists(region));
196     assertFalse("table directory not removed for empty directory", fs.exists(table));
197     assertTrue("archive directory", fs.exists(archivedHfileDir));
198   }
199 
200   static class DummyServer implements Server {
201 
202     @Override
203     public Configuration getConfiguration() {
204       return UTIL.getConfiguration();
205     }
206 
207     @Override
208     public ZooKeeperWatcher getZooKeeper() {
209       try {
210         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
211       } catch (IOException e) {
212         e.printStackTrace();
213       }
214       return null;
215     }
216 
217     @Override
218     public CoordinatedStateManager getCoordinatedStateManager() {
219       return null;
220     }
221 
222     @Override
223     public ClusterConnection getConnection() {
224       return null;
225     }
226 
227     @Override
228     public MetaTableLocator getMetaTableLocator() {
229       return null;
230     }
231 
232     @Override
233     public ServerName getServerName() {
234       return ServerName.valueOf("regionserver,60020,000000");
235     }
236 
237     @Override
238     public void abort(String why, Throwable e) {
239     }
240 
241     @Override
242     public boolean isAborted() {
243       return false;
244     }
245 
246     @Override
247     public void stop(String why) {
248     }
249 
250     @Override
251     public boolean isStopped() {
252       return false;
253     }
254 
255     @Override
256     public ChoreService getChoreService() {
257       return null;
258     }
259   }
260 
261   @Test
262   public void testThreadCleanup() throws Exception {
263     Configuration conf = UTIL.getConfiguration();
264     conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
265     Server server = new DummyServer();
266     Path archivedHfileDir =
267         new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
268 
269     // setup the cleaner
270     FileSystem fs = UTIL.getDFSCluster().getFileSystem();
271     HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
272     // clean up archive directory
273     fs.delete(archivedHfileDir, true);
274     fs.mkdirs(archivedHfileDir);
275     // create some file to delete
276     fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
277     // launch the chore
278     cleaner.chore();
279     // call cleanup
280     cleaner.cleanup();
281     // wait awhile for thread to die
282     Thread.sleep(100);
283     for (Thread thread : cleaner.getCleanerThreads()) {
284       Assert.assertFalse(thread.isAlive());
285     }
286   }
287 
288   @Test
289   public void testLargeSmallIsolation() throws Exception {
290     Configuration conf = UTIL.getConfiguration();
291     // no cleaner policies = delete all files
292     conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
293     conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, 512 * 1024);
294     Server server = new DummyServer();
295     Path archivedHfileDir =
296         new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
297 
298     // setup the cleaner
299     FileSystem fs = UTIL.getDFSCluster().getFileSystem();
300     HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
301     // clean up archive directory
302     fs.delete(archivedHfileDir, true);
303     fs.mkdirs(archivedHfileDir);
304     // necessary set up
305     final int LARGE_FILE_NUM = 5;
306     final int SMALL_FILE_NUM = 20;
307     createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
308     // call cleanup
309     cleaner.chore();
310 
311     Assert.assertEquals(LARGE_FILE_NUM, cleaner.getNumOfDeletedLargeFiles());
312     Assert.assertEquals(SMALL_FILE_NUM, cleaner.getNumOfDeletedSmallFiles());
313   }
314 
315   @Test(timeout = 60 * 1000)
316   public void testOnConfigurationChange() throws Exception {
317     // constants
318     final int ORIGINAL_THROTTLE_POINT = 512 * 1024;
319     final int ORIGINAL_QUEUE_INIT_SIZE = 512;
320     final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check
321     final int UPDATE_QUEUE_INIT_SIZE = 1024;
322     final int LARGE_FILE_NUM = 5;
323     final int SMALL_FILE_NUM = 20;
324     final int LARGE_THREAD_NUM = 2;
325     final int SMALL_THREAD_NUM = 4;
326 
327     Configuration conf = UTIL.getConfiguration();
328     // no cleaner policies = delete all files
329     conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
330     conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT);
331     conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
332     conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
333     Server server = new DummyServer();
334     Path archivedHfileDir =
335         new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
336 
337     // setup the cleaner
338     FileSystem fs = UTIL.getDFSCluster().getFileSystem();
339     final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);
340     Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint());
341     Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
342     Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
343 
344     // clean up archive directory and create files for testing
345     fs.delete(archivedHfileDir, true);
346     fs.mkdirs(archivedHfileDir);
347     createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
348 
349     // call cleaner, run as daemon to test the interrupt-at-middle case
350     Thread t = new Thread() {
351       @Override
352       public void run() {
353         cleaner.chore();
354       }
355     };
356     t.setDaemon(true);
357     t.start();
358     // wait until file clean started
359     while (cleaner.getNumOfDeletedSmallFiles() == 0) {
360       Thread.yield();
361     }
362 
363     // trigger configuration change
364     Configuration newConf = new Configuration(conf);
365     newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);
366     newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
367     newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
368     newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_THREAD_NUMBER, LARGE_THREAD_NUM);
369     newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_THREAD_NUMBER, SMALL_THREAD_NUM);
370     LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
371         + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
372     cleaner.onConfigurationChange(newConf);
373 
374     // check values after change
375     Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());
376     Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
377     Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
378     Assert.assertEquals(LARGE_THREAD_NUM + SMALL_THREAD_NUM, cleaner.getCleanerThreads().size());
379 
380     // make sure no cost when onConfigurationChange called with no change
381     List<Thread> oldThreads = cleaner.getCleanerThreads();
382     cleaner.onConfigurationChange(newConf);
383     List<Thread> newThreads = cleaner.getCleanerThreads();
384     Assert.assertArrayEquals(oldThreads.toArray(), newThreads.toArray());
385 
386     // wait until clean done and check
387     t.join();
388     LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
389         + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
390     Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM
391         + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),
392       cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);
393     Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM
394         + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),
395       cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);
396   }
397 
398   private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,
399       Path archivedHfileDir) throws IOException {
400     final Random rand = new Random();
401     final byte[] large = new byte[1024 * 1024];
402     for (int i = 0; i < large.length; i++) {
403       large[i] = (byte) rand.nextInt(128);
404     }
405     final byte[] small = new byte[1024];
406     for (int i = 0; i < small.length; i++) {
407       small[i] = (byte) rand.nextInt(128);
408     }
409     // create large and small files
410     for (int i = 1; i <= largeFileNum; i++) {
411       FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));
412       out.write(large);
413       out.close();
414     }
415     for (int i = 1; i <= smallFileNum; i++) {
416       FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));
417       out.write(small);
418       out.close();
419     }
420   }
421 }