View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.ChoreService;
31  import org.apache.hadoop.hbase.CoordinatedStateManager;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.Server;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.client.ClusterConnection;
38  import org.apache.hadoop.hbase.io.HFileLink;
39  import org.apache.hadoop.hbase.testclassification.SmallTests;
40  import org.apache.hadoop.hbase.util.FSUtils;
41  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
42  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
43  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * Test the HFileLink Cleaner.
49   * HFiles with links cannot be deleted until a link is present.
50   */
51  @Category(SmallTests.class)
52  public class TestHFileLinkCleaner {
53  
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    @Test
57    public void testHFileLinkCleaning() throws Exception {
58      Configuration conf = TEST_UTIL.getConfiguration();
59      FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
60      conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
61      Path rootDir = FSUtils.getRootDir(conf);
62      FileSystem fs = FileSystem.get(conf);
63  
64      final TableName tableName = TableName.valueOf("test-table");
65      final TableName tableLinkName = TableName.valueOf("test-link");
66      final String hfileName = "1234567890";
67      final String familyName = "cf";
68  
69      HRegionInfo hri = new HRegionInfo(tableName);
70      HRegionInfo hriLink = new HRegionInfo(tableLinkName);
71  
72      Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
73      Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
74            tableName, hri.getEncodedName(), familyName);
75      Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
76            tableLinkName, hriLink.getEncodedName(), familyName);
77  
78      // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
79      Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
80      fs.mkdirs(familyPath);
81      Path hfilePath = new Path(familyPath, hfileName);
82      fs.createNewFile(hfilePath);
83  
84      // Create link to hfile
85      Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
86                                          hriLink.getEncodedName(), familyName);
87      fs.mkdirs(familyLinkPath);
88      HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
89      Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
90      assertTrue(fs.exists(linkBackRefDir));
91      FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
92      assertEquals(1, backRefs.length);
93      Path linkBackRef = backRefs[0].getPath();
94  
95      // Initialize cleaner
96      final long ttl = 1000;
97      conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
98      Server server = new DummyServer();
99      HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
100 
101     // Link backref cannot be removed
102     cleaner.chore();
103     assertTrue(fs.exists(linkBackRef));
104     assertTrue(fs.exists(hfilePath));
105 
106     // Link backref can be removed
107     fs.rename(FSUtils.getTableDir(rootDir, tableLinkName),
108         FSUtils.getTableDir(archiveDir, tableLinkName));
109     cleaner.chore();
110     assertFalse("Link should be deleted", fs.exists(linkBackRef));
111 
112     // HFile can be removed
113     Thread.sleep(ttl * 2);
114     cleaner.chore();
115     assertFalse("HFile should be deleted", fs.exists(hfilePath));
116 
117     // Remove everything
118     for (int i = 0; i < 4; ++i) {
119       Thread.sleep(ttl * 2);
120       cleaner.chore();
121     }
122     assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName)));
123     assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName)));
124   }
125 
126   private static Path getFamilyDirPath (final Path rootDir, final TableName table,
127     final String region, final String family) {
128     return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family);
129   }
130 
131   static class DummyServer implements Server {
132 
133     @Override
134     public Configuration getConfiguration() {
135       return TEST_UTIL.getConfiguration();
136     }
137 
138     @Override
139     public ZooKeeperWatcher getZooKeeper() {
140       try {
141         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
142       } catch (IOException e) {
143         e.printStackTrace();
144       }
145       return null;
146     }
147 
148     @Override
149     public CoordinatedStateManager getCoordinatedStateManager() {
150       return null;
151     }
152 
153     @Override
154     public ClusterConnection getConnection() {
155       return null;
156     }
157 
158     @Override
159     public MetaTableLocator getMetaTableLocator() {
160       return null;
161     }
162 
163     @Override
164     public ServerName getServerName() {
165       return ServerName.valueOf("regionserver,60020,000000");
166     }
167 
168     @Override
169     public void abort(String why, Throwable e) {}
170 
171     @Override
172     public boolean isAborted() {
173       return false;
174     }
175 
176     @Override
177     public void stop(String why) {}
178 
179     @Override
180     public boolean isStopped() {
181       return false;
182     }
183 
184     @Override
185     public ChoreService getChoreService() {
186       return null;
187     }
188   }
189 }