View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.List;
23  import java.util.NavigableSet;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.fs.FileStatus;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.fs.PathFilter;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.wal.WALSplitter;
34  
35  /**
36   * Utility methods for interacting with the hbase.root file system.
37   */
38  @InterfaceAudience.Private
39  public final class FSVisitor {
40    private static final Log LOG = LogFactory.getLog(FSVisitor.class);
41  
42    public interface RegionVisitor {
43      void region(final String region) throws IOException;
44    }
45  
46    public interface StoreFileVisitor {
47      void storeFile(final String region, final String family, final String hfileName)
48         throws IOException;
49    }
50  
51    public interface RecoveredEditsVisitor {
52      void recoveredEdits (final String region, final String logfile)
53        throws IOException;
54    }
55  
56    public interface LogFileVisitor {
57      void logFile (final String server, final String logfile)
58        throws IOException;
59    }
60  
61    private FSVisitor() {
62      // private constructor for utility class
63    }
64  
65    /**
66     * Iterate over the table store files
67     *
68     * @param fs {@link FileSystem}
69     * @param tableDir {@link Path} to the table directory
70     * @param visitor callback object to get the store files
71     * @throws IOException if an error occurred while scanning the directory
72     */
73    public static void visitRegions(final FileSystem fs, final Path tableDir,
74        final RegionVisitor visitor) throws IOException {
75      FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
76      if (regions == null) {
77        if (LOG.isTraceEnabled()) {
78          LOG.trace("No regions under directory:" + tableDir);
79        }
80        return;
81      }
82  
83      for (FileStatus region: regions) {
84        visitor.region(region.getPath().getName());
85      }
86    }
87  
88    /**
89     * Iterate over the table store files
90     *
91     * @param fs {@link FileSystem}
92     * @param tableDir {@link Path} to the table directory
93     * @param visitor callback object to get the store files
94     * @throws IOException if an error occurred while scanning the directory
95     */
96    public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
97        final StoreFileVisitor visitor) throws IOException {
98      List<FileStatus> regions = FSUtils.listStatusWithStatusFilter(fs, tableDir, new FSUtils.RegionDirFilter(fs));
99      if (regions == null) {
100       if (LOG.isTraceEnabled()) {
101         LOG.trace("No regions under directory:" + tableDir);
102       }
103       return;
104     }
105 
106     for (FileStatus region: regions) {
107       visitRegionStoreFiles(fs, region.getPath(), visitor);
108     }
109   }
110 
111   /**
112    * Iterate over the region store files
113    *
114    * @param fs {@link FileSystem}
115    * @param regionDir {@link Path} to the region directory
116    * @param visitor callback object to get the store files
117    * @throws IOException if an error occurred while scanning the directory
118    */
119   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
120       final StoreFileVisitor visitor) throws IOException {
121     List<FileStatus> families = FSUtils.listStatusWithStatusFilter(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
122     if (families == null) {
123       if (LOG.isTraceEnabled()) {
124         LOG.trace("No families under region directory:" + regionDir);
125       }
126       return;
127     }
128 
129     PathFilter fileFilter = new FSUtils.FileFilter(fs);
130     for (FileStatus family: families) {
131       Path familyDir = family.getPath();
132       String familyName = familyDir.getName();
133 
134       // get all the storeFiles in the family
135       FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
136       if (storeFiles == null) {
137         if (LOG.isTraceEnabled()) {
138           LOG.trace("No hfiles found for family: " + familyDir + ", skipping.");
139         }
140         continue;
141       }
142 
143       for (FileStatus hfile: storeFiles) {
144         Path hfilePath = hfile.getPath();
145         visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
146       }
147     }
148   }
149 
150   /**
151    * Iterate over each region in the table and inform about recovered.edits
152    *
153    * @param fs {@link FileSystem}
154    * @param tableDir {@link Path} to the table directory
155    * @param visitor callback object to get the recovered.edits files
156    * @throws IOException if an error occurred while scanning the directory
157    */
158   public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir,
159       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
160     FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
161     if (regions == null) {
162       if (LOG.isTraceEnabled()) {
163         LOG.trace("No recoveredEdits regions under directory:" + tableDir);
164       }
165       return;
166     }
167 
168     for (FileStatus region: regions) {
169       visitRegionRecoveredEdits(fs, region.getPath(), visitor);
170     }
171   }
172 
173   /**
174    * Iterate over recovered.edits of the specified region
175    *
176    * @param fs {@link FileSystem}
177    * @param regionDir {@link Path} to the Region directory
178    * @param visitor callback object to get the recovered.edits files
179    * @throws IOException if an error occurred while scanning the directory
180    */
181   public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir,
182       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
183     NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regionDir);
184     if (files == null || files.size() == 0) return;
185 
186     for (Path source: files) {
187       // check to see if the file is zero length, in which case we can skip it
188       FileStatus stat = fs.getFileStatus(source);
189       if (stat.getLen() <= 0) continue;
190 
191       visitor.recoveredEdits(regionDir.getName(), source.getName());
192     }
193   }
194 
195   /**
196    * Iterate over hbase log files
197    *
198    * @param fs {@link FileSystem}
199    * @param rootDir {@link Path} to the HBase root folder
200    * @param visitor callback object to get the log files
201    * @throws IOException if an error occurred while scanning the directory
202    */
203   public static void visitLogFiles(final FileSystem fs, final Path rootDir,
204       final LogFileVisitor visitor) throws IOException {
205     Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
206     FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
207     if (logServerDirs == null) {
208       if (LOG.isTraceEnabled()) {
209         LOG.trace("No logs under directory:" + logsDir);
210       }
211       return;
212     }
213 
214     for (FileStatus serverLogs: logServerDirs) {
215       String serverName = serverLogs.getPath().getName();
216 
217       FileStatus[] wals = FSUtils.listStatus(fs, serverLogs.getPath());
218       if (wals == null) {
219         if (LOG.isTraceEnabled()) {
220           LOG.trace("No wals found for server: " + serverName + ", skipping.");
221         }
222         continue;
223       }
224 
225       for (FileStatus walRef: wals) {
226         visitor.logFile(serverName, walRef.getPath().getName());
227       }
228     }
229   }
230 }