View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.backup.util;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.net.URLDecoder;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.HashMap;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.TreeMap;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.LocatedFileStatus;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.fs.RemoteIterator;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.backup.BackupInfo;
44  import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
45  import org.apache.hadoop.hbase.classification.InterfaceAudience;
46  import org.apache.hadoop.hbase.classification.InterfaceStability;
47  
48  /**
49   * A collection of methods used by multiple classes to backup HBase tables.
50   */
51  @InterfaceAudience.Private
52  @InterfaceStability.Evolving
53  public final class BackupClientUtil {
54    protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class);
55    public static final String LOGNAME_SEPARATOR = ".";
56  
57    private BackupClientUtil(){
58      throw new AssertionError("Instantiating utility class...");
59    }
60  
61    /**
62     * Check whether the backup path exist
63     * @param backupStr backup
64     * @param conf configuration
65     * @return Yes if path exists
66     * @throws IOException exception
67     */
68    public static boolean checkPathExist(String backupStr, Configuration conf)
69      throws IOException {
70      boolean isExist = false;
71      Path backupPath = new Path(backupStr);
72      FileSystem fileSys = backupPath.getFileSystem(conf);
73      String targetFsScheme = fileSys.getUri().getScheme();
74      if (LOG.isTraceEnabled()) {
75        LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme);
76      }
77      if (fileSys.exists(backupPath)) {
78        isExist = true;
79      }
80      return isExist;
81    }
82  
83    // check target path first, confirm it doesn't exist before backup
84    public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException {
85      boolean targetExists = false;
86      try {
87        targetExists = checkPathExist(backupRootPath, conf);
88      } catch (IOException e) {
89        String expMsg = e.getMessage();
90        String newMsg = null;
91        if (expMsg.contains("No FileSystem for scheme")) {
92          newMsg =
93              "Unsupported filesystem scheme found in the backup target url. Error Message: "
94                  + newMsg;
95          LOG.error(newMsg);
96          throw new IOException(newMsg);
97        } else {
98          throw e;
99        }
100     }
101 
102     if (targetExists) {
103       LOG.info("Using existing backup root dir: " + backupRootPath);
104     } else {
105       LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
106     }
107   }
108 
109   /**
110    * Get the min value for all the Values a map.
111    * @param map map
112    * @return the min value
113    */
114   public static <T> Long getMinValue(HashMap<T, Long> map) {
115     Long minTimestamp = null;
116     if (map != null) {
117       ArrayList<Long> timestampList = new ArrayList<Long>(map.values());
118       Collections.sort(timestampList);
119       // The min among all the RS log timestamps will be kept in hbase:backup table.
120       minTimestamp = timestampList.get(0);
121     }
122     return minTimestamp;
123   }
124 
125   /**
126    * Parses host name:port from archived WAL path 
127    * @param p path
128    * @return host name
129    * @throws IOException exception
130    */
131   public static String parseHostFromOldLog(Path p) throws IOException {
132     try{
133       String n = p.getName();
134       int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
135       String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
136       return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
137     } catch(Exception e){
138       LOG.error("Failed to parse "+ p, e);
139       return null;
140     }
141   }
142 
143   /**
144    * Given the log file, parse the timestamp from the file name. The timestamp is the last number.
145    * @param p a path to the log file
146    * @return the timestamp
147    * @throws IOException exception
148    */
149   public static Long getCreationTime(Path p) throws IOException {
150     int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
151     if (idx < 0) {
152       throw new IOException("Cannot parse timestamp from path " + p);
153     }
154     String ts = p.getName().substring(idx + 1);
155     return Long.parseLong(ts);
156   }
157 
158   public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
159     PathFilter filter) throws FileNotFoundException, IOException {
160     RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
161 
162     while (it.hasNext()) {
163       LocatedFileStatus lfs = it.next();
164       if (lfs.isDirectory()) {
165         continue;
166       }
167       // apply filter
168       if (filter.accept(lfs.getPath())) {
169         files.add(lfs.getPath().toString());
170       }
171     }
172     return files;
173   }
174   
175   public static void cleanupBackupData(BackupInfo context, Configuration conf) 
176       throws IOException 
177   {
178     cleanupHLogDir(context, conf);
179     cleanupTargetDir(context, conf);
180   }
181 
182   /**
183    * Clean up directories which are generated when DistCp copying hlogs.
184    * @throws IOException
185    */
186   private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf)
187       throws IOException {
188 
189     String logDir = backupContext.getHLogTargetDir();
190     if (logDir == null) {
191       LOG.warn("No log directory specified for " + backupContext.getBackupId());
192       return;
193     }
194 
195     Path rootPath = new Path(logDir).getParent();
196     FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
197     FileStatus[] files = listStatus(fs, rootPath, null);
198     if (files == null) {
199       return;
200     }
201     for (FileStatus file : files) {
202       LOG.debug("Delete log files: " + file.getPath().getName());
203       fs.delete(file.getPath(), true);
204     }
205   }
206 
207   /**
208    * Clean up the data at target directory
209    */
210   private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) {
211     try {
212       // clean up the data at target directory
213       LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId());
214       String targetDir = backupContext.getTargetRootDir();
215       if (targetDir == null) {
216         LOG.warn("No target directory specified for " + backupContext.getBackupId());
217         return;
218       }
219 
220       FileSystem outputFs =
221           FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
222 
223       for (TableName table : backupContext.getTables()) {
224         Path targetDirPath =
225             new Path(getTableBackupDir(backupContext.getTargetRootDir(),
226               backupContext.getBackupId(), table));
227         if (outputFs.delete(targetDirPath, true)) {
228           LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
229         } else {
230           LOG.info("No data has been found in " + targetDirPath.toString() + ".");
231         }
232 
233         Path tableDir = targetDirPath.getParent();
234         if (tableDir != null) {
235           FileStatus[] backups = listStatus(outputFs, tableDir, null);
236           if (backups == null || backups.length == 0) {
237             outputFs.delete(tableDir, true);
238             LOG.debug(tableDir.toString() + " is empty, remove it.");
239           }
240         }
241       }
242       outputFs.delete(new Path(targetDir, backupContext.getBackupId()), true);
243     } catch (IOException e1) {
244       LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at "
245           + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
246     }
247   }
248 
249   /**
250    * Given the backup root dir, backup id and the table name, return the backup image location,
251    * which is also where the backup manifest file is. return value look like:
252    * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/backup_1396650096738/default/t1_dn/"
253    * @param backupRootDir backup root directory
254    * @param backupId  backup id
255    * @param table table name
256    * @return backupPath String for the particular table
257    */
258   public static String getTableBackupDir(String backupRootDir, String backupId,
259       TableName tableName) {
260     return backupRootDir + Path.SEPARATOR+ backupId + Path.SEPARATOR + 
261         tableName.getNamespaceAsString() + Path.SEPARATOR
262         + tableName.getQualifierAsString() + Path.SEPARATOR ;
263   }  
264   
265   public static TableName[] parseTableNames(String tables) {
266     if (tables == null) {
267       return null;
268     }
269     String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
270 
271     TableName[] ret = new TableName[tableArray.length];
272     for (int i = 0; i < tableArray.length; i++) {
273       ret[i] = TableName.valueOf(tableArray[i]);
274     }
275     return ret;
276   }
277 
278   /**
279    * Sort history list by start time in descending order.
280    * @param historyList history list
281    * @return sorted list of BackupCompleteData
282    */
283   public static ArrayList<BackupInfo> sortHistoryListDesc(
284     ArrayList<BackupInfo> historyList) {
285     ArrayList<BackupInfo> list = new ArrayList<BackupInfo>();
286     TreeMap<String, BackupInfo> map = new TreeMap<String, BackupInfo>();
287     for (BackupInfo h : historyList) {
288       map.put(Long.toString(h.getStartTs()), h);
289     }
290     Iterator<String> i = map.descendingKeySet().iterator();
291     while (i.hasNext()) {
292       list.add(map.get(i.next()));
293     }
294     return list;
295   }
296 
297   /**
298    * Returns WAL file name
299    * @param walFileName WAL file name
300    * @return WAL file name
301    * @throws IOException exception
302    * @throws IllegalArgumentException exception
303    */
304   public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
305     return getUniqueWALFileNamePart(new Path(walFileName));
306   }
307 
308   /**
309    * Returns WAL file name
310    * @param p - WAL file path
311    * @return WAL file name
312    * @throws IOException exception
313    */
314   public static String getUniqueWALFileNamePart(Path p) throws IOException {
315     return p.getName();
316   }
317   
318   /**
319    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
320    * This accommodates differences between hadoop versions, where hadoop 1
321    * does not throw a FileNotFoundException, and return an empty FileStatus[]
322    * while Hadoop 2 will throw FileNotFoundException.
323    *
324    * @param fs file system
325    * @param dir directory
326    * @param filter path filter
327    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
328    */
329   public static FileStatus [] listStatus(final FileSystem fs,
330       final Path dir, final PathFilter filter) throws IOException {
331     FileStatus [] status = null;
332     try {
333       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
334     } catch (FileNotFoundException fnfe) {
335       // if directory doesn't exist, return null
336       if (LOG.isTraceEnabled()) {
337         LOG.trace(dir + " doesn't exist");
338       }
339     }
340     if (status == null || status.length < 1) return null;
341     return status;
342   }  
343   
344   /**
345    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
346    * method returns the 'path' component of a Path's URI: e.g. If a Path is
347    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
348    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
349    * This method is useful if you want to print out a Path without qualifying
350    * Filesystem instance.
351    * @param p Filesystem Path whose 'path' component we are to return.
352    * @return Path portion of the Filesystem
353    */
354   public static String getPath(Path p) {
355     return p.toUri().getPath();
356   }
357  
358   /**
359    * Given the backup root dir and the backup id, return the log file location for an incremental
360    * backup.
361    * @param backupRootDir backup root directory
362    * @param backupId backup id
363    * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738"
364    */
365   public static String getLogBackupDir(String backupRootDir, String backupId) {
366     return backupRootDir + Path.SEPARATOR + backupId+ Path.SEPARATOR
367         + HConstants.HREGION_LOGDIR_NAME;
368   }
369 
370 }