1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.net.URLDecoder;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.TreeMap;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.LocatedFileStatus;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.fs.PathFilter;
39 import org.apache.hadoop.fs.RemoteIterator;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.backup.BackupInfo;
44 import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
45 import org.apache.hadoop.hbase.classification.InterfaceAudience;
46 import org.apache.hadoop.hbase.classification.InterfaceStability;
47
48
49
50
51 @InterfaceAudience.Private
52 @InterfaceStability.Evolving
53 public final class BackupClientUtil {
54 protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class);
55 public static final String LOGNAME_SEPARATOR = ".";
56
57 private BackupClientUtil(){
58 throw new AssertionError("Instantiating utility class...");
59 }
60
61
62
63
64
65
66
67
68 public static boolean checkPathExist(String backupStr, Configuration conf)
69 throws IOException {
70 boolean isExist = false;
71 Path backupPath = new Path(backupStr);
72 FileSystem fileSys = backupPath.getFileSystem(conf);
73 String targetFsScheme = fileSys.getUri().getScheme();
74 if (LOG.isTraceEnabled()) {
75 LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme);
76 }
77 if (fileSys.exists(backupPath)) {
78 isExist = true;
79 }
80 return isExist;
81 }
82
83
84 public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException {
85 boolean targetExists = false;
86 try {
87 targetExists = checkPathExist(backupRootPath, conf);
88 } catch (IOException e) {
89 String expMsg = e.getMessage();
90 String newMsg = null;
91 if (expMsg.contains("No FileSystem for scheme")) {
92 newMsg =
93 "Unsupported filesystem scheme found in the backup target url. Error Message: "
94 + newMsg;
95 LOG.error(newMsg);
96 throw new IOException(newMsg);
97 } else {
98 throw e;
99 }
100 }
101
102 if (targetExists) {
103 LOG.info("Using existing backup root dir: " + backupRootPath);
104 } else {
105 LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
106 }
107 }
108
109
110
111
112
113
114 public static <T> Long getMinValue(HashMap<T, Long> map) {
115 Long minTimestamp = null;
116 if (map != null) {
117 ArrayList<Long> timestampList = new ArrayList<Long>(map.values());
118 Collections.sort(timestampList);
119
120 minTimestamp = timestampList.get(0);
121 }
122 return minTimestamp;
123 }
124
125
126
127
128
129
130
131 public static String parseHostFromOldLog(Path p) throws IOException {
132 try{
133 String n = p.getName();
134 int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
135 String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
136 return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
137 } catch(Exception e){
138 LOG.error("Failed to parse "+ p, e);
139 return null;
140 }
141 }
142
143
144
145
146
147
148
149 public static Long getCreationTime(Path p) throws IOException {
150 int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
151 if (idx < 0) {
152 throw new IOException("Cannot parse timestamp from path " + p);
153 }
154 String ts = p.getName().substring(idx + 1);
155 return Long.parseLong(ts);
156 }
157
158 public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
159 PathFilter filter) throws FileNotFoundException, IOException {
160 RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
161
162 while (it.hasNext()) {
163 LocatedFileStatus lfs = it.next();
164 if (lfs.isDirectory()) {
165 continue;
166 }
167
168 if (filter.accept(lfs.getPath())) {
169 files.add(lfs.getPath().toString());
170 }
171 }
172 return files;
173 }
174
175 public static void cleanupBackupData(BackupInfo context, Configuration conf)
176 throws IOException
177 {
178 cleanupHLogDir(context, conf);
179 cleanupTargetDir(context, conf);
180 }
181
182
183
184
185
186 private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf)
187 throws IOException {
188
189 String logDir = backupContext.getHLogTargetDir();
190 if (logDir == null) {
191 LOG.warn("No log directory specified for " + backupContext.getBackupId());
192 return;
193 }
194
195 Path rootPath = new Path(logDir).getParent();
196 FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
197 FileStatus[] files = listStatus(fs, rootPath, null);
198 if (files == null) {
199 return;
200 }
201 for (FileStatus file : files) {
202 LOG.debug("Delete log files: " + file.getPath().getName());
203 fs.delete(file.getPath(), true);
204 }
205 }
206
207
208
209
210 private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) {
211 try {
212
213 LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId());
214 String targetDir = backupContext.getTargetRootDir();
215 if (targetDir == null) {
216 LOG.warn("No target directory specified for " + backupContext.getBackupId());
217 return;
218 }
219
220 FileSystem outputFs =
221 FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
222
223 for (TableName table : backupContext.getTables()) {
224 Path targetDirPath =
225 new Path(getTableBackupDir(backupContext.getTargetRootDir(),
226 backupContext.getBackupId(), table));
227 if (outputFs.delete(targetDirPath, true)) {
228 LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
229 } else {
230 LOG.info("No data has been found in " + targetDirPath.toString() + ".");
231 }
232
233 Path tableDir = targetDirPath.getParent();
234 if (tableDir != null) {
235 FileStatus[] backups = listStatus(outputFs, tableDir, null);
236 if (backups == null || backups.length == 0) {
237 outputFs.delete(tableDir, true);
238 LOG.debug(tableDir.toString() + " is empty, remove it.");
239 }
240 }
241 }
242 outputFs.delete(new Path(targetDir, backupContext.getBackupId()), true);
243 } catch (IOException e1) {
244 LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at "
245 + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
246 }
247 }
248
249
250
251
252
253
254
255
256
257
258 public static String getTableBackupDir(String backupRootDir, String backupId,
259 TableName tableName) {
260 return backupRootDir + Path.SEPARATOR+ backupId + Path.SEPARATOR +
261 tableName.getNamespaceAsString() + Path.SEPARATOR
262 + tableName.getQualifierAsString() + Path.SEPARATOR ;
263 }
264
265 public static TableName[] parseTableNames(String tables) {
266 if (tables == null) {
267 return null;
268 }
269 String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
270
271 TableName[] ret = new TableName[tableArray.length];
272 for (int i = 0; i < tableArray.length; i++) {
273 ret[i] = TableName.valueOf(tableArray[i]);
274 }
275 return ret;
276 }
277
278
279
280
281
282
283 public static ArrayList<BackupInfo> sortHistoryListDesc(
284 ArrayList<BackupInfo> historyList) {
285 ArrayList<BackupInfo> list = new ArrayList<BackupInfo>();
286 TreeMap<String, BackupInfo> map = new TreeMap<String, BackupInfo>();
287 for (BackupInfo h : historyList) {
288 map.put(Long.toString(h.getStartTs()), h);
289 }
290 Iterator<String> i = map.descendingKeySet().iterator();
291 while (i.hasNext()) {
292 list.add(map.get(i.next()));
293 }
294 return list;
295 }
296
297
298
299
300
301
302
303
304 public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
305 return getUniqueWALFileNamePart(new Path(walFileName));
306 }
307
308
309
310
311
312
313
314 public static String getUniqueWALFileNamePart(Path p) throws IOException {
315 return p.getName();
316 }
317
318
319
320
321
322
323
324
325
326
327
328
329 public static FileStatus [] listStatus(final FileSystem fs,
330 final Path dir, final PathFilter filter) throws IOException {
331 FileStatus [] status = null;
332 try {
333 status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
334 } catch (FileNotFoundException fnfe) {
335
336 if (LOG.isTraceEnabled()) {
337 LOG.trace(dir + " doesn't exist");
338 }
339 }
340 if (status == null || status.length < 1) return null;
341 return status;
342 }
343
344
345
346
347
348
349
350
351
352
353
354 public static String getPath(Path p) {
355 return p.toUri().getPath();
356 }
357
358
359
360
361
362
363
364
365 public static String getLogBackupDir(String backupRootDir, String backupId) {
366 return backupRootDir + Path.SEPARATOR + backupId+ Path.SEPARATOR
367 + HConstants.HREGION_LOGDIR_NAME;
368 }
369
370 }