1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup.impl;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Map.Entry;
28 import java.util.TreeMap;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FSDataInputStream;
34 import org.apache.hadoop.fs.FSDataOutputStream;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.backup.BackupInfo;
41 import org.apache.hadoop.hbase.backup.BackupType;
42 import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.classification.InterfaceStability;
45 import org.apache.hadoop.hbase.exceptions.DeserializationException;
46 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47 import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
48 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
49 import org.apache.hadoop.hbase.protobuf.generated.TableProtos;
50
51 import com.google.protobuf.InvalidProtocolBufferException;
52
53
54
55
56
57
58
59 @InterfaceAudience.Private
60 @InterfaceStability.Evolving
61 public class BackupManifest {
62
63 private static final Log LOG = LogFactory.getLog(BackupManifest.class);
64
65
66 public static final String MANIFEST_FILE_NAME = ".backup.manifest";
67
68
69 public static final String MANIFEST_VERSION = "1.0";
70
71
72
73 public static class BackupImage implements Comparable<BackupImage> {
74
75 private String backupId;
76 private BackupType type;
77 private String rootDir;
78 private List<TableName> tableList;
79 private long startTs;
80 private long completeTs;
81 private ArrayList<BackupImage> ancestors;
82
83 public BackupImage() {
84 super();
85 }
86
87 public BackupImage(String backupId, BackupType type, String rootDir,
88 List<TableName> tableList, long startTs, long completeTs) {
89 this.backupId = backupId;
90 this.type = type;
91 this.rootDir = rootDir;
92 this.tableList = tableList;
93 this.startTs = startTs;
94 this.completeTs = completeTs;
95 }
96
97 static BackupImage fromProto(BackupProtos.BackupImage im) {
98 String backupId = im.getBackupId();
99 String rootDir = im.getRootDir();
100 long startTs = im.getStartTs();
101 long completeTs = im.getCompleteTs();
102 List<TableProtos.TableName> tableListList = im.getTableListList();
103 List<TableName> tableList = new ArrayList<TableName>();
104 for(TableProtos.TableName tn : tableListList) {
105 tableList.add(ProtobufUtil.toTableName(tn));
106 }
107
108 List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
109
110 BackupType type =
111 im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL:
112 BackupType.INCREMENTAL;
113
114 BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
115 for(BackupProtos.BackupImage img: ancestorList) {
116 image.addAncestor(fromProto(img));
117 }
118 return image;
119 }
120
121 BackupProtos.BackupImage toProto() {
122 BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
123 builder.setBackupId(backupId);
124 builder.setCompleteTs(completeTs);
125 builder.setStartTs(startTs);
126 builder.setRootDir(rootDir);
127 if (type == BackupType.FULL) {
128 builder.setBackupType(BackupProtos.BackupType.FULL);
129 } else{
130 builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
131 }
132
133 for (TableName name: tableList) {
134 builder.addTableList(ProtobufUtil.toProtoTableName(name));
135 }
136
137 if (ancestors != null){
138 for (BackupImage im: ancestors){
139 builder.addAncestors(im.toProto());
140 }
141 }
142
143 return builder.build();
144 }
145
146 public String getBackupId() {
147 return backupId;
148 }
149
150 public void setBackupId(String backupId) {
151 this.backupId = backupId;
152 }
153
154 public BackupType getType() {
155 return type;
156 }
157
158 public void setType(BackupType type) {
159 this.type = type;
160 }
161
162 public String getRootDir() {
163 return rootDir;
164 }
165
166 public void setRootDir(String rootDir) {
167 this.rootDir = rootDir;
168 }
169
170 public List<TableName> getTableNames() {
171 return tableList;
172 }
173
174 public void setTableList(List<TableName> tableList) {
175 this.tableList = tableList;
176 }
177
178 public long getStartTs() {
179 return startTs;
180 }
181
182 public void setStartTs(long startTs) {
183 this.startTs = startTs;
184 }
185
186 public long getCompleteTs() {
187 return completeTs;
188 }
189
190 public void setCompleteTs(long completeTs) {
191 this.completeTs = completeTs;
192 }
193
194 public ArrayList<BackupImage> getAncestors() {
195 if (this.ancestors == null) {
196 this.ancestors = new ArrayList<BackupImage>();
197 }
198 return this.ancestors;
199 }
200
201 public void addAncestor(BackupImage backupImage) {
202 this.getAncestors().add(backupImage);
203 }
204
205 public boolean hasAncestor(String token) {
206 for (BackupImage image : this.getAncestors()) {
207 if (image.getBackupId().equals(token)) {
208 return true;
209 }
210 }
211 return false;
212 }
213
214 public boolean hasTable(TableName table) {
215 for (TableName t : tableList) {
216 if (t.equals(table)) {
217 return true;
218 }
219 }
220 return false;
221 }
222
223 @Override
224 public int compareTo(BackupImage other) {
225 String thisBackupId = this.getBackupId();
226 String otherBackupId = other.getBackupId();
227 Long thisTS = new Long(thisBackupId.substring(thisBackupId.lastIndexOf("_") + 1));
228 Long otherTS = new Long(otherBackupId.substring(otherBackupId.lastIndexOf("_") + 1));
229 return thisTS.compareTo(otherTS);
230 }
231 }
232
233
234 private String version = MANIFEST_VERSION;
235
236
237 protected Configuration config = null;
238
239
240 private String rootDir = null;
241
242
243 private String tableBackupDir = null;
244
245
246 private String logBackupDir = null;
247
248
249 private String backupId;
250
251
252 private BackupType type;
253
254
255 private ArrayList<TableName> tableList;
256
257
258 private long startTs;
259
260
261 private long completeTs;
262
263
264
265 private Map<TableName, HashMap<String, Long>> incrTimeRanges;
266
267
268 private Map<String, BackupImage> dependency;
269
270
271
272
273
274 public BackupManifest(BackupInfo backupCtx) {
275 this.backupId = backupCtx.getBackupId();
276 this.type = backupCtx.getType();
277 this.rootDir = backupCtx.getTargetRootDir();
278 if (this.type == BackupType.INCREMENTAL) {
279 this.logBackupDir = backupCtx.getHLogTargetDir();
280 }
281 this.startTs = backupCtx.getStartTs();
282 this.completeTs = backupCtx.getEndTs();
283 this.loadTableList(backupCtx.getTableNames());
284 }
285
286
287
288
289
290
291 public BackupManifest(BackupInfo backupCtx, TableName table) {
292 this.backupId = backupCtx.getBackupId();
293 this.type = backupCtx.getType();
294 this.rootDir = backupCtx.getTargetRootDir();
295 this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir();
296 if (this.type == BackupType.INCREMENTAL) {
297 this.logBackupDir = backupCtx.getHLogTargetDir();
298 }
299 this.startTs = backupCtx.getStartTs();
300 this.completeTs = backupCtx.getEndTs();
301 List<TableName> tables = new ArrayList<TableName>();
302 tables.add(table);
303 this.loadTableList(tables);
304 }
305
306
307
308
309
310
311
312
313 public BackupManifest(Configuration conf, Path backupPath) throws BackupException {
314 if (LOG.isDebugEnabled()) {
315 LOG.debug("Loading manifest from: " + backupPath.toString());
316 }
317
318
319
320
321 this.tableBackupDir = backupPath.toString();
322 this.config = conf;
323 try {
324
325 FileSystem fs = backupPath.getFileSystem(conf);
326 FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null);
327 if (subFiles == null) {
328 String errorMsg = backupPath.toString() + " does not exist";
329 LOG.error(errorMsg);
330 throw new IOException(errorMsg);
331 }
332 for (FileStatus subFile : subFiles) {
333 if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
334
335
336 FSDataInputStream in = fs.open(subFile.getPath());
337 long len = subFile.getLen();
338 byte[] pbBytes = new byte[(int) len];
339 in.readFully(pbBytes);
340 BackupProtos.BackupManifest proto = null;
341 try{
342 proto = parseFrom(pbBytes);
343 } catch(Exception e){
344 throw new BackupException(e);
345 }
346 this.version = proto.getVersion();
347 this.backupId = proto.getBackupId();
348 this.type = BackupType.valueOf(proto.getType().name());
349
350
351
352 this.rootDir = backupPath.getParent().getParent().getParent().toString();
353
354 Path p = backupPath.getParent();
355 if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
356 this.rootDir = p.getParent().toString();
357 } else {
358 this.rootDir = p.getParent().getParent().toString();
359 }
360
361 loadTableList(proto);
362 this.startTs = proto.getStartTs();
363 this.completeTs = proto.getCompleteTs();
364 loadIncrementalTimestampMap(proto);
365 loadDependency(proto);
366
367 LOG.debug("Loaded manifest instance from manifest file: "
368 + BackupClientUtil.getPath(subFile.getPath()));
369 return;
370 }
371 }
372 String errorMsg = "No manifest file found in: " + backupPath.toString();
373 throw new IOException(errorMsg);
374
375 } catch (IOException e) {
376 throw new BackupException(e.getMessage());
377 }
378 }
379
380 private void loadIncrementalTimestampMap(BackupProtos.BackupManifest proto) {
381 List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
382 if(list == null || list.size() == 0) return;
383 this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>();
384 for(BackupProtos.TableServerTimestamp tst: list){
385 TableName tn = ProtobufUtil.toTableName(tst.getTable());
386 HashMap<String, Long> map = this.incrTimeRanges.get(tn);
387 if(map == null){
388 map = new HashMap<String, Long>();
389 this.incrTimeRanges.put(tn, map);
390 }
391 List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
392 for(BackupProtos.ServerTimestamp stm: listSt) {
393 map.put(stm.getServer(), stm.getTimestamp());
394 }
395 }
396 }
397
398 private void loadDependency(BackupProtos.BackupManifest proto) {
399 if(LOG.isDebugEnabled()) {
400 LOG.debug("load dependency for: "+proto.getBackupId());
401 }
402
403 dependency = new HashMap<String, BackupImage>();
404 List<BackupProtos.BackupImage> list = proto.getDependentBackupImageList();
405 for (BackupProtos.BackupImage im : list) {
406 BackupImage bim = BackupImage.fromProto(im);
407 if(im.getBackupId() != null){
408 dependency.put(im.getBackupId(), bim);
409 } else{
410 LOG.warn("Load dependency for backup manifest: "+ backupId+
411 ". Null backup id in dependent image");
412 }
413 }
414 }
415
416 private void loadTableList(BackupProtos.BackupManifest proto) {
417 this.tableList = new ArrayList<TableName>();
418 List<TableProtos.TableName> list = proto.getTableListList();
419 for (TableProtos.TableName name: list) {
420 this.tableList.add(ProtobufUtil.toTableName(name));
421 }
422 }
423
424 public BackupType getType() {
425 return type;
426 }
427
428 public void setType(BackupType type) {
429 this.type = type;
430 }
431
432
433
434
435
436 private void loadTableList(List<TableName> tableList) {
437
438 this.tableList = this.getTableList();
439 if (this.tableList.size() > 0) {
440 this.tableList.clear();
441 }
442 for (int i = 0; i < tableList.size(); i++) {
443 this.tableList.add(tableList.get(i));
444 }
445
446 LOG.debug(tableList.size() + " tables exist in table set.");
447 }
448
449
450
451
452
453 public ArrayList<TableName> getTableList() {
454 if (this.tableList == null) {
455 this.tableList = new ArrayList<TableName>();
456 }
457 return this.tableList;
458 }
459
460
461
462
463
464
465 public void store(Configuration conf) throws BackupException {
466 byte[] data = toByteArray();
467
468
469 Path manifestFilePath =
470 new Path(new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir))
471 ,MANIFEST_FILE_NAME);
472 try {
473 FSDataOutputStream out =
474 manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);
475 out.write(data);
476 out.close();
477 } catch (IOException e) {
478 throw new BackupException(e.getMessage());
479 }
480
481 LOG.info("Manifest file stored to " + manifestFilePath);
482 }
483
484
485
486
487
488 public byte[] toByteArray() {
489 BackupProtos.BackupManifest.Builder builder = BackupProtos.BackupManifest.newBuilder();
490 builder.setVersion(this.version);
491 builder.setBackupId(this.backupId);
492 builder.setType(BackupProtos.BackupType.valueOf(this.type.name()));
493 setTableList(builder);
494 builder.setStartTs(this.startTs);
495 builder.setCompleteTs(this.completeTs);
496 setIncrementalTimestampMap(builder);
497 setDependencyMap(builder);
498 return builder.build().toByteArray();
499 }
500
501 private void setIncrementalTimestampMap(BackupProtos.BackupManifest.Builder builder) {
502 if (this.incrTimeRanges == null) {
503 return;
504 }
505 for (Entry<TableName, HashMap<String,Long>> entry: this.incrTimeRanges.entrySet()) {
506 TableName key = entry.getKey();
507 HashMap<String, Long> value = entry.getValue();
508 BackupProtos.TableServerTimestamp.Builder tstBuilder =
509 BackupProtos.TableServerTimestamp.newBuilder();
510 tstBuilder.setTable(ProtobufUtil.toProtoTableName(key));
511
512 for (String s : value.keySet()) {
513 BackupProtos.ServerTimestamp.Builder stBuilder = BackupProtos.ServerTimestamp.newBuilder();
514 stBuilder.setServer(s);
515 stBuilder.setTimestamp(value.get(s));
516 tstBuilder.addServerTimestamp(stBuilder.build());
517 }
518 builder.addTstMap(tstBuilder.build());
519 }
520 }
521
522 private void setDependencyMap(BackupProtos.BackupManifest.Builder builder) {
523 for (BackupImage image: getDependency().values()) {
524 builder.addDependentBackupImage(image.toProto());
525 }
526 }
527
528 private void setTableList(BackupProtos.BackupManifest.Builder builder) {
529 for(TableName name: tableList){
530 builder.addTableList(ProtobufUtil.toProtoTableName(name));
531 }
532 }
533
534
535
536
537
538
539
540 private static BackupProtos.BackupManifest parseFrom(final byte[] pbBytes)
541 throws DeserializationException {
542 BackupProtos.BackupManifest proto;
543 try {
544 proto = BackupProtos.BackupManifest.parseFrom(pbBytes);
545 } catch (InvalidProtocolBufferException e) {
546 throw new DeserializationException(e);
547 }
548 return proto;
549 }
550
551
552
553
554
555 public String getVersion() {
556 return version;
557 }
558
559
560
561
562
563 public BackupImage getBackupImage() {
564 return this.getDependency().get(this.backupId);
565 }
566
567
568
569
570
571 public void addDependentImage(BackupImage image) {
572 this.getDependency().get(this.backupId).addAncestor(image);
573 this.setDependencyMap(this.getDependency(), image);
574 }
575
576
577
578
579
580
581
582 public Map<String, BackupImage> getDependency() {
583 if (this.dependency == null) {
584 this.dependency = new HashMap<String, BackupImage>();
585 LOG.debug(this.rootDir + " " + this.backupId + " " + this.type);
586 this.dependency.put(this.backupId,
587 new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs,
588 this.completeTs));
589 }
590 return this.dependency;
591 }
592
593
594
595
596
597 public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
598 this.incrTimeRanges = incrTimestampMap;
599 }
600
601
602 public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
603 if (this.incrTimeRanges == null) {
604 this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>();
605 }
606 return this.incrTimeRanges;
607 }
608
609
610
611
612
613
614
615 public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
616 TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
617 for (BackupImage image : this.getDependency().values()) {
618 restoreImages.put(Long.valueOf(image.startTs), image);
619 }
620 return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
621 : (restoreImages.values()));
622 }
623
624
625
626
627
628
629
630 public ArrayList<BackupImage> getDependentListByTable(TableName table) {
631 ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
632 ArrayList<BackupImage> imageList = getRestoreDependentList(true);
633 for (BackupImage image : imageList) {
634 if (image.hasTable(table)) {
635 tableImageList.add(image);
636 if (image.getType() == BackupType.FULL) {
637 break;
638 }
639 }
640 }
641 Collections.reverse(tableImageList);
642 return tableImageList;
643 }
644
645
646
647
648
649
650
651
652 public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
653 ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
654 ArrayList<BackupImage> imageList = getRestoreDependentList(false);
655 for (BackupImage image : imageList) {
656 if (image.hasTable(table)) {
657 tableImageList.add(image);
658 }
659 }
660 return tableImageList;
661 }
662
663
664
665
666
667
668
669 private void setDependencyMap(Map<String, BackupImage> map, BackupImage image) {
670 if (image == null) {
671 return;
672 } else {
673 map.put(image.getBackupId(), image);
674 for (BackupImage img : image.getAncestors()) {
675 setDependencyMap(map, img);
676 }
677 }
678 }
679
680
681
682
683
684
685
686 public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
687
688
689
690
691 if (image1.getType() == BackupType.INCREMENTAL) {
692 return false;
693 }
694 if (image1.getStartTs() < image2.getStartTs()) {
695 return false;
696 }
697 List<TableName> image1TableList = image1.getTableNames();
698 List<TableName> image2TableList = image2.getTableNames();
699 boolean found = false;
700 for (int i = 0; i < image2TableList.size(); i++) {
701 found = false;
702 for (int j = 0; j < image1TableList.size(); j++) {
703 if (image2TableList.get(i).equals(image1TableList.get(j))) {
704 found = true;
705 break;
706 }
707 }
708 if (!found) {
709 return false;
710 }
711 }
712
713 LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
714 return true;
715 }
716
717
718
719
720
721
722
723 public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
724
725
726
727
728 for (BackupImage image1 : fullImages) {
729 if (image1.getType() == BackupType.INCREMENTAL) {
730 return false;
731 }
732 if (image1.getStartTs() < image.getStartTs()) {
733 return false;
734 }
735 }
736
737 ArrayList<String> image1TableList = new ArrayList<String>();
738 for (BackupImage image1 : fullImages) {
739 List<TableName> tableList = image1.getTableNames();
740 for (TableName table : tableList) {
741 image1TableList.add(table.getNameAsString());
742 }
743 }
744 ArrayList<String> image2TableList = new ArrayList<String>();
745 List<TableName> tableList = image.getTableNames();
746 for (TableName table : tableList) {
747 image2TableList.add(table.getNameAsString());
748 }
749
750 for (int i = 0; i < image2TableList.size(); i++) {
751 if (image1TableList.contains(image2TableList.get(i)) == false) {
752 return false;
753 }
754 }
755
756 LOG.debug("Full image set can cover image " + image.getBackupId());
757 return true;
758 }
759 }