1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.LinkedList;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Map.Entry;
32 import java.util.Set;
33 import java.util.TreeMap;
34 import java.util.concurrent.ThreadPoolExecutor;
35
36 import com.google.common.collect.ListMultimap;
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.hbase.classification.InterfaceAudience;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.fs.FileStatus;
42 import org.apache.hadoop.fs.FileSystem;
43 import org.apache.hadoop.fs.Path;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.TableName;
48 import org.apache.hadoop.hbase.backup.HFileArchiver;
49 import org.apache.hadoop.hbase.MetaTableAccessor;
50 import org.apache.hadoop.hbase.client.Connection;
51 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
52 import org.apache.hadoop.hbase.io.HFileLink;
53 import org.apache.hadoop.hbase.io.Reference;
54 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
55 import org.apache.hadoop.hbase.mob.MobUtils;
56 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
57 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
58 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
61 import org.apache.hadoop.hbase.regionserver.HRegion;
62 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
63 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
64 import org.apache.hadoop.hbase.security.access.AccessControlClient;
65 import org.apache.hadoop.hbase.security.access.TablePermission;
66 import org.apache.hadoop.hbase.util.Bytes;
67 import org.apache.hadoop.hbase.util.FSUtils;
68 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
69 import org.apache.hadoop.hbase.util.Pair;
70 import org.apache.hadoop.io.IOUtils;
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113 @InterfaceAudience.Private
114 public class RestoreSnapshotHelper {
115 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
116
117 private final Map<byte[], byte[]> regionsMap =
118 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
119
120 private final Map<String, Pair<String, String> > parentsMap =
121 new HashMap<String, Pair<String, String> >();
122
123 private final ForeignExceptionDispatcher monitor;
124 private final MonitoredTask status;
125
126 private final SnapshotManifest snapshotManifest;
127 private final SnapshotDescription snapshotDesc;
128 private final TableName snapshotTable;
129
130 private final HTableDescriptor tableDesc;
131 private final Path rootDir;
132 private final Path tableDir;
133
134 private final Configuration conf;
135 private final FileSystem fs;
136 private final boolean createBackRefs;
137
138 public RestoreSnapshotHelper(final Configuration conf,
139 final FileSystem fs,
140 final SnapshotManifest manifest,
141 final HTableDescriptor tableDescriptor,
142 final Path rootDir,
143 final ForeignExceptionDispatcher monitor,
144 final MonitoredTask status) {
145 this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
146 }
147
148 public RestoreSnapshotHelper(final Configuration conf,
149 final FileSystem fs,
150 final SnapshotManifest manifest,
151 final HTableDescriptor tableDescriptor,
152 final Path rootDir,
153 final ForeignExceptionDispatcher monitor,
154 final MonitoredTask status,
155 final boolean createBackRefs)
156 {
157 this.fs = fs;
158 this.conf = conf;
159 this.snapshotManifest = manifest;
160 this.snapshotDesc = manifest.getSnapshotDescription();
161 this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
162 this.tableDesc = tableDescriptor;
163 this.rootDir = rootDir;
164 this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
165 this.monitor = monitor;
166 this.status = status;
167 this.createBackRefs = createBackRefs;
168 }
169
170
171
172
173
174 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
175 ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
176 try {
177 return restoreHdfsRegions(exec);
178 } finally {
179 exec.shutdown();
180 }
181 }
182
183 private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
184 LOG.debug("starting restore");
185
186 Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
187 if (regionManifests == null) {
188 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
189 return null;
190 }
191
192 RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
193
194
195
196 Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
197
198 List<HRegionInfo> tableRegions = getTableRegions();
199
200 HRegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor()
201 .getTableName());
202 if (tableRegions != null) {
203
204 if (regionNames.contains(mobRegion.getEncodedName())) {
205 monitor.rethrowException();
206 status.setStatus("Restoring mob region...");
207 List<HRegionInfo> mobRegions = new ArrayList<>(1);
208 mobRegions.add(mobRegion);
209 restoreHdfsMobRegions(exec, regionManifests, mobRegions);
210 regionNames.remove(mobRegion.getEncodedName());
211 status.setStatus("Finished restoring mob region.");
212 }
213 }
214 if (regionNames.contains(mobRegion.getEncodedName())) {
215
216 monitor.rethrowException();
217 status.setStatus("Cloning mob region...");
218 cloneHdfsMobRegion(regionManifests, mobRegion);
219 regionNames.remove(mobRegion.getEncodedName());
220 status.setStatus("Finished cloning mob region.");
221 }
222
223
224
225 if (tableRegions != null) {
226 monitor.rethrowException();
227 for (HRegionInfo regionInfo: tableRegions) {
228 String regionName = regionInfo.getEncodedName();
229 if (regionNames.contains(regionName)) {
230 LOG.info("region to restore: " + regionName);
231 regionNames.remove(regionName);
232 metaChanges.addRegionToRestore(regionInfo);
233 } else {
234 LOG.info("region to remove: " + regionName);
235 metaChanges.addRegionToRemove(regionInfo);
236 }
237 }
238 }
239
240
241 List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
242 if (regionNames.size() > 0) {
243 monitor.rethrowException();
244
245 if (regionNames.contains(mobRegion.getEncodedName())) {
246 cloneHdfsMobRegion(regionManifests, mobRegion);
247 regionNames.remove(mobRegion.getEncodedName());
248 }
249 for (String regionName: regionNames) {
250 LOG.info("region to add: " + regionName);
251 regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
252 }
253 }
254
255
256
257
258 monitor.rethrowException();
259 status.setStatus("Cloning regions...");
260 HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
261 metaChanges.setNewRegions(clonedRegions);
262 status.setStatus("Finished cloning regions.");
263
264
265 monitor.rethrowException();
266 status.setStatus("Restoring table regions...");
267 restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
268 status.setStatus("Finished restoring all table regions.");
269
270
271 monitor.rethrowException();
272 status.setStatus("Starting to delete excess regions from table");
273 removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
274 status.setStatus("Finished deleting excess regions from table.");
275
276 return metaChanges;
277 }
278
279
280
281
282 public static class RestoreMetaChanges {
283 private final Map<String, Pair<String, String> > parentsMap;
284
285 private List<HRegionInfo> regionsToRestore = null;
286 private List<HRegionInfo> regionsToRemove = null;
287 private List<HRegionInfo> regionsToAdd = null;
288
289 RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
290 this.parentsMap = parentsMap;
291 }
292
293
294
295
296 public boolean hasRegionsToAdd() {
297 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
298 }
299
300
301
302
303
304
305
306 public List<HRegionInfo> getRegionsToAdd() {
307 return this.regionsToAdd;
308 }
309
310
311
312
313 public boolean hasRegionsToRestore() {
314 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
315 }
316
317
318
319
320
321
322 public List<HRegionInfo> getRegionsToRestore() {
323 return this.regionsToRestore;
324 }
325
326
327
328
329 public boolean hasRegionsToRemove() {
330 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
331 }
332
333
334
335
336
337
338
339 public List<HRegionInfo> getRegionsToRemove() {
340 return this.regionsToRemove;
341 }
342
343 void setNewRegions(final HRegionInfo[] hris) {
344 if (hris != null) {
345 regionsToAdd = Arrays.asList(hris);
346 } else {
347 regionsToAdd = null;
348 }
349 }
350
351 void addRegionToRemove(final HRegionInfo hri) {
352 if (regionsToRemove == null) {
353 regionsToRemove = new LinkedList<HRegionInfo>();
354 }
355 regionsToRemove.add(hri);
356 }
357
358 void addRegionToRestore(final HRegionInfo hri) {
359 if (regionsToRestore == null) {
360 regionsToRestore = new LinkedList<HRegionInfo>();
361 }
362 regionsToRestore.add(hri);
363 }
364
365 public void updateMetaParentRegions(Connection connection,
366 final List<HRegionInfo> regionInfos) throws IOException {
367 if (regionInfos == null || parentsMap.isEmpty()) return;
368
369
370 Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
371 List<HRegionInfo> parentRegions = new LinkedList<>();
372 for (HRegionInfo regionInfo: regionInfos) {
373 if (regionInfo.isSplitParent()) {
374 parentRegions.add(regionInfo);
375 } else {
376 regionsByName.put(regionInfo.getEncodedName(), regionInfo);
377 }
378 }
379
380
381 for (HRegionInfo regionInfo: parentRegions) {
382 Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
383 if (daughters == null) {
384
385
386 LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
387 continue;
388 }
389
390
391 if (daughters.getSecond() == null) {
392 daughters.setSecond(daughters.getFirst());
393 }
394
395 LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
396 MetaTableAccessor.addRegionToMeta(connection, regionInfo,
397 regionsByName.get(daughters.getFirst()),
398 regionsByName.get(daughters.getSecond()));
399 }
400 }
401 }
402
403
404
405
406 private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
407 throws IOException {
408 if (regions == null || regions.size() == 0) return;
409 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
410 @Override
411 public void editRegion(final HRegionInfo hri) throws IOException {
412 HFileArchiver.archiveRegion(conf, fs, hri);
413 }
414 });
415 }
416
417
418
419
420 private void restoreHdfsRegions(final ThreadPoolExecutor exec,
421 final Map<String, SnapshotRegionManifest> regionManifests,
422 final List<HRegionInfo> regions) throws IOException {
423 if (regions == null || regions.size() == 0) return;
424 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
425 @Override
426 public void editRegion(final HRegionInfo hri) throws IOException {
427 restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
428 }
429 });
430 }
431
432
433
434
435 private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
436 final Map<String, SnapshotRegionManifest> regionManifests,
437 final List<HRegionInfo> regions) throws IOException {
438 if (regions == null || regions.size() == 0) return;
439 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
440 @Override
441 public void editRegion(final HRegionInfo hri) throws IOException {
442 restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
443 }
444 });
445 }
446
447 private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
448 final SnapshotRegionManifest manifest) {
449 Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
450 new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
451 for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
452 familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
453 new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
454 }
455 return familyMap;
456 }
457
458
459
460
461
462 private void restoreRegion(final HRegionInfo regionInfo,
463 final SnapshotRegionManifest regionManifest) throws IOException {
464 restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
465 }
466
467
468
469
470
471 private void restoreMobRegion(final HRegionInfo regionInfo,
472 final SnapshotRegionManifest regionManifest) throws IOException {
473 if (regionManifest == null) {
474 return;
475 }
476 restoreRegion(regionInfo, regionManifest,
477 MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
478 }
479
480
481
482
483
484 private void restoreRegion(final HRegionInfo regionInfo,
485 final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
486 Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
487 getRegionHFileReferences(regionManifest);
488
489 String tableName = tableDesc.getTableName().getNameAsString();
490
491
492 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
493 byte[] family = Bytes.toBytes(familyDir.getName());
494 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
495 List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
496 snapshotFiles.remove(familyDir.getName());
497 if (snapshotFamilyFiles != null) {
498 List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
499 new ArrayList<SnapshotRegionManifest.StoreFile>();
500 for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
501 if (familyFiles.contains(storeFile.getName())) {
502
503 familyFiles.remove(storeFile.getName());
504 } else {
505
506 hfilesToAdd.add(storeFile);
507 }
508 }
509
510
511 for (String hfileName: familyFiles) {
512 Path hfile = new Path(familyDir, hfileName);
513 LOG.trace("Removing hfile=" + hfileName +
514 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
515 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
516 }
517
518
519 for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
520 LOG.debug("Adding HFileLink " + storeFile.getName() +
521 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
522 restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
523 }
524 } else {
525
526 LOG.trace("Removing family=" + Bytes.toString(family) +
527 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
528 HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
529 fs.delete(familyDir, true);
530 }
531 }
532
533
534 for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
535 snapshotFiles.entrySet()) {
536 Path familyDir = new Path(regionDir, familyEntry.getKey());
537 if (!fs.mkdirs(familyDir)) {
538 throw new IOException("Unable to create familyDir=" + familyDir);
539 }
540
541 for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
542 LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
543 restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
544 }
545 }
546 }
547
548
549
550
551 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
552 Set<String> familyFiles = new HashSet<String>();
553
554 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
555 if (hfiles == null) return familyFiles;
556
557 for (FileStatus hfileRef: hfiles) {
558 String hfileName = hfileRef.getPath().getName();
559 familyFiles.add(hfileName);
560 }
561
562 return familyFiles;
563 }
564
565
566
567
568
569 private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
570 final Map<String, SnapshotRegionManifest> regionManifests,
571 final List<HRegionInfo> regions) throws IOException {
572 if (regions == null || regions.size() == 0) return null;
573
574 final Map<String, HRegionInfo> snapshotRegions =
575 new HashMap<String, HRegionInfo>(regions.size());
576
577
578 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
579 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
580
581 HRegionInfo snapshotRegionInfo = regions.get(i);
582 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
583
584
585 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
586 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
587 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
588 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
589
590
591 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
592 }
593
594
595 ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDir,
596 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
597 @Override
598 public void fillRegion(final HRegion region) throws IOException {
599 HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
600 cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
601 }
602 });
603
604 return clonedRegionsInfo;
605 }
606
607
608
609
610
611 private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
612 final HRegionInfo region) throws IOException {
613
614 Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
615 cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
616 }
617
618
619
620
621
622
623
624
625
626
627
628
629 private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo,
630 final SnapshotRegionManifest manifest) throws IOException {
631 final String tableName = tableDesc.getTableName().getNameAsString();
632 for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
633 Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
634 for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
635 LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
636 restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
637 }
638 }
639 }
640
641
642
643
644
645
646
647
648
649
650
651
652 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
653 final SnapshotRegionManifest manifest) throws IOException {
654 cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
655 manifest);
656 }
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
672 final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
673 throws IOException {
674 String hfileName = storeFile.getName();
675 if (HFileLink.isHFileLink(hfileName)) {
676 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
677 } else if (StoreFileInfo.isReference(hfileName)) {
678 restoreReferenceFile(familyDir, regionInfo, storeFile);
679 } else {
680 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
681 }
682 }
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
703 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
704 String hfileName = storeFile.getName();
705
706
707 Path refPath =
708 StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable
709 .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo
710 .getEncodedName()), familyDir.getName()), hfileName));
711 String snapshotRegionName = refPath.getParent().getParent().getName();
712 String fileName = refPath.getName();
713
714
715 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
716 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
717
718
719 Path linkPath = null;
720 String refLink = fileName;
721 if (!HFileLink.isHFileLink(fileName)) {
722 refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
723 linkPath = new Path(familyDir,
724 HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
725 }
726
727 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
728
729
730 if (storeFile.hasReference()) {
731 Reference reference = Reference.convert(storeFile.getReference());
732 reference.write(fs, outPath);
733 } else {
734 InputStream in;
735 if (linkPath != null) {
736 in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
737 } else {
738 linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
739 regionInfo.getEncodedName()), familyDir.getName()), hfileName);
740 in = fs.open(linkPath);
741 }
742 OutputStream out = fs.create(outPath);
743 IOUtils.copyBytes(in, out, conf);
744 }
745
746
747 String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
748 if (regionName == null) {
749 regionName = regionInfo.getEncodedName();
750 }
751 LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
752 synchronized (parentsMap) {
753 Pair<String, String> daughters = parentsMap.get(clonedRegionName);
754 if (daughters == null) {
755
756
757 daughters = new Pair<String, String>(regionName, regionName);
758 parentsMap.put(clonedRegionName, daughters);
759 } else if (!regionName.equals(daughters.getFirst())) {
760 daughters.setSecond(regionName);
761 }
762 }
763 }
764
765
766
767
768
769
770
771
772
773 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
774 HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
775 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
776 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
777 regionInfo.setOffline(snapshotRegionInfo.isOffline());
778 return regionInfo;
779 }
780
781
782
783
784 private List<HRegionInfo> getTableRegions() throws IOException {
785 LOG.debug("get table regions: " + tableDir);
786 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
787 if (regionDirs == null) return null;
788
789 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
790 for (FileStatus regionDir: regionDirs) {
791 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
792 regions.add(hri);
793 }
794 LOG.debug("found " + regions.size() + " regions for table=" +
795 tableDesc.getTableName().getNameAsString());
796 return regions;
797 }
798
799
800
801
802
803
804
805
806
807 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
808 final TableName tableName) throws IOException {
809 HTableDescriptor htd = new HTableDescriptor(tableName);
810 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
811 htd.addFamily(hcd);
812 }
813 for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
814 snapshotTableDescriptor.getValues().entrySet()) {
815 htd.setValue(e.getKey(), e.getValue());
816 }
817 for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
818 htd.setConfiguration(e.getKey(), e.getValue());
819 }
820 return htd;
821 }
822
823
824
825
826
827
828
829
830
831
832 public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir,
833 Path restoreDir, String snapshotName) throws IOException {
834
835 if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
836 throw new IllegalArgumentException("Filesystems for restore directory (" +
837 restoreDir.getFileSystem(conf).getUri() + ") and HBase root directory (" +
838 rootDir.getFileSystem(conf).getUri() + ") should be the same");
839 }
840 if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
841 throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
842 "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
843 }
844
845 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
846 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
847 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
848
849 MonitoredTask status = TaskMonitor.get().createStatus(
850 "Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
851 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
852
853
854
855 RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
856 manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
857 helper.restoreHdfsRegions();
858
859 if (LOG.isDebugEnabled()) {
860 LOG.debug("Restored table dir:" + restoreDir);
861 FSUtils.logFileSystemState(fs, restoreDir, LOG);
862 }
863 }
864
865 public static void restoreSnapshotACL(SnapshotDescription snapshot, TableName newTableName,
866 Configuration conf) throws IOException {
867 if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) {
868 LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName);
869 ListMultimap<String, TablePermission> perms =
870 ProtobufUtil.toUserTablePermissions(snapshot.getUsersAndPermissions());
871 try {
872 for (Entry<String, TablePermission> e : perms.entries()) {
873 String user = e.getKey();
874 TablePermission perm = e.getValue();
875 perm.setTableName(newTableName);
876 AccessControlClient.grant(conf, perm.getTableName(), user, perm.getFamily(),
877 perm.getQualifier(), perm.getActions());
878 }
879 } catch (Throwable e) {
880 throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot
881 + ", table: " + newTableName, e);
882 }
883 }
884 }
885 }