View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.HashMap;
27  import java.util.HashSet;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Map.Entry;
32  import java.util.Set;
33  import java.util.TreeMap;
34  import java.util.concurrent.ThreadPoolExecutor;
35  
36  import com.google.common.collect.ListMultimap;
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.FileStatus;
42  import org.apache.hadoop.fs.FileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.backup.HFileArchiver;
49  import org.apache.hadoop.hbase.MetaTableAccessor;
50  import org.apache.hadoop.hbase.client.Connection;
51  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
52  import org.apache.hadoop.hbase.io.HFileLink;
53  import org.apache.hadoop.hbase.io.Reference;
54  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
55  import org.apache.hadoop.hbase.mob.MobUtils;
56  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
57  import org.apache.hadoop.hbase.monitoring.TaskMonitor;
58  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
61  import org.apache.hadoop.hbase.regionserver.HRegion;
62  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
63  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
64  import org.apache.hadoop.hbase.security.access.AccessControlClient;
65  import org.apache.hadoop.hbase.security.access.TablePermission;
66  import org.apache.hadoop.hbase.util.Bytes;
67  import org.apache.hadoop.hbase.util.FSUtils;
68  import org.apache.hadoop.hbase.util.ModifyRegionUtils;
69  import org.apache.hadoop.hbase.util.Pair;
70  import org.apache.hadoop.io.IOUtils;
71  
72  /**
73   * Helper to Restore/Clone a Snapshot
74   *
75   * <p>The helper assumes that a table is already created, and by calling restore()
76   * the content present in the snapshot will be restored as the new content of the table.
77   *
78   * <p>Clone from Snapshot: If the target table is empty, the restore operation
79   * is just a "clone operation", where the only operations are:
80   * <ul>
81   *  <li>for each region in the snapshot create a new region
82   *    (note that the region will have a different name, since the encoding contains the table name)
83   *  <li>for each file in the region create a new HFileLink to point to the original file.
84   *  <li>restore the logs, if any
85   * </ul>
86   *
87   * <p>Restore from Snapshot:
88   * <ul>
89   *  <li>for each region in the table verify which are available in the snapshot and which are not
90   *    <ul>
91   *    <li>if the region is not present in the snapshot, remove it.
92   *    <li>if the region is present in the snapshot
93   *      <ul>
94   *      <li>for each file in the table region verify which are available in the snapshot
95   *        <ul>
96   *          <li>if the hfile is not present in the snapshot, remove it
97   *          <li>if the hfile is present, keep it (nothing to do)
98   *        </ul>
99   *      <li>for each file in the snapshot region but not in the table
100  *        <ul>
101  *          <li>create a new HFileLink that point to the original file
102  *        </ul>
103  *      </ul>
104  *    </ul>
105  *  <li>for each region in the snapshot not present in the current table state
106  *    <ul>
107  *    <li>create a new region and for each file in the region create a new HFileLink
108  *      (This is the same as the clone operation)
109  *    </ul>
110  *  <li>restore the logs, if any
111  * </ul>
112  */
113 @InterfaceAudience.Private
114 public class RestoreSnapshotHelper {
115   private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
116 
117   private final Map<byte[], byte[]> regionsMap =
118         new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
119 
120   private final Map<String, Pair<String, String> > parentsMap =
121       new HashMap<String, Pair<String, String> >();
122 
123   private final ForeignExceptionDispatcher monitor;
124   private final MonitoredTask status;
125 
126   private final SnapshotManifest snapshotManifest;
127   private final SnapshotDescription snapshotDesc;
128   private final TableName snapshotTable;
129 
130   private final HTableDescriptor tableDesc;
131   private final Path rootDir;
132   private final Path tableDir;
133 
134   private final Configuration conf;
135   private final FileSystem fs;
136   private final boolean createBackRefs;
137 
138   public RestoreSnapshotHelper(final Configuration conf,
139       final FileSystem fs,
140       final SnapshotManifest manifest,
141       final HTableDescriptor tableDescriptor,
142       final Path rootDir,
143       final ForeignExceptionDispatcher monitor,
144       final MonitoredTask status) {
145     this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
146   }
147 
148   public RestoreSnapshotHelper(final Configuration conf,
149       final FileSystem fs,
150       final SnapshotManifest manifest,
151       final HTableDescriptor tableDescriptor,
152       final Path rootDir,
153       final ForeignExceptionDispatcher monitor,
154       final MonitoredTask status,
155       final boolean createBackRefs)
156   {
157     this.fs = fs;
158     this.conf = conf;
159     this.snapshotManifest = manifest;
160     this.snapshotDesc = manifest.getSnapshotDescription();
161     this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
162     this.tableDesc = tableDescriptor;
163     this.rootDir = rootDir;
164     this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
165     this.monitor = monitor;
166     this.status = status;
167     this.createBackRefs = createBackRefs;
168   }
169 
170   /**
171    * Restore the on-disk table to a specified snapshot state.
172    * @return the set of regions touched by the restore operation
173    */
174   public RestoreMetaChanges restoreHdfsRegions() throws IOException {
175     ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
176     try {
177       return restoreHdfsRegions(exec);
178     } finally {
179       exec.shutdown();
180     }
181   }
182 
183   private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
184     LOG.debug("starting restore");
185 
186     Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
187     if (regionManifests == null) {
188       LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
189       return null;
190     }
191 
192     RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
193 
194     // Take a copy of the manifest.keySet() since we are going to modify
195     // this instance, by removing the regions already present in the restore dir.
196     Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
197 
198     List<HRegionInfo> tableRegions = getTableRegions();
199 
200     HRegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor()
201         .getTableName());
202     if (tableRegions != null) {
203       // restore the mob region in case
204       if (regionNames.contains(mobRegion.getEncodedName())) {
205         monitor.rethrowException();
206         status.setStatus("Restoring mob region...");
207         List<HRegionInfo> mobRegions = new ArrayList<>(1);
208         mobRegions.add(mobRegion);
209         restoreHdfsMobRegions(exec, regionManifests, mobRegions);
210         regionNames.remove(mobRegion.getEncodedName());
211         status.setStatus("Finished restoring mob region.");
212       }
213     }
214     if (regionNames.contains(mobRegion.getEncodedName())) {
215       // add the mob region
216       monitor.rethrowException();
217       status.setStatus("Cloning mob region...");
218       cloneHdfsMobRegion(regionManifests, mobRegion);
219       regionNames.remove(mobRegion.getEncodedName());
220       status.setStatus("Finished cloning mob region.");
221     }
222 
223     // Identify which region are still available and which not.
224     // NOTE: we rely upon the region name as: "table name, start key, end key"
225     if (tableRegions != null) {
226       monitor.rethrowException();
227       for (HRegionInfo regionInfo: tableRegions) {
228         String regionName = regionInfo.getEncodedName();
229         if (regionNames.contains(regionName)) {
230           LOG.info("region to restore: " + regionName);
231           regionNames.remove(regionName);
232           metaChanges.addRegionToRestore(regionInfo);
233         } else {
234           LOG.info("region to remove: " + regionName);
235           metaChanges.addRegionToRemove(regionInfo);
236         }
237       }
238     }
239 
240     // Regions to Add: present in the snapshot but not in the current table
241     List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
242     if (regionNames.size() > 0) {
243       monitor.rethrowException();
244       // add the mob region
245       if (regionNames.contains(mobRegion.getEncodedName())) {
246         cloneHdfsMobRegion(regionManifests, mobRegion);
247         regionNames.remove(mobRegion.getEncodedName());
248       }
249       for (String regionName: regionNames) {
250         LOG.info("region to add: " + regionName);
251         regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
252       }
253     }
254 
255     // Create new regions cloning from the snapshot
256     // HBASE-20008: We need to call cloneHdfsRegions() before restoreHdfsRegions() because
257     // regionsMap is constructed in cloneHdfsRegions() and it can be used in restoreHdfsRegions().
258     monitor.rethrowException();
259     status.setStatus("Cloning regions...");
260     HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
261     metaChanges.setNewRegions(clonedRegions);
262     status.setStatus("Finished cloning regions.");
263 
264     // Restore regions using the snapshot data
265     monitor.rethrowException();
266     status.setStatus("Restoring table regions...");
267     restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
268     status.setStatus("Finished restoring all table regions.");
269 
270     // Remove regions from the current table
271     monitor.rethrowException();
272     status.setStatus("Starting to delete excess regions from table");
273     removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
274     status.setStatus("Finished deleting excess regions from table.");
275 
276     return metaChanges;
277   }
278 
279   /**
280    * Describe the set of operations needed to update hbase:meta after restore.
281    */
282   public static class RestoreMetaChanges {
283     private final Map<String, Pair<String, String> > parentsMap;
284 
285     private List<HRegionInfo> regionsToRestore = null;
286     private List<HRegionInfo> regionsToRemove = null;
287     private List<HRegionInfo> regionsToAdd = null;
288 
289     RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
290       this.parentsMap = parentsMap;
291     }
292 
293     /**
294      * @return true if there're new regions
295      */
296     public boolean hasRegionsToAdd() {
297       return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
298     }
299 
300     /**
301      * Returns the list of new regions added during the on-disk restore.
302      * The caller is responsible to add the regions to META.
303      * e.g MetaTableAccessor.addRegionsToMeta(...)
304      * @return the list of regions to add to META
305      */
306     public List<HRegionInfo> getRegionsToAdd() {
307       return this.regionsToAdd;
308     }
309 
310     /**
311      * @return true if there're regions to restore
312      */
313     public boolean hasRegionsToRestore() {
314       return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
315     }
316 
317     /**
318      * Returns the list of 'restored regions' during the on-disk restore.
319      * The caller is responsible to add the regions to hbase:meta if not present.
320      * @return the list of regions restored
321      */
322     public List<HRegionInfo> getRegionsToRestore() {
323       return this.regionsToRestore;
324     }
325 
326     /**
327      * @return true if there're regions to remove
328      */
329     public boolean hasRegionsToRemove() {
330       return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
331     }
332 
333     /**
334      * Returns the list of regions removed during the on-disk restore.
335      * The caller is responsible to remove the regions from META.
336      * e.g. MetaTableAccessor.deleteRegions(...)
337      * @return the list of regions to remove from META
338      */
339     public List<HRegionInfo> getRegionsToRemove() {
340       return this.regionsToRemove;
341     }
342 
343     void setNewRegions(final HRegionInfo[] hris) {
344       if (hris != null) {
345         regionsToAdd = Arrays.asList(hris);
346       } else {
347         regionsToAdd = null;
348       }
349     }
350 
351     void addRegionToRemove(final HRegionInfo hri) {
352       if (regionsToRemove == null) {
353         regionsToRemove = new LinkedList<HRegionInfo>();
354       }
355       regionsToRemove.add(hri);
356     }
357 
358     void addRegionToRestore(final HRegionInfo hri) {
359       if (regionsToRestore == null) {
360         regionsToRestore = new LinkedList<HRegionInfo>();
361       }
362       regionsToRestore.add(hri);
363     }
364 
365     public void updateMetaParentRegions(Connection connection,
366         final List<HRegionInfo> regionInfos) throws IOException {
367       if (regionInfos == null || parentsMap.isEmpty()) return;
368 
369       // Extract region names and offlined regions
370       Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
371       List<HRegionInfo> parentRegions = new LinkedList<>();
372       for (HRegionInfo regionInfo: regionInfos) {
373         if (regionInfo.isSplitParent()) {
374           parentRegions.add(regionInfo);
375         } else {
376           regionsByName.put(regionInfo.getEncodedName(), regionInfo);
377         }
378       }
379 
380       // Update Offline parents
381       for (HRegionInfo regionInfo: parentRegions) {
382         Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
383         if (daughters == null) {
384           // The snapshot contains an unreferenced region.
385           // It will be removed by the CatalogJanitor.
386           LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
387           continue;
388         }
389 
390         // One side of the split is already compacted
391         if (daughters.getSecond() == null) {
392           daughters.setSecond(daughters.getFirst());
393         }
394 
395         LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
396         MetaTableAccessor.addRegionToMeta(connection, regionInfo,
397           regionsByName.get(daughters.getFirst()),
398           regionsByName.get(daughters.getSecond()));
399       }
400     }
401   }
402 
403   /**
404    * Remove specified regions from the file-system, using the archiver.
405    */
406   private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
407       throws IOException {
408     if (regions == null || regions.size() == 0) return;
409     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
410       @Override
411       public void editRegion(final HRegionInfo hri) throws IOException {
412         HFileArchiver.archiveRegion(conf, fs, hri);
413       }
414     });
415   }
416 
417   /**
418    * Restore specified regions by restoring content to the snapshot state.
419    */
420   private void restoreHdfsRegions(final ThreadPoolExecutor exec,
421       final Map<String, SnapshotRegionManifest> regionManifests,
422       final List<HRegionInfo> regions) throws IOException {
423     if (regions == null || regions.size() == 0) return;
424     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
425       @Override
426       public void editRegion(final HRegionInfo hri) throws IOException {
427         restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
428       }
429     });
430   }
431 
432   /**
433    * Restore specified mob regions by restoring content to the snapshot state.
434    */
435   private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
436       final Map<String, SnapshotRegionManifest> regionManifests,
437       final List<HRegionInfo> regions) throws IOException {
438     if (regions == null || regions.size() == 0) return;
439     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
440       @Override
441       public void editRegion(final HRegionInfo hri) throws IOException {
442         restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
443       }
444     });
445   }
446 
447   private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
448       final SnapshotRegionManifest manifest) {
449     Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
450       new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
451     for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
452       familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
453         new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
454     }
455     return familyMap;
456   }
457 
458   /**
459    * Restore region by removing files not in the snapshot
460    * and adding the missing ones from the snapshot.
461    */
462   private void restoreRegion(final HRegionInfo regionInfo,
463       final SnapshotRegionManifest regionManifest) throws IOException {
464     restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
465   }
466 
467   /**
468    * Restore mob region by removing files not in the snapshot
469    * and adding the missing ones from the snapshot.
470    */
471   private void restoreMobRegion(final HRegionInfo regionInfo,
472       final SnapshotRegionManifest regionManifest) throws IOException {
473     if (regionManifest == null) {
474       return;
475     }
476     restoreRegion(regionInfo, regionManifest,
477       MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
478   }
479 
480   /**
481    * Restore region by removing files not in the snapshot
482    * and adding the missing ones from the snapshot.
483    */
484   private void restoreRegion(final HRegionInfo regionInfo,
485       final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
486     Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
487                 getRegionHFileReferences(regionManifest);
488 
489     String tableName = tableDesc.getTableName().getNameAsString();
490 
491     // Restore families present in the table
492     for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
493       byte[] family = Bytes.toBytes(familyDir.getName());
494       Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
495       List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
496           snapshotFiles.remove(familyDir.getName());
497       if (snapshotFamilyFiles != null) {
498         List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
499             new ArrayList<SnapshotRegionManifest.StoreFile>();
500         for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
501           if (familyFiles.contains(storeFile.getName())) {
502             // HFile already present
503             familyFiles.remove(storeFile.getName());
504           } else {
505             // HFile missing
506             hfilesToAdd.add(storeFile);
507           }
508         }
509 
510         // Remove hfiles not present in the snapshot
511         for (String hfileName: familyFiles) {
512           Path hfile = new Path(familyDir, hfileName);
513           LOG.trace("Removing hfile=" + hfileName +
514             " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
515           HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
516         }
517 
518         // Restore Missing files
519         for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
520           LOG.debug("Adding HFileLink " + storeFile.getName() +
521             " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
522           restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
523         }
524       } else {
525         // Family doesn't exists in the snapshot
526         LOG.trace("Removing family=" + Bytes.toString(family) +
527           " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
528         HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
529         fs.delete(familyDir, true);
530       }
531     }
532 
533     // Add families not present in the table
534     for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
535                                                                       snapshotFiles.entrySet()) {
536       Path familyDir = new Path(regionDir, familyEntry.getKey());
537       if (!fs.mkdirs(familyDir)) {
538         throw new IOException("Unable to create familyDir=" + familyDir);
539       }
540 
541       for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
542         LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
543         restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
544       }
545     }
546   }
547 
548   /**
549    * @return The set of files in the specified family directory.
550    */
551   private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
552     Set<String> familyFiles = new HashSet<String>();
553 
554     FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
555     if (hfiles == null) return familyFiles;
556 
557     for (FileStatus hfileRef: hfiles) {
558       String hfileName = hfileRef.getPath().getName();
559       familyFiles.add(hfileName);
560     }
561 
562     return familyFiles;
563   }
564 
565   /**
566    * Clone specified regions. For each region create a new region
567    * and create a HFileLink for each hfile.
568    */
569   private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
570       final Map<String, SnapshotRegionManifest> regionManifests,
571       final List<HRegionInfo> regions) throws IOException {
572     if (regions == null || regions.size() == 0) return null;
573 
574     final Map<String, HRegionInfo> snapshotRegions =
575       new HashMap<String, HRegionInfo>(regions.size());
576 
577     // clone region info (change embedded tableName with the new one)
578     HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
579     for (int i = 0; i < clonedRegionsInfo.length; ++i) {
580       // clone the region info from the snapshot region info
581       HRegionInfo snapshotRegionInfo = regions.get(i);
582       clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
583 
584       // add the region name mapping between snapshot and cloned
585       String snapshotRegionName = snapshotRegionInfo.getEncodedName();
586       String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
587       regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
588       LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
589 
590       // Add mapping between cloned region name and snapshot region info
591       snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
592     }
593 
594     // create the regions on disk
595     ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDir,
596       tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
597         @Override
598         public void fillRegion(final HRegion region) throws IOException {
599           HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
600           cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
601         }
602       });
603 
604     return clonedRegionsInfo;
605   }
606 
607   /**
608    * Clone the mob region. For the region create a new region
609    * and create a HFileLink for each hfile.
610    */
611   private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
612       final HRegionInfo region) throws IOException {
613     // clone region info (change embedded tableName with the new one)
614     Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
615     cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
616   }
617 
618   /**
619    * Clone region directory content from the snapshot info.
620    *
621    * Each region is encoded with the table name, so the cloned region will have
622    * a different region name.
623    *
624    * Instead of copying the hfiles a HFileLink is created.
625    *
626    * @param regionDir {@link Path} cloned dir
627    * @param snapshotRegionInfo
628    */
629   private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo,
630       final SnapshotRegionManifest manifest) throws IOException {
631     final String tableName = tableDesc.getTableName().getNameAsString();
632     for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
633       Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
634       for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
635         LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
636         restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
637       }
638     }
639   }
640 
641   /**
642    * Clone region directory content from the snapshot info.
643    *
644    * Each region is encoded with the table name, so the cloned region will have
645    * a different region name.
646    *
647    * Instead of copying the hfiles a HFileLink is created.
648    *
649    * @param region {@link HRegion} cloned
650    * @param snapshotRegionInfo
651    */
652   private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
653       final SnapshotRegionManifest manifest) throws IOException {
654     cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
655       manifest);
656   }
657 
658   /**
659    * Create a new {@link HFileLink} to reference the store file.
660    * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
661    * <ul>
662    *   <li>hfile: abc -> table=region-abc
663    *   <li>reference: abc.1234 -> table=region-abc.1234
664    *   <li>hfilelink: table=region-hfile -> table=region-hfile
665    * </ul>
666    * @param familyDir destination directory for the store file
667    * @param regionInfo destination region info for the table
668    * @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
669    * @param createBackRef - Whether back reference should be created. Defaults to true.
670    */
671   private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
672       final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
673           throws IOException {
674     String hfileName = storeFile.getName();
675     if (HFileLink.isHFileLink(hfileName)) {
676       HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
677     } else if (StoreFileInfo.isReference(hfileName)) {
678       restoreReferenceFile(familyDir, regionInfo, storeFile);
679     } else {
680       HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
681     }
682   }
683 
684   /**
685    * Create a new {@link Reference} as copy of the source one.
686    * <p><blockquote><pre>
687    * The source table looks like:
688    *    1234/abc      (original file)
689    *    5678/abc.1234 (reference file)
690    *
691    * After the clone operation looks like:
692    *   wxyz/table=1234-abc
693    *   stuv/table=1234-abc.wxyz
694    *
695    * NOTE that the region name in the clone changes (md5 of regioninfo)
696    * and the reference should reflect that change.
697    * </pre></blockquote>
698    * @param familyDir destination directory for the store file
699    * @param regionInfo destination region info for the table
700    * @param storeFile reference file name
701    */
702   private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
703       final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
704     String hfileName = storeFile.getName();
705 
706     // Extract the referred information (hfile name and parent region)
707     Path refPath =
708         StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable
709             .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo
710             .getEncodedName()), familyDir.getName()), hfileName)); 
711     String snapshotRegionName = refPath.getParent().getParent().getName();
712     String fileName = refPath.getName();
713 
714     // The new reference should have the cloned region name as parent, if it is a clone.
715     String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
716     if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
717 
718     // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
719     Path linkPath = null;
720     String refLink = fileName;
721     if (!HFileLink.isHFileLink(fileName)) {
722       refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
723       linkPath = new Path(familyDir,
724         HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
725     }
726 
727     Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
728 
729     // Create the new reference
730     if (storeFile.hasReference()) {
731       Reference reference = Reference.convert(storeFile.getReference());
732       reference.write(fs, outPath);
733     } else {
734       InputStream in;
735       if (linkPath != null) {
736         in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
737       } else {
738         linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
739                         regionInfo.getEncodedName()), familyDir.getName()), hfileName);
740         in = fs.open(linkPath);
741       }
742       OutputStream out = fs.create(outPath);
743       IOUtils.copyBytes(in, out, conf);
744     }
745 
746     // Add the daughter region to the map
747     String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
748     if (regionName == null) {
749       regionName = regionInfo.getEncodedName();
750     }
751     LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
752     synchronized (parentsMap) {
753       Pair<String, String> daughters = parentsMap.get(clonedRegionName);
754       if (daughters == null) {
755         // In case one side of the split is already compacted, regionName is put as both first and
756         // second of Pair
757         daughters = new Pair<String, String>(regionName, regionName);
758         parentsMap.put(clonedRegionName, daughters);
759       } else if (!regionName.equals(daughters.getFirst())) {
760         daughters.setSecond(regionName);
761       }
762     }
763   }
764 
765   /**
766    * Create a new {@link HRegionInfo} from the snapshot region info.
767    * Keep the same startKey, endKey, regionId and split information but change
768    * the table name.
769    *
770    * @param snapshotRegionInfo Info for region to clone.
771    * @return the new HRegion instance
772    */
773   public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
774     HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
775                       snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
776                       snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
777     regionInfo.setOffline(snapshotRegionInfo.isOffline());
778     return regionInfo;
779   }
780 
781   /**
782    * @return the set of the regions contained in the table
783    */
784   private List<HRegionInfo> getTableRegions() throws IOException {
785     LOG.debug("get table regions: " + tableDir);
786     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
787     if (regionDirs == null) return null;
788 
789     List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
790     for (FileStatus regionDir: regionDirs) {
791       HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
792       regions.add(hri);
793     }
794     LOG.debug("found " + regions.size() + " regions for table=" +
795         tableDesc.getTableName().getNameAsString());
796     return regions;
797   }
798 
799   /**
800    * Create a new table descriptor cloning the snapshot table schema.
801    *
802    * @param snapshotTableDescriptor
803    * @param tableName
804    * @return cloned table descriptor
805    * @throws IOException
806    */
807   public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
808       final TableName tableName) throws IOException {
809     HTableDescriptor htd = new HTableDescriptor(tableName);
810     for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
811       htd.addFamily(hcd);
812     }
813     for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
814         snapshotTableDescriptor.getValues().entrySet()) {
815       htd.setValue(e.getKey(), e.getValue());
816     }
817     for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
818       htd.setConfiguration(e.getKey(), e.getValue());
819     }
820     return htd;
821   }
822 
823   /**
824    * Copy the snapshot files for a snapshot scanner, discards meta changes.
825    * @param conf
826    * @param fs
827    * @param rootDir
828    * @param restoreDir
829    * @param snapshotName
830    * @throws IOException
831    */
832   public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir,
833       Path restoreDir, String snapshotName) throws IOException {
834     // ensure that restore dir is not under root dir
835     if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
836       throw new IllegalArgumentException("Filesystems for restore directory (" +
837           restoreDir.getFileSystem(conf).getUri() + ") and HBase root directory (" +
838           rootDir.getFileSystem(conf).getUri() + ") should be the same");
839     }
840     if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
841       throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
842           "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
843     }
844 
845     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
846     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
847     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
848 
849     MonitoredTask status = TaskMonitor.get().createStatus(
850         "Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
851     ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
852 
853     // we send createBackRefs=false so that restored hfiles do not create back reference links
854     // in the base hbase root dir.
855     RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
856       manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
857     helper.restoreHdfsRegions(); // TODO: parallelize.
858 
859     if (LOG.isDebugEnabled()) {
860       LOG.debug("Restored table dir:" + restoreDir);
861       FSUtils.logFileSystemState(fs, restoreDir, LOG);
862     }
863   }
864 
865   public static void restoreSnapshotACL(SnapshotDescription snapshot, TableName newTableName,
866       Configuration conf) throws IOException {
867     if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) {
868       LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName);
869       ListMultimap<String, TablePermission> perms =
870           ProtobufUtil.toUserTablePermissions(snapshot.getUsersAndPermissions());
871       try {
872         for (Entry<String, TablePermission> e : perms.entries()) {
873           String user = e.getKey();
874           TablePermission perm = e.getValue();
875           perm.setTableName(newTableName);
876           AccessControlClient.grant(conf, perm.getTableName(), user, perm.getFamily(),
877             perm.getQualifier(), perm.getActions());
878         }
879       } catch (Throwable e) {
880         throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot
881             + ", table: " + newTableName, e);
882       }
883     }
884   }
885 }