1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import com.google.protobuf.InvalidProtocolBufferException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
44 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
45 import org.apache.hadoop.hbase.util.ByteStringer;
46 import org.apache.hadoop.hbase.util.FSUtils;
47
48
49
50
51
52
53
54
55
56 @InterfaceAudience.Private
57 public class SnapshotManifestV2 {
58 private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
59
60 public static final int DESCRIPTOR_VERSION = 2;
61
62 public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
63
64 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
65 SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
66 private final Configuration conf;
67 private final Path snapshotDir;
68 private final FileSystem fs;
69
70 public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
71 this.snapshotDir = snapshotDir;
72 this.conf = conf;
73 this.fs = fs;
74 }
75
76 public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
77 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
78 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
79 return manifest;
80 }
81
82 public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
83 SnapshotRegionManifest manifest = region.build();
84 FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
85 try {
86 manifest.writeTo(stream);
87 } finally {
88 stream.close();
89 }
90 }
91
92 public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
93 final SnapshotRegionManifest.Builder region, final byte[] familyName) {
94 SnapshotRegionManifest.FamilyFiles.Builder family =
95 SnapshotRegionManifest.FamilyFiles.newBuilder();
96 family.setFamilyName(ByteStringer.wrap(familyName));
97 return family;
98 }
99
100 public void familyClose(final SnapshotRegionManifest.Builder region,
101 final SnapshotRegionManifest.FamilyFiles.Builder family) {
102 region.addFamilyFiles(family.build());
103 }
104
105 public void storeFile(final SnapshotRegionManifest.Builder region,
106 final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
107 throws IOException {
108 SnapshotRegionManifest.StoreFile.Builder sfManifest =
109 SnapshotRegionManifest.StoreFile.newBuilder();
110 sfManifest.setName(storeFile.getPath().getName());
111 if (storeFile.isReference()) {
112 sfManifest.setReference(storeFile.getReference().convert());
113 }
114 sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
115 family.addStoreFiles(sfManifest.build());
116 }
117 }
118
119 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
120 final Executor executor,final FileSystem fs, final Path snapshotDir,
121 final SnapshotDescription desc) throws IOException {
122 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
123 @Override
124 public boolean accept(Path path) {
125 return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
126 }
127 });
128
129 if (manifestFiles == null || manifestFiles.length == 0) return null;
130
131 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
132 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
133 for (final FileStatus st: manifestFiles) {
134 completionService.submit(new Callable<SnapshotRegionManifest>() {
135 @Override
136 public SnapshotRegionManifest call() throws IOException {
137 FSDataInputStream stream = fs.open(st.getPath());
138 try {
139 return SnapshotRegionManifest.parseFrom(stream);
140 } finally {
141 stream.close();
142 }
143 }
144 });
145 }
146
147 ArrayList<SnapshotRegionManifest> regionsManifest =
148 new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
149 try {
150 for (int i = 0; i < manifestFiles.length; ++i) {
151 regionsManifest.add(completionService.take().get());
152 }
153 } catch (InterruptedException e) {
154 throw new InterruptedIOException(e.getMessage());
155 } catch (ExecutionException e) {
156 Throwable t = e.getCause();
157
158 if(t instanceof InvalidProtocolBufferException) {
159 throw (InvalidProtocolBufferException)t;
160 } else {
161 IOException ex = new IOException("ExecutionException");
162 ex.initCause(e.getCause());
163 throw ex;
164 }
165 }
166 return regionsManifest;
167 }
168
169 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
170 final SnapshotRegionManifest manifest) throws IOException {
171 fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
172 }
173
174 private static Path getRegionManifestPath(final Path snapshotDir,
175 final SnapshotRegionManifest manifest) {
176 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
177 return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
178 }
179 }