1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28 import java.util.concurrent.atomic.AtomicInteger;
29 import java.util.concurrent.atomic.AtomicLong;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.hbase.classification.InterfaceAudience;
38 import org.apache.hadoop.hbase.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configured;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.util.StringUtils;
43 import org.apache.hadoop.util.Tool;
44 import org.apache.hadoop.util.ToolRunner;
45
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.hbase.HBaseConfiguration;
48 import org.apache.hadoop.hbase.io.HFileLink;
49 import org.apache.hadoop.hbase.io.WALLink;
50 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
51 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
52 import org.apache.hadoop.hbase.util.FSUtils;
53
54
55
56
57
58
59
60
61
62
63 @InterfaceAudience.Public
64 @InterfaceStability.Evolving
65 public final class SnapshotInfo extends Configured implements Tool {
66 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
67
68
69
70
71
72
73
74
75
76 public static class SnapshotStats {
77
78 static class FileInfo {
79 private final boolean corrupted;
80 private final boolean inArchive;
81 private final long size;
82
83 FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
84 this.corrupted = corrupted;
85 this.inArchive = inArchive;
86 this.size = size;
87 }
88
89
90 public boolean inArchive() {
91 return this.inArchive;
92 }
93
94
95 public boolean isCorrupted() {
96 return this.corrupted;
97 }
98
99
100 public boolean isMissing() {
101 return this.size < 0;
102 }
103
104
105 public long getSize() {
106 return this.size;
107 }
108
109 String getStateToString() {
110 if (isCorrupted()) return "CORRUPTED";
111 if (isMissing()) return "NOT FOUND";
112 if (inArchive()) return "archive";
113 return null;
114 }
115 }
116
117 private AtomicInteger hfilesArchiveCount = new AtomicInteger();
118 private AtomicInteger hfilesCorrupted = new AtomicInteger();
119 private AtomicInteger hfilesMissing = new AtomicInteger();
120 private AtomicInteger hfilesCount = new AtomicInteger();
121 private AtomicInteger hfilesMobCount = new AtomicInteger();
122 private AtomicInteger logsMissing = new AtomicInteger();
123 private AtomicInteger logsCount = new AtomicInteger();
124 private AtomicLong hfilesArchiveSize = new AtomicLong();
125 private AtomicLong hfilesSize = new AtomicLong();
126 private AtomicLong hfilesMobSize = new AtomicLong();
127 private AtomicLong logSize = new AtomicLong();
128
129 private final SnapshotDescription snapshot;
130 private final TableName snapshotTable;
131 private final Configuration conf;
132 private final FileSystem fs;
133
134 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
135 {
136 this.snapshot = snapshot;
137 this.snapshotTable = TableName.valueOf(snapshot.getTable());
138 this.conf = conf;
139 this.fs = fs;
140 }
141
142
143 public SnapshotDescription getSnapshotDescription() {
144 return this.snapshot;
145 }
146
147
148 public boolean isSnapshotCorrupted() {
149 return hfilesMissing.get() > 0 ||
150 logsMissing.get() > 0 ||
151 hfilesCorrupted.get() > 0;
152 }
153
154
155 public int getStoreFilesCount() {
156 return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get();
157 }
158
159
160 public int getArchivedStoreFilesCount() {
161 return hfilesArchiveCount.get();
162 }
163
164
165 public int getMobStoreFilesCount() { return hfilesMobCount.get(); }
166
167
168 public int getLogsCount() {
169 return logsCount.get();
170 }
171
172
173 public int getMissingStoreFilesCount() {
174 return hfilesMissing.get();
175 }
176
177
178 public int getCorruptedStoreFilesCount() {
179 return hfilesCorrupted.get();
180 }
181
182
183 public int getMissingLogsCount() {
184 return logsMissing.get();
185 }
186
187
188 public long getStoreFilesSize() {
189 return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get();
190 }
191
192
193 public long getSharedStoreFilesSize() {
194 return hfilesSize.get();
195 }
196
197
198 public long getArchivedStoreFileSize() {
199 return hfilesArchiveSize.get();
200 }
201
202
203 public long getMobStoreFilesSize() { return hfilesMobSize.get(); }
204
205
206 public float getSharedStoreFilePercentage() {
207 return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100;
208 }
209
210
211 public float getMobStoreFilePercentage() {
212 return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100;
213 }
214
215
216 public long getLogsSize() {
217 return logSize.get();
218 }
219
220
221
222
223
224
225
226
227 FileInfo addStoreFile(final HRegionInfo region, final String family,
228 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
229 HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
230 family, storeFile.getName());
231 boolean isCorrupted = false;
232 boolean inArchive = false;
233 long size = -1;
234 try {
235 if ((inArchive = fs.exists(link.getArchivePath()))) {
236 size = fs.getFileStatus(link.getArchivePath()).getLen();
237 hfilesArchiveSize.addAndGet(size);
238 hfilesArchiveCount.incrementAndGet();
239 } else if (inArchive = fs.exists(link.getMobPath())) {
240 size = fs.getFileStatus(link.getMobPath()).getLen();
241 hfilesMobSize.addAndGet(size);
242 hfilesMobCount.incrementAndGet();
243 } else {
244 size = link.getFileStatus(fs).getLen();
245 hfilesSize.addAndGet(size);
246 hfilesCount.incrementAndGet();
247 }
248 isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
249 if (isCorrupted) hfilesCorrupted.incrementAndGet();
250 } catch (FileNotFoundException e) {
251 hfilesMissing.incrementAndGet();
252 }
253 return new FileInfo(inArchive, size, isCorrupted);
254 }
255
256
257
258
259
260
261
262 FileInfo addLogFile(final String server, final String logfile) throws IOException {
263 WALLink logLink = new WALLink(conf, server, logfile);
264 long size = -1;
265 try {
266 size = logLink.getFileStatus(fs).getLen();
267 logSize.addAndGet(size);
268 logsCount.incrementAndGet();
269 } catch (FileNotFoundException e) {
270 logsMissing.incrementAndGet();
271 }
272 return new FileInfo(false, size, false);
273 }
274 }
275
276 private boolean printSizeInBytes = false;
277 private FileSystem fs;
278 private Path rootDir;
279
280 private SnapshotManifest snapshotManifest;
281
282 @Override
283 public int run(String[] args) throws IOException, InterruptedException {
284 final Configuration conf = getConf();
285 boolean listSnapshots = false;
286 String snapshotName = null;
287 boolean showSchema = false;
288 boolean showFiles = false;
289 boolean showStats = false;
290
291
292 for (int i = 0; i < args.length; i++) {
293 String cmd = args[i];
294 try {
295 if (cmd.equals("-snapshot")) {
296 snapshotName = args[++i];
297 } else if (cmd.equals("-files")) {
298 showFiles = true;
299 showStats = true;
300 } else if (cmd.equals("-stats")) {
301 showStats = true;
302 } else if (cmd.equals("-schema")) {
303 showSchema = true;
304 } else if (cmd.equals("-remote-dir")) {
305 Path sourceDir = new Path(args[++i]);
306 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
307 FSUtils.setFsDefault(conf, new Path(defaultFs));
308 FSUtils.setRootDir(conf, sourceDir);
309 } else if (cmd.equals("-list-snapshots")) {
310 listSnapshots = true;
311 } else if (cmd.equals("-size-in-bytes")) {
312 printSizeInBytes = true;
313 } else if (cmd.equals("-h") || cmd.equals("--help")) {
314 printUsageAndExit();
315 } else {
316 System.err.println("UNEXPECTED: " + cmd);
317 printUsageAndExit();
318 }
319 } catch (Exception e) {
320 printUsageAndExit();
321 }
322 }
323
324
325 if (listSnapshots) {
326 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
327 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
328 for (SnapshotDescription desc: getSnapshotList(conf)) {
329 System.out.printf("%-20s | %20s | %s%n",
330 desc.getName(),
331 df.format(new Date(desc.getCreationTime())),
332 desc.getTable());
333 }
334 return 0;
335 }
336
337 if (snapshotName == null) {
338 System.err.println("Missing snapshot name!");
339 printUsageAndExit();
340 return 1;
341 }
342
343 rootDir = FSUtils.getRootDir(conf);
344 fs = FileSystem.get(rootDir.toUri(), conf);
345 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
346
347
348 if (!loadSnapshotInfo(snapshotName)) {
349 System.err.println("Snapshot '" + snapshotName + "' not found!");
350 return 1;
351 }
352
353 printInfo();
354 if (showSchema) printSchema();
355 printFiles(showFiles, showStats);
356
357 return 0;
358 }
359
360
361
362
363
364
365 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
366 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
367 if (!fs.exists(snapshotDir)) {
368 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
369 return false;
370 }
371
372 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
373 snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
374 return true;
375 }
376
377
378
379
380 private void printInfo() {
381 SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
382 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
383 System.out.println("Snapshot Info");
384 System.out.println("----------------------------------------");
385 System.out.println(" Name: " + snapshotDesc.getName());
386 System.out.println(" Type: " + snapshotDesc.getType());
387 System.out.println(" Table: " + snapshotDesc.getTable());
388 System.out.println(" Format: " + snapshotDesc.getVersion());
389 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
390 System.out.println(" Owner: " + snapshotDesc.getOwner());
391 System.out.println();
392 }
393
394
395
396
397 private void printSchema() {
398 System.out.println("Table Descriptor");
399 System.out.println("----------------------------------------");
400 System.out.println(snapshotManifest.getTableDescriptor().toString());
401 System.out.println();
402 }
403
404
405
406
407
408 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
409 if (showFiles) {
410 System.out.println("Snapshot Files");
411 System.out.println("----------------------------------------");
412 }
413
414
415 final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
416 final String table = snapshotDesc.getTable();
417 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
418 SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
419 new SnapshotReferenceUtil.SnapshotVisitor() {
420 @Override
421 public void storeFile(final HRegionInfo regionInfo, final String family,
422 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
423 if (storeFile.hasReference()) return;
424
425 SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
426 if (showFiles) {
427 String state = info.getStateToString();
428 System.out.printf("%8s %s/%s/%s/%s %s%n",
429 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
430 table, regionInfo.getEncodedName(), family, storeFile.getName(),
431 state == null ? "" : "(" + state + ")");
432 }
433 }
434
435 @Override
436 public void logFile (final String server, final String logfile)
437 throws IOException {
438 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
439
440 if (showFiles) {
441 String state = info.getStateToString();
442 System.out.printf("%8s log %s on server %s (%s)%n",
443 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
444 logfile, server,
445 state == null ? "" : "(" + state + ")");
446 }
447 }
448 });
449
450
451 System.out.println();
452 if (stats.isSnapshotCorrupted()) {
453 System.out.println("**************************************************************");
454 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
455 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
456 System.out.printf(" %d hfile(s) corrupted.%n",
457 stats.getCorruptedStoreFilesCount());
458 System.out.println("**************************************************************");
459 }
460
461 if (showStats) {
462 System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " +
463 "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n",
464 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), stats.getMobStoreFilesCount(),
465 fileSizeToString(stats.getStoreFilesSize()),
466 stats.getSharedStoreFilePercentage(),
467 fileSizeToString(stats.getSharedStoreFilesSize()),
468 stats.getMobStoreFilePercentage(),
469 fileSizeToString(stats.getMobStoreFilesSize())
470 );
471 System.out.printf("%d Logs, total size %s%n",
472 stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
473 System.out.println();
474 }
475 }
476
477 private String fileSizeToString(long size) {
478 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
479 }
480
481 private void printUsageAndExit() {
482 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
483 System.err.println(" where [options] are:");
484 System.err.println(" -h|-help Show this help and exit.");
485 System.err.println(" -remote-dir Root directory that contains the snapshots.");
486 System.err.println(" -list-snapshots List all the available snapshots and exit.");
487 System.err.println(" -size-in-bytes Print the size of the files in bytes.");
488 System.err.println(" -snapshot NAME Snapshot to examine.");
489 System.err.println(" -files Files and logs list.");
490 System.err.println(" -stats Files and logs stats.");
491 System.err.println(" -schema Describe the snapshotted table.");
492 System.err.println();
493 System.err.println("Examples:");
494 System.err.println(" hbase " + getClass() + " \\");
495 System.err.println(" -snapshot MySnapshot -files");
496 System.exit(1);
497 }
498
499
500
501
502
503
504
505 public static SnapshotStats getSnapshotStats(final Configuration conf,
506 final SnapshotDescription snapshot) throws IOException {
507 Path rootDir = FSUtils.getRootDir(conf);
508 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
509 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
510 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
511 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
512 SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
513 new SnapshotReferenceUtil.SnapshotVisitor() {
514 @Override
515 public void storeFile(final HRegionInfo regionInfo, final String family,
516 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
517 if (!storeFile.hasReference()) {
518 stats.addStoreFile(regionInfo, family, storeFile);
519 }
520 }
521
522 @Override
523 public void logFile (final String server, final String logfile) throws IOException {
524 stats.addLogFile(server, logfile);
525 }
526 });
527 return stats;
528 }
529
530
531
532
533
534
535 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
536 throws IOException {
537 Path rootDir = FSUtils.getRootDir(conf);
538 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
539 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
540 FileStatus[] snapshots = fs.listStatus(snapshotDir,
541 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
542 List<SnapshotDescription> snapshotLists =
543 new ArrayList<SnapshotDescription>(snapshots.length);
544 for (FileStatus snapshotDirStat: snapshots) {
545 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
546 }
547 return snapshotLists;
548 }
549
550
551
552
553
554
555
556
557 static int innerMain(final String [] args) throws Exception {
558 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
559 }
560
561 public static void main(String[] args) throws Exception {
562 System.exit(innerMain(args));
563 }
564 }