1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.regex.Matcher;
25 import java.util.regex.Pattern;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.HFileLink;
37 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
38 import org.apache.hadoop.hbase.io.Reference;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53
54 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:(?:_SeqId_[0-9]+_)|(?:_del))?";
55
56
57 private static final Pattern HFILE_NAME_PATTERN =
58 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
59
60
61
62
63
64 public static final String DELFILE_NAME_REGEX = "[0-9a-f]+(?:_del)";
65
66
67 private static final Pattern DELFILE_NAME_PATTERN =
68 Pattern.compile("^(" + DELFILE_NAME_REGEX + ")");
69
70
71
72
73
74
75
76
77 private static final Pattern REF_NAME_PATTERN =
78 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
79 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
80
81
82 private Configuration conf;
83
84
85 private final FileSystem fs;
86
87
88 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
89
90
91 private final Reference reference;
92
93
94 private final HFileLink link;
95
96 private final Path initialPath;
97
98 private RegionCoprocessorHost coprocessorHost;
99
100
101 private long createdTimestamp;
102
103
104
105
106
107
108
109 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath)
110 throws IOException {
111 assert fs != null;
112 assert initialPath != null;
113 assert conf != null;
114
115 this.fs = fs;
116 this.conf = conf;
117 this.initialPath = initialPath;
118 Path p = initialPath;
119 if (HFileLink.isHFileLink(p)) {
120
121 this.reference = null;
122 this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
123 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
124 } else if (isReference(p)) {
125 this.reference = Reference.read(fs, p);
126 Path referencePath = getReferredToFile(p);
127 if (HFileLink.isHFileLink(referencePath)) {
128
129 this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
130 } else {
131
132 this.link = null;
133 }
134 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
135 " reference to " + referencePath);
136 } else if (isHFile(p)) {
137
138 this.createdTimestamp = fs.getFileStatus(initialPath).getModificationTime();
139 this.reference = null;
140 this.link = null;
141 } else {
142 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
143 }
144 }
145
146
147
148
149
150
151
152 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
153 throws IOException {
154 this(conf, fs, fileStatus.getPath());
155 }
156
157
158
159
160
161
162
163 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
164 final HFileLink link)
165 throws IOException {
166 this.fs = fs;
167 this.conf = conf;
168
169 this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
170
171 this.reference = null;
172 this.link = link;
173 }
174
175
176
177
178
179
180
181
182
183 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
184 final Reference reference)
185 throws IOException {
186 this.fs = fs;
187 this.conf = conf;
188 this.initialPath = fileStatus.getPath();
189 this.createdTimestamp = fileStatus.getModificationTime();
190 this.reference = reference;
191 this.link = null;
192 }
193
194
195
196
197
198 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
199 this.coprocessorHost = coprocessorHost;
200 }
201
202
203
204
205
206 public Reference getReference() {
207 return this.reference;
208 }
209
210
211 public boolean isReference() {
212 return this.reference != null;
213 }
214
215
216 public boolean isTopReference() {
217 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
218 }
219
220
221 public boolean isLink() {
222 return this.link != null && this.reference == null;
223 }
224
225
226 public HDFSBlocksDistribution getHDFSBlockDistribution() {
227 return this.hdfsBlocksDistribution;
228 }
229
230
231
232
233
234
235
236 public StoreFile.Reader open(final FileSystem fs,
237 final CacheConfig cacheConf) throws IOException {
238 FSDataInputStreamWrapper in;
239 FileStatus status;
240
241 if (this.link != null) {
242
243 in = new FSDataInputStreamWrapper(fs, this.link);
244 status = this.link.getFileStatus(fs);
245 } else if (this.reference != null) {
246
247 Path referencePath = getReferredToFile(this.getPath());
248 in = new FSDataInputStreamWrapper(fs, referencePath);
249 status = fs.getFileStatus(referencePath);
250 } else {
251 in = new FSDataInputStreamWrapper(fs, this.getPath());
252 status = fs.getFileStatus(initialPath);
253 }
254 long length = status.getLen();
255 hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
256
257 StoreFile.Reader reader = null;
258 if (this.coprocessorHost != null) {
259 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
260 cacheConf, reference);
261 }
262 if (reader == null) {
263 if (this.reference != null) {
264 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
265 conf);
266 } else {
267 reader = new StoreFile.Reader(fs, status.getPath(), in, length, cacheConf, conf);
268 }
269 }
270 if (this.coprocessorHost != null) {
271 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
272 cacheConf, reference, reader);
273 }
274 return reader;
275 }
276
277
278
279
280 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
281 throws IOException {
282
283
284
285 if (this.link != null) {
286 FileNotFoundException exToThrow = null;
287 for (int i = 0; i < this.link.getLocations().length; i++) {
288 try {
289 return computeHDFSBlocksDistributionInternal(fs);
290 } catch (FileNotFoundException ex) {
291
292 exToThrow = ex;
293 }
294 }
295 throw exToThrow;
296 } else {
297 return computeHDFSBlocksDistributionInternal(fs);
298 }
299 }
300
301 private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
302 throws IOException {
303 FileStatus status = getReferencedFileStatus(fs);
304 if (this.reference != null) {
305 return computeRefFileHDFSBlockDistribution(fs, reference, status);
306 } else {
307 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
308 }
309 }
310
311
312
313
314
315
316 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
317 FileStatus status;
318 if (this.reference != null) {
319 if (this.link != null) {
320 FileNotFoundException exToThrow = null;
321 for (int i = 0; i < this.link.getLocations().length; i++) {
322
323 try {
324 return link.getFileStatus(fs);
325 } catch (FileNotFoundException ex) {
326
327 exToThrow = ex;
328 }
329 }
330 throw exToThrow;
331 } else {
332
333 Path referencePath = getReferredToFile(this.getPath());
334 status = fs.getFileStatus(referencePath);
335 }
336 } else {
337 if (this.link != null) {
338 FileNotFoundException exToThrow = null;
339 for (int i = 0; i < this.link.getLocations().length; i++) {
340
341 try {
342 return link.getFileStatus(fs);
343 } catch (FileNotFoundException ex) {
344
345 exToThrow = ex;
346 }
347 }
348 throw exToThrow;
349 } else {
350 status = fs.getFileStatus(initialPath);
351 }
352 }
353 return status;
354 }
355
356
357 public Path getPath() {
358 return initialPath;
359 }
360
361
362 public FileStatus getFileStatus() throws IOException {
363 return getReferencedFileStatus(fs);
364 }
365
366
367 public long getModificationTime() throws IOException {
368 return getFileStatus().getModificationTime();
369 }
370
371 @Override
372 public String toString() {
373 return this.getPath() +
374 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
375 }
376
377
378
379
380
381 public static boolean isHFile(final Path path) {
382 return isHFile(path.getName());
383 }
384
385 public static boolean isHFile(final String fileName) {
386 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
387 return m.matches() && m.groupCount() > 0;
388 }
389
390
391
392
393
394 public static boolean isDelFile(final Path path) {
395 return isDelFile(path.getName());
396 }
397
398
399
400
401
402 public static boolean isDelFile(final String fileName) {
403 Matcher m = DELFILE_NAME_PATTERN.matcher(fileName);
404 return m.matches() && m.groupCount() > 0;
405 }
406
407
408
409
410
411 public static boolean isReference(final Path path) {
412 return isReference(path.getName());
413 }
414
415
416
417
418
419 public static boolean isReference(final String name) {
420 Matcher m = REF_NAME_PATTERN.matcher(name);
421 return m.matches() && m.groupCount() > 1;
422 }
423
424
425
426
427 public long getCreatedTimestamp() {
428 return createdTimestamp;
429 }
430
431
432
433
434
435
436
437
438 public static Path getReferredToFile(final Path p) {
439 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
440 if (m == null || !m.matches()) {
441 LOG.warn("Failed match of store file name " + p.toString());
442 throw new IllegalArgumentException("Failed match of store file name " +
443 p.toString());
444 }
445
446
447 String otherRegion = m.group(2);
448
449 Path tableDir = p.getParent().getParent().getParent();
450 String nameStrippedOfSuffix = m.group(1);
451 if (LOG.isDebugEnabled()) {
452 LOG.debug("reference '" + p + "' to region=" + otherRegion
453 + " hfile=" + nameStrippedOfSuffix);
454 }
455
456
457
458 return new Path(new Path(new Path(tableDir, otherRegion),
459 p.getParent().getName()), nameStrippedOfSuffix);
460 }
461
462
463
464
465
466
467 public static boolean validateStoreFileName(final String fileName) {
468 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
469 return(true);
470 return !fileName.contains("-");
471 }
472
473
474
475
476
477
478 public static boolean isValid(final FileStatus fileStatus)
479 throws IOException {
480 final Path p = fileStatus.getPath();
481
482 if (fileStatus.isDirectory())
483 return false;
484
485
486
487
488 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
489 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
490 return false;
491 }
492
493 return validateStoreFileName(p.getName());
494 }
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
510 final FileSystem fs, final Reference reference, final FileStatus status)
511 throws IOException {
512 if (status == null) {
513 return null;
514 }
515
516 long start = 0;
517 long length = 0;
518
519 if (Reference.isTopFileRegion(reference.getFileRegion())) {
520 start = status.getLen()/2;
521 length = status.getLen() - status.getLen()/2;
522 } else {
523 start = 0;
524 length = status.getLen()/2;
525 }
526 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
527 }
528
529 @Override
530 public boolean equals(Object that) {
531 if (this == that) return true;
532 if (that == null) return false;
533
534 if (!(that instanceof StoreFileInfo)) return false;
535
536 StoreFileInfo o = (StoreFileInfo)that;
537 if (initialPath != null && o.initialPath == null) return false;
538 if (initialPath == null && o.initialPath != null) return false;
539 if (initialPath != o.initialPath && initialPath != null
540 && !initialPath.equals(o.initialPath)) return false;
541
542 if (reference != null && o.reference == null) return false;
543 if (reference == null && o.reference != null) return false;
544 if (reference != o.reference && reference != null
545 && !reference.equals(o.reference)) return false;
546
547 if (link != null && o.link == null) return false;
548 if (link == null && o.link != null) return false;
549 if (link != o.link && link != null && !link.equals(o.link)) return false;
550
551 return true;
552 };
553
554
555 @Override
556 public int hashCode() {
557 int hash = 17;
558 hash = hash * 31 + ((reference == null) ? 0 : reference.hashCode());
559 hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
560 hash = hash * 31 + ((link == null) ? 0 : link.hashCode());
561 return hash;
562 }
563 }