1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mob;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.security.Key;
24 import java.security.KeyException;
25 import java.text.ParseException;
26 import java.text.SimpleDateFormat;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.UUID;
32 import java.util.concurrent.ExecutorService;
33 import java.util.concurrent.RejectedExecutionException;
34 import java.util.concurrent.RejectedExecutionHandler;
35 import java.util.concurrent.SynchronousQueue;
36 import java.util.concurrent.ThreadPoolExecutor;
37 import java.util.concurrent.TimeUnit;
38
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.hbase.classification.InterfaceAudience;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FileStatus;
44 import org.apache.hadoop.fs.FileSystem;
45 import org.apache.hadoop.fs.Path;
46 import org.apache.hadoop.hbase.Cell;
47 import org.apache.hadoop.hbase.CellComparator;
48 import org.apache.hadoop.hbase.HBaseConfiguration;
49 import org.apache.hadoop.hbase.HColumnDescriptor;
50 import org.apache.hadoop.hbase.HConstants;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.HTableDescriptor;
53 import org.apache.hadoop.hbase.KeyValue;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.Tag;
56 import org.apache.hadoop.hbase.TagType;
57 import org.apache.hadoop.hbase.backup.HFileArchiver;
58 import org.apache.hadoop.hbase.client.Scan;
59 import org.apache.hadoop.hbase.io.HFileLink;
60 import org.apache.hadoop.hbase.io.compress.Compression;
61 import org.apache.hadoop.hbase.io.crypto.Cipher;
62 import org.apache.hadoop.hbase.io.crypto.Encryption;
63 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
64 import org.apache.hadoop.hbase.io.hfile.HFileContext;
65 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
66 import org.apache.hadoop.hbase.master.TableLockManager;
67 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
68 import org.apache.hadoop.hbase.mob.compactions.MobCompactor;
69 import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
70 import org.apache.hadoop.hbase.regionserver.BloomType;
71 import org.apache.hadoop.hbase.regionserver.HStore;
72 import org.apache.hadoop.hbase.regionserver.StoreFile;
73 import org.apache.hadoop.hbase.security.EncryptionUtil;
74 import org.apache.hadoop.hbase.security.User;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.FSUtils;
78 import org.apache.hadoop.hbase.util.ReflectionUtils;
79 import org.apache.hadoop.hbase.util.Threads;
80
81
82
83
84 @InterfaceAudience.Private
85 public class MobUtils {
86
87 private static final Log LOG = LogFactory.getLog(MobUtils.class);
88
89 private static final ThreadLocal<SimpleDateFormat> LOCAL_FORMAT =
90 new ThreadLocal<SimpleDateFormat>() {
91 @Override
92 protected SimpleDateFormat initialValue() {
93 return new SimpleDateFormat("yyyyMMdd");
94 }
95 };
96
97
98
99
100
101
102 public static String formatDate(Date date) {
103 return LOCAL_FORMAT.get().format(date);
104 }
105
106
107
108
109
110
111
112 public static Date parseDate(String dateString) throws ParseException {
113 return LOCAL_FORMAT.get().parse(dateString);
114 }
115
116
117
118
119
120
121 public static boolean isMobReferenceCell(Cell cell) {
122 if (cell.getTagsLength() > 0) {
123 Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
124 TagType.MOB_REFERENCE_TAG_TYPE);
125 return tag != null;
126 }
127 return false;
128 }
129
130
131
132
133
134
135 public static Tag getTableNameTag(Cell cell) {
136 if (cell.getTagsLength() > 0) {
137 Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
138 TagType.MOB_TABLE_NAME_TAG_TYPE);
139 return tag;
140 }
141 return null;
142 }
143
144
145
146
147
148
149 public static boolean hasMobReferenceTag(List<Tag> tags) {
150 if (!tags.isEmpty()) {
151 for (Tag tag : tags) {
152 if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) {
153 return true;
154 }
155 }
156 }
157 return false;
158 }
159
160
161
162
163
164
165
166
167
168
169 public static boolean isRawMobScan(Scan scan) {
170 byte[] raw = scan.getAttribute(MobConstants.MOB_SCAN_RAW);
171 try {
172 return raw != null && Bytes.toBoolean(raw);
173 } catch (IllegalArgumentException e) {
174 return false;
175 }
176 }
177
178
179
180
181
182
183
184
185 public static boolean isRefOnlyScan(Scan scan) {
186 byte[] refOnly = scan.getAttribute(MobConstants.MOB_SCAN_REF_ONLY);
187 try {
188 return refOnly != null && Bytes.toBoolean(refOnly);
189 } catch (IllegalArgumentException e) {
190 return false;
191 }
192 }
193
194
195
196
197
198
199
200 public static boolean isCacheMobBlocks(Scan scan) {
201 byte[] cache = scan.getAttribute(MobConstants.MOB_CACHE_BLOCKS);
202 try {
203 return cache != null && Bytes.toBoolean(cache);
204 } catch (IllegalArgumentException e) {
205 return false;
206 }
207 }
208
209
210
211
212
213
214
215
216
217
218
219 public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) {
220 scan.setAttribute(MobConstants.MOB_CACHE_BLOCKS, Bytes.toBytes(cacheBlocks));
221 }
222
223
224
225
226
227
228
229
230
231
232
233
234
235 public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
236 HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
237 throws IOException {
238 long timeToLive = columnDescriptor.getTimeToLive();
239 if (Integer.MAX_VALUE == timeToLive) {
240
241 return;
242 }
243
244 Date expireDate = new Date(current - timeToLive * 1000);
245 expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate());
246 LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
247
248 FileStatus[] stats = null;
249 Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
250 Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
251 try {
252 stats = fs.listStatus(path);
253 } catch (FileNotFoundException e) {
254 LOG.warn("Failed to find the mob file " + path, e);
255 }
256 if (null == stats) {
257
258 return;
259 }
260 List<StoreFile> filesToClean = new ArrayList<StoreFile>();
261 int deletedFileCount = 0;
262 for (FileStatus file : stats) {
263 String fileName = file.getPath().getName();
264 try {
265 MobFileName mobFileName = null;
266 if (!HFileLink.isHFileLink(file.getPath())) {
267 mobFileName = MobFileName.create(fileName);
268 } else {
269 HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
270 mobFileName = MobFileName.create(hfileLink.getOriginPath().getName());
271 }
272 Date fileDate = parseDate(mobFileName.getDate());
273 if (LOG.isDebugEnabled()) {
274 LOG.debug("Checking file " + fileName);
275 }
276 if (fileDate.getTime() < expireDate.getTime()) {
277 if (LOG.isDebugEnabled()) {
278 LOG.debug(fileName + " is an expired file");
279 }
280 filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
281 }
282 } catch (Exception e) {
283 LOG.error("Cannot parse the fileName " + fileName, e);
284 }
285 }
286 if (!filesToClean.isEmpty()) {
287 try {
288 removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(),
289 filesToClean);
290 deletedFileCount = filesToClean.size();
291 } catch (IOException e) {
292 LOG.error("Failed to delete the mob files " + filesToClean, e);
293 }
294 }
295 LOG.info(deletedFileCount + " expired mob files are deleted");
296 }
297
298
299
300
301
302
303
304 public static Path getMobHome(Configuration conf) {
305 Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
306 return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
307 }
308
309
310
311
312
313
314
315 public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
316 Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
317 Path mobRootDir = new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
318 FileSystem fs = mobRootDir.getFileSystem(conf);
319 return mobRootDir.makeQualified(fs);
320 }
321
322
323
324
325
326
327
328
329 public static Path getMobRegionPath(Configuration conf, TableName tableName) {
330 Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
331 HRegionInfo regionInfo = getMobRegionInfo(tableName);
332 return new Path(tablePath, regionInfo.getEncodedName());
333 }
334
335
336
337
338
339
340
341
342
343 public static Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
344 return new Path(getMobRegionPath(conf, tableName), familyName);
345 }
346
347
348
349
350
351
352
353
354 public static Path getMobFamilyPath(Path regionPath, String familyName) {
355 return new Path(regionPath, familyName);
356 }
357
358
359
360
361
362
363
364
365 public static HRegionInfo getMobRegionInfo(TableName tableName) {
366 HRegionInfo info = new HRegionInfo(tableName, MobConstants.MOB_REGION_NAME_BYTES,
367 HConstants.EMPTY_END_ROW, false, 0);
368 return info;
369 }
370
371
372
373
374
375
376 public static boolean isMobRegionInfo(HRegionInfo regionInfo) {
377 return regionInfo == null ? false : getMobRegionInfo(regionInfo.getTable()).getEncodedName()
378 .equals(regionInfo.getEncodedName());
379 }
380
381
382
383
384
385
386
387 public static boolean isMobRegionName(TableName tableName, byte[] regionName) {
388 return Bytes.equals(regionName, getMobRegionInfo(tableName).getRegionName());
389 }
390
391
392
393
394
395
396
397 public static Path getCompactionWorkingPath(Path root, String jobName) {
398 return new Path(root, jobName);
399 }
400
401
402
403
404
405
406
407
408
409
410
411 public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
412 Path tableDir, byte[] family, Collection<StoreFile> storeFiles) throws IOException {
413 HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family,
414 storeFiles);
415 }
416
417
418
419
420
421
422
423
424
425
426 public static Cell createMobRefCell(Cell cell, byte[] fileName, Tag tableNameTag) {
427
428
429 List<Tag> tags = new ArrayList<Tag>();
430
431 tags.add(MobConstants.MOB_REF_TAG);
432
433
434
435
436
437 tags.add(tableNameTag);
438 return createMobRefCell(cell, fileName, tags);
439 }
440
441 public static Cell createMobRefCell(Cell cell, byte[] fileName, List<Tag> refCellTags) {
442 byte[] refValue = Bytes.add(Bytes.toBytes(cell.getValueLength()), fileName);
443 List<Tag> tags = Tag.carryForwardTags(refCellTags, cell);
444 KeyValue reference = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
445 cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
446 cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
447 cell.getTimestamp(), KeyValue.Type.Put, refValue, 0, refValue.length, tags);
448 reference.setSequenceId(cell.getSequenceId());
449 return reference;
450 }
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467 public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
468 HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
469 Compression.Algorithm compression, String startKey, CacheConfig cacheConfig,
470 Encryption.Context cryptoContext)
471 throws IOException {
472 MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
473 .replaceAll("-", ""));
474 return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
475 cacheConfig, cryptoContext);
476 }
477
478
479
480
481
482
483
484
485
486
487
488
489
490 public static StoreFile.Writer createRefFileWriter(Configuration conf, FileSystem fs,
491 HColumnDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig,
492 Encryption.Context cryptoContext)
493 throws IOException {
494 HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(true)
495 .withIncludesTags(true).withCompression(family.getCompactionCompression())
496 .withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf))
497 .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
498 .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding())
499 .withEncryptionContext(cryptoContext).withCreateTime(EnvironmentEdgeManager.currentTime())
500 .build();
501 Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", ""));
502 StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath)
503 .withComparator(KeyValue.COMPARATOR).withBloomType(family.getBloomFilterType())
504 .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
505 return w;
506 }
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523 public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
524 HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
525 Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig,
526 Encryption.Context cryptoContext)
527 throws IOException {
528 MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
529 .replaceAll("-", ""));
530 return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
531 cacheConfig, cryptoContext);
532 }
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549 public static StoreFile.Writer createDelFileWriter(Configuration conf, FileSystem fs,
550 HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
551 Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig,
552 Encryption.Context cryptoContext)
553 throws IOException {
554 String suffix = UUID
555 .randomUUID().toString().replaceAll("-", "") + "_del";
556 MobFileName mobFileName = MobFileName.create(startKey, date, suffix);
557 return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
558 cacheConfig, cryptoContext);
559 }
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575 private static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
576 HColumnDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
577 Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext)
578 throws IOException {
579 HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
580 .withIncludesMvcc(true).withIncludesTags(true)
581 .withCompressTags(family.isCompressTags())
582 .withChecksumType(HStore.getChecksumType(conf))
583 .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
584 .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding())
585 .withEncryptionContext(cryptoContext)
586 .withCreateTime(EnvironmentEdgeManager.currentTime()).build();
587
588 StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs)
589 .withFilePath(new Path(basePath, mobFileName.getFileName()))
590 .withComparator(KeyValue.COMPARATOR).withBloomType(BloomType.NONE)
591 .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
592 return w;
593 }
594
595
596
597
598
599
600
601
602
603
604
605 public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
606 Path targetPath, CacheConfig cacheConfig) throws IOException {
607 if (sourceFile == null) {
608 return null;
609 }
610 Path dstPath = new Path(targetPath, sourceFile.getName());
611 validateMobFile(conf, fs, sourceFile, cacheConfig);
612 String msg = "Renaming flushed file from " + sourceFile + " to " + dstPath;
613 LOG.info(msg);
614 Path parent = dstPath.getParent();
615 if (!fs.exists(parent)) {
616 fs.mkdirs(parent);
617 }
618 if (!fs.rename(sourceFile, dstPath)) {
619 throw new IOException("Failed rename of " + sourceFile + " to " + dstPath);
620 }
621 return dstPath;
622 }
623
624
625
626
627
628
629
630
631 private static void validateMobFile(Configuration conf, FileSystem fs, Path path,
632 CacheConfig cacheConfig) throws IOException {
633 StoreFile storeFile = null;
634 try {
635 storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE);
636 storeFile.createReader();
637 } catch (IOException e) {
638 LOG.error("Failed to open mob file[" + path + "], keep it in temp directory.", e);
639 throw e;
640 } finally {
641 if (storeFile != null) {
642 storeFile.closeReader(false);
643 }
644 }
645 }
646
647
648
649
650
651
652
653
654
655
656 public static boolean hasValidMobRefCellValue(Cell cell) {
657 return cell.getValueLength() > Bytes.SIZEOF_INT;
658 }
659
660
661
662
663
664
665
666
667
668
669 public static int getMobValueLength(Cell cell) {
670 return Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), Bytes.SIZEOF_INT);
671 }
672
673
674
675
676
677
678
679
680
681
682 public static String getMobFileName(Cell cell) {
683 return Bytes.toString(cell.getValueArray(), cell.getValueOffset() + Bytes.SIZEOF_INT,
684 cell.getValueLength() - Bytes.SIZEOF_INT);
685 }
686
687
688
689
690
691
692
693 public static TableName getTableLockName(TableName tn) {
694 byte[] tableName = tn.getName();
695 return TableName.valueOf(Bytes.add(tableName, MobConstants.MOB_TABLE_LOCK_SUFFIX));
696 }
697
698
699
700
701
702
703
704
705
706
707
708 public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
709 HColumnDescriptor hcd, ExecutorService pool, TableLockManager tableLockManager,
710 boolean allFiles) throws IOException {
711 String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY,
712 PartitionedMobCompactor.class.getName());
713
714 MobCompactor compactor = null;
715 try {
716 compactor = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
717 Configuration.class, FileSystem.class, TableName.class, HColumnDescriptor.class,
718 ExecutorService.class }, new Object[] { conf, fs, tableName, hcd, pool });
719 } catch (Exception e) {
720 throw new IOException("Unable to load configured mob file compactor '" + className + "'", e);
721 }
722
723
724
725 boolean tableLocked = false;
726 TableLock lock = null;
727 try {
728
729 if (tableLockManager != null) {
730 lock = tableLockManager.writeLock(MobUtils.getTableLockName(tableName),
731 "Run MobCompactor");
732 lock.acquire();
733 }
734 tableLocked = true;
735 compactor.compact(allFiles);
736 } catch (Exception e) {
737 LOG.error("Failed to compact the mob files for the column " + hcd.getNameAsString()
738 + " in the table " + tableName.getNameAsString(), e);
739 } finally {
740 if (lock != null && tableLocked) {
741 try {
742 lock.release();
743 } catch (IOException e) {
744 LOG.error(
745 "Failed to release the write lock for the table " + tableName.getNameAsString(), e);
746 }
747 }
748 }
749 }
750
751
752
753
754
755
756 public static ExecutorService createMobCompactorThreadPool(Configuration conf) {
757 int maxThreads = conf.getInt(MobConstants.MOB_COMPACTION_THREADS_MAX,
758 MobConstants.DEFAULT_MOB_COMPACTION_THREADS_MAX);
759 if (maxThreads == 0) {
760 maxThreads = 1;
761 }
762 final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
763 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue,
764 Threads.newDaemonThreadFactory("MobCompactor"), new RejectedExecutionHandler() {
765 @Override
766 public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
767 try {
768
769 queue.put(r);
770 } catch (InterruptedException e) {
771 throw new RejectedExecutionException(e);
772 }
773 }
774 });
775 ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
776 return pool;
777 }
778
779
780
781
782
783
784
785
786 public static Encryption.Context createEncryptionContext(Configuration conf,
787 HColumnDescriptor family) throws IOException {
788
789 Encryption.Context cryptoContext = Encryption.Context.NONE;
790 String cipherName = family.getEncryptionType();
791 if (cipherName != null) {
792 Cipher cipher;
793 Key key;
794 byte[] keyBytes = family.getEncryptionKey();
795 if (keyBytes != null) {
796
797 String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User
798 .getCurrent().getShortName());
799 try {
800
801 key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
802 } catch (KeyException e) {
803
804
805 if (LOG.isDebugEnabled()) {
806 LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
807 }
808 String alternateKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
809 if (alternateKeyName != null) {
810 try {
811 key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
812 } catch (KeyException ex) {
813 throw new IOException(ex);
814 }
815 } else {
816 throw new IOException(e);
817 }
818 }
819
820 cipher = Encryption.getCipher(conf, key.getAlgorithm());
821 if (cipher == null) {
822 throw new RuntimeException("Cipher '" + key.getAlgorithm() + "' is not available");
823 }
824
825
826
827 if (!cipher.getName().equalsIgnoreCase(cipherName)) {
828 throw new RuntimeException("Encryption for family '" + family.getNameAsString()
829 + "' configured with type '" + cipherName + "' but key specifies algorithm '"
830 + cipher.getName() + "'");
831 }
832 } else {
833
834 cipher = Encryption.getCipher(conf, cipherName);
835 if (cipher == null) {
836 throw new RuntimeException("Cipher '" + cipherName + "' is not available");
837 }
838 key = cipher.getRandomKey();
839 }
840 cryptoContext = Encryption.newContext(conf);
841 cryptoContext.setCipher(cipher);
842 cryptoContext.setKey(key);
843 }
844 return cryptoContext;
845 }
846
847
848
849
850
851
852 public static boolean hasMobColumns(HTableDescriptor htd) {
853 HColumnDescriptor[] hcds = htd.getColumnFamilies();
854 for (HColumnDescriptor hcd : hcds) {
855 if (hcd.isMobEnabled()) {
856 return true;
857 }
858 }
859 return false;
860 }
861
862
863
864
865
866
867
868 public static boolean isReadEmptyValueOnMobCellMiss(Scan scan) {
869 byte[] readEmptyValueOnMobCellMiss = scan.getAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS);
870 try {
871 return readEmptyValueOnMobCellMiss != null && Bytes.toBoolean(readEmptyValueOnMobCellMiss);
872 } catch (IllegalArgumentException e) {
873 return false;
874 }
875 }
876
877
878
879
880
881
882
883
884
885
886 public static void archiveMobStoreFiles(Configuration conf, FileSystem fs,
887 HRegionInfo mobRegionInfo, Path mobFamilyDir, byte[] family) throws IOException {
888
889 Configuration copyOfConf = HBaseConfiguration.create(conf);
890 copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
891 CacheConfig cacheConfig = new CacheConfig(copyOfConf);
892
893 FileStatus[] fileStatus = FSUtils.listStatus(fs, mobFamilyDir);
894 List<StoreFile> storeFileList = new ArrayList<StoreFile>();
895 for (FileStatus file : fileStatus) {
896 storeFileList.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
897 }
898 HFileArchiver.archiveStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, family, storeFileList);
899 }
900
901
902
903
904
905 public static Cell createMobRefDeleteMarker(Cell cell) {
906 List<Tag> refTag = new ArrayList<Tag>();
907 refTag.add(MobConstants.MOB_REF_TAG);
908 List<Tag> tags = Tag.carryForwardTags(refTag, cell);
909 KeyValue reference = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
910 cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
911 cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
912 cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()), cell.getValueArray(),
913 cell.getValueOffset(), cell.getValueLength(), tags);
914 reference.setSequenceId(cell.getSequenceId());
915 return reference;
916 }
917
918
919
920
921
922
923
924
925 public static boolean isMobFileExpired(HColumnDescriptor column, long current, String fileDate) {
926 if (column.getMinVersions() > 0) {
927 return false;
928 }
929 long timeToLive = column.getTimeToLive();
930 if (Integer.MAX_VALUE == timeToLive) {
931 return false;
932 }
933
934 Date expireDate = new Date(current - timeToLive * 1000);
935 expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate());
936 try {
937 Date date = parseDate(fileDate);
938 if (date.getTime() < expireDate.getTime()) {
939 return true;
940 }
941 } catch (ParseException e) {
942 LOG.warn("Failed to parse the date " + fileDate, e);
943 return false;
944 }
945 return false;
946 }
947 }