1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22
23 import java.io.IOException;
24 import java.lang.management.ManagementFactory;
25 import java.lang.management.MemoryUsage;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35 import org.apache.hadoop.hbase.util.ReflectionUtils;
36 import org.apache.hadoop.util.StringUtils;
37
38 import com.google.common.annotations.VisibleForTesting;
39
40
41
42
43 @InterfaceAudience.Private
44 public class CacheConfig {
45 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
46
47
48
49
50
51
52 public static final CacheConfig DISABLED = new CacheConfig();
53
54
55
56
57
58
59 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
60 "hbase.rs.cacheblocksonwrite";
61
62
63
64
65
66 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
67 "hfile.block.index.cacheonwrite";
68
69
70
71
72 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
73 "hfile.block.bloom.cacheonwrite";
74
75
76
77
78 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
79 "hbase.block.data.cachecompressed";
80
81
82
83
84
85 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
86 "hbase.rs.evictblocksonclose";
87
88
89
90
91
92
93
94
95
96 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
97 "hbase.bucketcache.persistent.path";
98
99
100
101
102
103
104 public static final String BUCKET_CACHE_COMBINED_KEY =
105 "hbase.bucketcache.combinedcache.enabled";
106
107 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
108 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
109 "hbase.bucketcache.writer.queuelength";
110
111
112
113
114 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
115
116
117
118
119 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
120 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
121 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
122
123
124
125
126
127 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
128 "hbase.rs.prefetchblocksonopen";
129
130
131
132
133
134
135
136 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
137
138 private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
139 private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
140
141 private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
142
143
144
145
146
147 private static enum ExternalBlockCaches {
148 memcached(MemcachedBlockCache.class);
149
150 Class<? extends BlockCache> clazz;
151 ExternalBlockCaches(Class<? extends BlockCache> clazz) {
152 this.clazz = clazz;
153 }
154 }
155
156
157 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
158 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
159 public static final boolean DEFAULT_IN_MEMORY = false;
160 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
161 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
162 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
163 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
164 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
165
166
167 private final BlockCache blockCache;
168
169
170
171
172
173
174
175 private boolean cacheDataOnRead;
176
177
178 private final boolean inMemory;
179
180
181 private boolean cacheDataOnWrite;
182
183
184 private final boolean cacheIndexesOnWrite;
185
186
187 private final boolean cacheBloomsOnWrite;
188
189
190 private boolean evictOnClose;
191
192
193 private final boolean cacheDataCompressed;
194
195
196 private final boolean prefetchOnOpen;
197
198
199
200
201
202
203 private boolean cacheDataInL1;
204
205
206
207
208
209
210
211 public CacheConfig(Configuration conf, HColumnDescriptor family) {
212 this(CacheConfig.instantiateBlockCache(conf),
213 family.isBlockCacheEnabled(),
214 family.isInMemory(),
215
216
217 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
218 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(),
219 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
220 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(),
221 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
222 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(),
223 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
224 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(),
225 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
226 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
227 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
228 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
229 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1()
230 );
231 }
232
233
234
235
236
237
238 public CacheConfig(Configuration conf) {
239 this(CacheConfig.instantiateBlockCache(conf),
240 DEFAULT_CACHE_DATA_ON_READ,
241 DEFAULT_IN_MEMORY,
242
243 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
244 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
245 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
246 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
247 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
248 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
249 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
250 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
251 );
252 }
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270 CacheConfig(final BlockCache blockCache,
271 final boolean cacheDataOnRead, final boolean inMemory,
272 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
273 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
274 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
275 final boolean cacheDataInL1) {
276 this.blockCache = blockCache;
277 this.cacheDataOnRead = cacheDataOnRead;
278 this.inMemory = inMemory;
279 this.cacheDataOnWrite = cacheDataOnWrite;
280 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
281 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
282 this.evictOnClose = evictOnClose;
283 this.cacheDataCompressed = cacheDataCompressed;
284 this.prefetchOnOpen = prefetchOnOpen;
285 this.cacheDataInL1 = cacheDataInL1;
286 LOG.info(this);
287 }
288
289
290
291
292
293 public CacheConfig(CacheConfig cacheConf) {
294 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
295 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
296 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
297 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
298 cacheConf.cacheDataInL1);
299 }
300
301 private CacheConfig() {
302 this(null, false, false, false, false,
303 false, false, false, false, false);
304 }
305
306
307
308
309 public boolean isBlockCacheEnabled() {
310 return this.blockCache != null;
311 }
312
313
314
315
316
317 public BlockCache getBlockCache() {
318 return this.blockCache;
319 }
320
321
322
323
324
325
326 public boolean shouldCacheDataOnRead() {
327 return isBlockCacheEnabled() && cacheDataOnRead;
328 }
329
330
331
332
333
334
335 public boolean shouldCacheBlockOnRead(BlockCategory category) {
336 return isBlockCacheEnabled()
337 && (cacheDataOnRead ||
338 category == BlockCategory.INDEX ||
339 category == BlockCategory.BLOOM ||
340 (prefetchOnOpen &&
341 (category != BlockCategory.META &&
342 category != BlockCategory.UNKNOWN)));
343 }
344
345
346
347
348 public boolean isInMemory() {
349 return isBlockCacheEnabled() && this.inMemory;
350 }
351
352
353
354
355 public boolean isCacheDataInL1() {
356 return isBlockCacheEnabled() && this.cacheDataInL1;
357 }
358
359
360
361
362
363 public boolean shouldCacheDataOnWrite() {
364 return isBlockCacheEnabled() && this.cacheDataOnWrite;
365 }
366
367
368
369
370
371
372 @VisibleForTesting
373 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
374 this.cacheDataOnWrite = cacheDataOnWrite;
375 }
376
377
378
379
380
381
382 @VisibleForTesting
383 public void setCacheDataInL1(boolean cacheDataInL1) {
384 this.cacheDataInL1 = cacheDataInL1;
385 }
386
387
388
389
390
391 public boolean shouldCacheIndexesOnWrite() {
392 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
393 }
394
395
396
397
398
399 public boolean shouldCacheBloomsOnWrite() {
400 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
401 }
402
403
404
405
406
407 public boolean shouldEvictOnClose() {
408 return isBlockCacheEnabled() && this.evictOnClose;
409 }
410
411
412
413
414
415
416 public void setEvictOnClose(boolean evictOnClose) {
417 this.evictOnClose = evictOnClose;
418 }
419
420
421
422
423 public boolean shouldCacheDataCompressed() {
424 return isBlockCacheEnabled() && this.cacheDataCompressed;
425 }
426
427
428
429
430 public boolean shouldCacheCompressed(BlockCategory category) {
431 if (!isBlockCacheEnabled()) return false;
432 switch (category) {
433 case DATA:
434 return this.cacheDataCompressed;
435 default:
436 return false;
437 }
438 }
439
440
441
442
443 public boolean shouldPrefetchOnOpen() {
444 return isBlockCacheEnabled() && this.prefetchOnOpen;
445 }
446
447
448
449
450
451
452
453
454
455 public boolean shouldReadBlockFromCache(BlockType blockType) {
456 if (!isBlockCacheEnabled()) {
457 return false;
458 }
459 if (cacheDataOnRead) {
460 return true;
461 }
462 if (prefetchOnOpen) {
463 return true;
464 }
465 if (cacheDataOnWrite) {
466 return true;
467 }
468 if (blockType == null) {
469 return true;
470 }
471 if (blockType.getCategory() == BlockCategory.BLOOM ||
472 blockType.getCategory() == BlockCategory.INDEX) {
473 return true;
474 }
475 return false;
476 }
477
478
479
480
481
482 public boolean shouldLockOnCacheMiss(BlockType blockType) {
483 if (blockType == null) {
484 return true;
485 }
486 return shouldCacheBlockOnRead(blockType.getCategory());
487 }
488
489 @Override
490 public String toString() {
491 if (!isBlockCacheEnabled()) {
492 return "CacheConfig:disabled";
493 }
494 return "blockCache=" + getBlockCache() +
495 ", cacheDataOnRead=" + shouldCacheDataOnRead() +
496 ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
497 ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
498 ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
499 ", cacheEvictOnClose=" + shouldEvictOnClose() +
500 ", cacheDataCompressed=" + shouldCacheDataCompressed() +
501 ", prefetchOnOpen=" + shouldPrefetchOnOpen();
502 }
503
504
505
506
507
508
509
510
511 @VisibleForTesting
512 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
513
514
515 @VisibleForTesting
516 static boolean blockCacheDisabled = false;
517
518 static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
519 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
520 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
521 if (cachePercentage <= 0.0001f) {
522 blockCacheDisabled = true;
523 return -1;
524 }
525 if (cachePercentage > 1.0) {
526 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
527 " must be between 0.0 and 1.0, and not > 1.0");
528 }
529
530
531 return (long) (mu.getMax() * cachePercentage);
532 }
533
534
535
536
537
538
539 private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
540 long lruCacheSize = getLruCacheSize(c, mu);
541 if (lruCacheSize < 0) return null;
542 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
543 LOG.info("Allocating LruBlockCache size=" +
544 StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
545 return new LruBlockCache(lruCacheSize, blockSize, true, c);
546 }
547
548
549
550
551
552
553
554 private static BlockCache getL2(final Configuration c, final MemoryUsage mu) {
555 final boolean useExternal = c.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
556 if (LOG.isDebugEnabled()) {
557 LOG.debug("Trying to use " + (useExternal?" External":" Internal") + " l2 cache");
558 }
559
560
561 if (useExternal) {
562 return getExternalBlockcache(c);
563 }
564
565
566 return getBucketCache(c, mu);
567
568 }
569
570 private static BlockCache getExternalBlockcache(Configuration c) {
571 Class klass = null;
572
573
574 try {
575 klass = ExternalBlockCaches.valueOf(c.get(EXTERNAL_BLOCKCACHE_CLASS_KEY, "memcache")).clazz;
576 } catch (IllegalArgumentException exception) {
577 klass = c.getClass(EXTERNAL_BLOCKCACHE_CLASS_KEY, MemcachedBlockCache.class);
578 }
579
580
581 try {
582 LOG.info("Creating external block cache of type: " + klass);
583 return (BlockCache) ReflectionUtils.newInstance(klass, c);
584 } catch (Exception e) {
585 LOG.warn("Error creating external block cache", e);
586 }
587 return null;
588
589 }
590
591 private static BlockCache getBucketCache(Configuration c, MemoryUsage mu) {
592
593 String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
594 if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
595
596 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
597 float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
598 long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
599 bucketCachePercentage * 1024 * 1024);
600 if (bucketCacheSize <= 0) {
601 throw new IllegalStateException("bucketCacheSize <= 0; Check " +
602 BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
603 }
604 if (c.get("hbase.bucketcache.percentage.in.combinedcache") != null) {
605 LOG.warn("Configuration 'hbase.bucketcache.percentage.in.combinedcache' is no longer "
606 + "respected. See comments in http://hbase.apache.org/book.html#_changes_of_note");
607 }
608 int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
609 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
610 int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
611 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
612 String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
613 String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
614 int [] bucketSizes = null;
615 if (configuredBucketSizes != null) {
616 bucketSizes = new int[configuredBucketSizes.length];
617 for (int i = 0; i < configuredBucketSizes.length; i++) {
618 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
619 }
620 }
621 BucketCache bucketCache = null;
622 try {
623 int ioErrorsTolerationDuration = c.getInt(
624 "hbase.bucketcache.ioengine.errors.tolerated.duration",
625 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
626
627 bucketCache = new BucketCache(bucketCacheIOEngineName,
628 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
629 ioErrorsTolerationDuration);
630 } catch (IOException ioex) {
631 LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
632 }
633 return bucketCache;
634 }
635
636
637
638
639
640
641
642
643 public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
644 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
645 if (blockCacheDisabled) return null;
646 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
647 LruBlockCache l1 = getL1(conf, mu);
648
649 if (blockCacheDisabled) return null;
650 BlockCache l2 = getL2(conf, mu);
651 if (l2 == null) {
652 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
653 } else {
654 boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
655 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
656 DEFAULT_BUCKET_CACHE_COMBINED);
657 if (useExternal) {
658 GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
659 } else {
660 if (combinedWithLru) {
661 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
662 } else {
663
664
665
666
667 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
668 }
669 }
670 l1.setVictimCache(l2);
671 }
672 return GLOBAL_BLOCK_CACHE_INSTANCE;
673 }
674 }