1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22
23 import java.io.IOException;
24 import java.lang.management.ManagementFactory;
25 import java.lang.management.MemoryUsage;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35 import org.apache.hadoop.hbase.util.ReflectionUtils;
36 import org.apache.hadoop.util.StringUtils;
37
38 import com.google.common.annotations.VisibleForTesting;
39
40
41
42
43 @InterfaceAudience.Private
44 public class CacheConfig {
45 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
46
47
48
49
50
51 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
52 "hbase.rs.cacheblocksonwrite";
53
54
55
56
57
58 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
59 "hfile.block.index.cacheonwrite";
60
61
62
63
64 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
65 "hfile.block.bloom.cacheonwrite";
66
67
68
69
70 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
71 "hbase.block.data.cachecompressed";
72
73
74
75
76
77 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
78 "hbase.rs.evictblocksonclose";
79
80
81
82
83
84
85
86
87
88 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
89 "hbase.bucketcache.persistent.path";
90
91
92
93
94
95
96 public static final String BUCKET_CACHE_COMBINED_KEY =
97 "hbase.bucketcache.combinedcache.enabled";
98
99 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
100 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
101 "hbase.bucketcache.writer.queuelength";
102
103
104
105
106 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
107
108
109
110
111 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
112 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
113 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
114
115
116
117
118
119 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
120 "hbase.rs.prefetchblocksonopen";
121
122
123
124
125
126
127
128 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
129
130 private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
131 private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
132
133 private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
134 private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction";
135 private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
136
137
138
139
140
141 private static enum ExternalBlockCaches {
142 memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache");
143
144 Class<? extends BlockCache> clazz;
145 ExternalBlockCaches(String clazzName) {
146 try {
147 clazz = (Class<? extends BlockCache>) Class.forName(clazzName);
148 } catch (ClassNotFoundException cnef) {
149 clazz = null;
150 }
151 }
152 ExternalBlockCaches(Class<? extends BlockCache> clazz) {
153 this.clazz = clazz;
154 }
155 }
156
157
158 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
159 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
160 public static final boolean DEFAULT_IN_MEMORY = false;
161 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
162 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
163 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
164 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
165 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
166
167
168 private final BlockCache blockCache;
169
170
171
172
173
174
175
176 private boolean cacheDataOnRead;
177
178
179 private final boolean inMemory;
180
181
182 private boolean cacheDataOnWrite;
183
184
185 private final boolean cacheIndexesOnWrite;
186
187
188 private final boolean cacheBloomsOnWrite;
189
190
191 private boolean evictOnClose;
192
193
194 private final boolean cacheDataCompressed;
195
196
197 private final boolean prefetchOnOpen;
198
199
200
201
202
203
204 private boolean cacheDataInL1;
205
206 private final boolean dropBehindCompaction;
207
208
209
210
211
212
213
214 public CacheConfig(Configuration conf, HColumnDescriptor family) {
215 this(CacheConfig.instantiateBlockCache(conf),
216 family.isBlockCacheEnabled(),
217 family.isInMemory(),
218
219
220 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
221 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(),
222 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
223 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(),
224 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
225 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(),
226 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
227 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(),
228 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
229 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
230 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
231 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
232 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(),
233 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
234 );
235 }
236
237
238
239
240
241
242 public CacheConfig(Configuration conf) {
243 this(CacheConfig.instantiateBlockCache(conf),
244 DEFAULT_CACHE_DATA_ON_READ,
245 DEFAULT_IN_MEMORY,
246
247 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
248 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
249 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
250 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
251 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
252 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
253 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
254 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1),
255 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
256 );
257 }
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275 CacheConfig(final BlockCache blockCache,
276 final boolean cacheDataOnRead, final boolean inMemory,
277 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
278 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
279 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
280 final boolean cacheDataInL1, final boolean dropBehindCompaction) {
281 this.blockCache = blockCache;
282 this.cacheDataOnRead = cacheDataOnRead;
283 this.inMemory = inMemory;
284 this.cacheDataOnWrite = cacheDataOnWrite;
285 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
286 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
287 this.evictOnClose = evictOnClose;
288 this.cacheDataCompressed = cacheDataCompressed;
289 this.prefetchOnOpen = prefetchOnOpen;
290 this.cacheDataInL1 = cacheDataInL1;
291 this.dropBehindCompaction = dropBehindCompaction;
292 LOG.info(this);
293 }
294
295
296
297
298
299 public CacheConfig(CacheConfig cacheConf) {
300 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
301 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
302 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
303 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
304 cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction);
305 }
306
307
308
309
310 public boolean isBlockCacheEnabled() {
311 return this.blockCache != null;
312 }
313
314
315
316
317
318 public BlockCache getBlockCache() {
319 return this.blockCache;
320 }
321
322
323
324
325
326
327 public boolean shouldCacheDataOnRead() {
328 return isBlockCacheEnabled() && cacheDataOnRead;
329 }
330
331 public boolean shouldDropBehindCompaction() {
332 return dropBehindCompaction;
333 }
334
335
336
337
338
339
340 public boolean shouldCacheBlockOnRead(BlockCategory category) {
341 return isBlockCacheEnabled()
342 && (cacheDataOnRead ||
343 category == BlockCategory.INDEX ||
344 category == BlockCategory.BLOOM ||
345 (prefetchOnOpen &&
346 (category != BlockCategory.META &&
347 category != BlockCategory.UNKNOWN)));
348 }
349
350
351
352
353 public boolean isInMemory() {
354 return isBlockCacheEnabled() && this.inMemory;
355 }
356
357
358
359
360 public boolean isCacheDataInL1() {
361 return isBlockCacheEnabled() && this.cacheDataInL1;
362 }
363
364
365
366
367
368 public boolean shouldCacheDataOnWrite() {
369 return isBlockCacheEnabled() && this.cacheDataOnWrite;
370 }
371
372
373
374
375
376
377 @VisibleForTesting
378 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
379 this.cacheDataOnWrite = cacheDataOnWrite;
380 }
381
382
383
384
385
386
387 @VisibleForTesting
388 public void setCacheDataInL1(boolean cacheDataInL1) {
389 this.cacheDataInL1 = cacheDataInL1;
390 }
391
392
393
394
395
396 public boolean shouldCacheIndexesOnWrite() {
397 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
398 }
399
400
401
402
403
404 public boolean shouldCacheBloomsOnWrite() {
405 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
406 }
407
408
409
410
411
412 public boolean shouldEvictOnClose() {
413 return isBlockCacheEnabled() && this.evictOnClose;
414 }
415
416
417
418
419
420
421 public void setEvictOnClose(boolean evictOnClose) {
422 this.evictOnClose = evictOnClose;
423 }
424
425
426
427
428 public boolean shouldCacheDataCompressed() {
429 return isBlockCacheEnabled() && this.cacheDataCompressed;
430 }
431
432
433
434
435 public boolean shouldCacheCompressed(BlockCategory category) {
436 if (!isBlockCacheEnabled()) return false;
437 switch (category) {
438 case DATA:
439 return this.cacheDataCompressed;
440 default:
441 return false;
442 }
443 }
444
445
446
447
448 public boolean shouldPrefetchOnOpen() {
449 return isBlockCacheEnabled() && this.prefetchOnOpen;
450 }
451
452
453
454
455
456
457
458
459
460 public boolean shouldReadBlockFromCache(BlockType blockType) {
461 if (!isBlockCacheEnabled()) {
462 return false;
463 }
464 if (cacheDataOnRead) {
465 return true;
466 }
467 if (prefetchOnOpen) {
468 return true;
469 }
470 if (cacheDataOnWrite) {
471 return true;
472 }
473 if (blockType == null) {
474 return true;
475 }
476 if (blockType.getCategory() == BlockCategory.BLOOM ||
477 blockType.getCategory() == BlockCategory.INDEX) {
478 return true;
479 }
480 return false;
481 }
482
483
484
485
486
487 public boolean shouldLockOnCacheMiss(BlockType blockType) {
488 if (blockType == null) {
489 return true;
490 }
491 return shouldCacheBlockOnRead(blockType.getCategory());
492 }
493
494 @Override
495 public String toString() {
496 if (!isBlockCacheEnabled()) {
497 return "CacheConfig:disabled";
498 }
499 return "blockCache=" + getBlockCache() +
500 ", cacheDataOnRead=" + shouldCacheDataOnRead() +
501 ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
502 ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
503 ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
504 ", cacheEvictOnClose=" + shouldEvictOnClose() +
505 ", cacheDataCompressed=" + shouldCacheDataCompressed() +
506 ", prefetchOnOpen=" + shouldPrefetchOnOpen();
507 }
508
509
510
511
512
513
514
515
516 @VisibleForTesting
517 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
518
519
520 @VisibleForTesting
521 static boolean blockCacheDisabled = false;
522
523 static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
524 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
525 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
526 if (cachePercentage <= 0.0001f) {
527 blockCacheDisabled = true;
528 return -1;
529 }
530 if (cachePercentage > 1.0) {
531 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
532 " must be between 0.0 and 1.0, and not > 1.0");
533 }
534
535
536 return (long) (mu.getMax() * cachePercentage);
537 }
538
539
540
541
542
543
544 private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
545 long lruCacheSize = getLruCacheSize(c, mu);
546 if (lruCacheSize < 0) return null;
547 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
548 LOG.info("Allocating LruBlockCache size=" +
549 StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
550 return new LruBlockCache(lruCacheSize, blockSize, true, c);
551 }
552
553
554
555
556
557
558
559 private static BlockCache getL2(final Configuration c, final MemoryUsage mu) {
560 final boolean useExternal = c.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
561 if (LOG.isDebugEnabled()) {
562 LOG.debug("Trying to use " + (useExternal?" External":" Internal") + " l2 cache");
563 }
564
565
566 if (useExternal) {
567 return getExternalBlockcache(c);
568 }
569
570
571 return getBucketCache(c, mu);
572
573 }
574
575 private static BlockCache getExternalBlockcache(Configuration c) {
576 Class klass = null;
577
578
579 try {
580 klass = ExternalBlockCaches.valueOf(c.get(EXTERNAL_BLOCKCACHE_CLASS_KEY, "memcache")).clazz;
581 } catch (IllegalArgumentException exception) {
582 try {
583 klass = c.getClass(EXTERNAL_BLOCKCACHE_CLASS_KEY, Class.forName(
584 "org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache"));
585 } catch (ClassNotFoundException e) {
586 return null;
587 }
588 }
589
590
591 try {
592 LOG.info("Creating external block cache of type: " + klass);
593 return (BlockCache) ReflectionUtils.newInstance(klass, c);
594 } catch (Exception e) {
595 LOG.warn("Error creating external block cache", e);
596 }
597 return null;
598
599 }
600
601 private static BlockCache getBucketCache(Configuration c, MemoryUsage mu) {
602
603 String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
604 if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
605
606 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
607 float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
608 long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
609 bucketCachePercentage * 1024 * 1024);
610 if (bucketCacheSize <= 0) {
611 throw new IllegalStateException("bucketCacheSize <= 0; Check " +
612 BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
613 }
614 if (c.get("hbase.bucketcache.percentage.in.combinedcache") != null) {
615 LOG.warn("Configuration 'hbase.bucketcache.percentage.in.combinedcache' is no longer "
616 + "respected. See comments in http://hbase.apache.org/book.html#_changes_of_note");
617 }
618 int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
619 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
620 int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
621 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
622 String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
623 String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
624 int [] bucketSizes = null;
625 if (configuredBucketSizes != null) {
626 bucketSizes = new int[configuredBucketSizes.length];
627 for (int i = 0; i < configuredBucketSizes.length; i++) {
628 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
629 }
630 }
631 BucketCache bucketCache = null;
632 try {
633 int ioErrorsTolerationDuration = c.getInt(
634 "hbase.bucketcache.ioengine.errors.tolerated.duration",
635 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
636
637 bucketCache = new BucketCache(bucketCacheIOEngineName,
638 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
639 ioErrorsTolerationDuration);
640 } catch (IOException ioex) {
641 LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
642 }
643 return bucketCache;
644 }
645
646
647
648
649
650
651
652
653 public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
654 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
655 if (blockCacheDisabled) return null;
656 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
657 LruBlockCache l1 = getL1(conf, mu);
658
659 if (blockCacheDisabled) return null;
660 BlockCache l2 = getL2(conf, mu);
661 if (l2 == null) {
662 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
663 } else {
664 boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
665 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
666 DEFAULT_BUCKET_CACHE_COMBINED);
667 if (useExternal) {
668 GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
669 } else {
670 if (combinedWithLru) {
671 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
672 } else {
673
674
675
676
677 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
678 }
679 }
680 l1.setVictimCache(l2);
681 }
682 return GLOBAL_BLOCK_CACHE_INSTANCE;
683 }
684 }