View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.util.concurrent.atomic.AtomicLong;
22  
23  import org.apache.hadoop.hbase.classification.InterfaceAudience;
24  
25  import org.apache.hadoop.hbase.util.Counter;
26  import org.apache.hadoop.hbase.util.FastLongHistogram;
27  
28  /**
29   * Class that implements cache metrics.
30   */
31  @InterfaceAudience.Private
32  public class CacheStats {
33  
34    /** Sliding window statistics. The number of metric periods to include in
35     * sliding window hit ratio calculations.
36     */
37    static final int DEFAULT_WINDOW_PERIODS = 5;
38  
39    /** The number of getBlock requests that were cache hits */
40    private final Counter hitCount = new Counter();
41  
42    /** The number of getBlock requests that were cache hits from primary replica */
43    private final Counter primaryHitCount = new Counter();
44    
45    /**
46     * The number of getBlock requests that were cache hits, but only from
47     * requests that were set to use the block cache.  This is because all reads
48     * attempt to read from the block cache even if they will not put new blocks
49     * into the block cache.  See HBASE-2253 for more information.
50     */
51    private final Counter hitCachingCount = new Counter();
52  
53    /** The number of getBlock requests that were cache misses */
54    private final Counter missCount = new Counter();
55  
56    /** The number of getBlock requests for primary replica that were cache misses */
57    private final Counter primaryMissCount = new Counter();
58    /**
59     * The number of getBlock requests that were cache misses, but only from
60     * requests that were set to use the block cache.
61     */
62    private final Counter missCachingCount = new Counter();
63  
64    /** The number of times an eviction has occurred */
65    private final Counter evictionCount = new Counter();
66  
67    /** The total number of blocks that have been evicted */
68    private final Counter evictedBlockCount = new Counter();
69  
70    /** The total number of blocks for primary replica that have been evicted */
71    private final Counter primaryEvictedBlockCount = new Counter();
72  
73    /** Per Block Type Counts */
74    private final Counter dataMissCount = new Counter(0);
75    private final Counter leafIndexMissCount = new Counter(0);
76    private final Counter bloomChunkMissCount = new Counter(0);
77    private final Counter metaMissCount = new Counter(0);
78    private final Counter rootIndexMissCount = new Counter(0);
79    private final Counter intermediateIndexMissCount = new Counter(0);
80    private final Counter fileInfoMissCount = new Counter(0);
81    private final Counter generalBloomMetaMissCount = new Counter(0);
82    private final Counter deleteFamilyBloomMissCount = new Counter(0);
83    private final Counter trailerMissCount = new Counter(0);
84  
85    private final Counter dataHitCount = new Counter(0);
86    private final Counter leafIndexHitCount = new Counter(0);
87    private final Counter bloomChunkHitCount = new Counter(0);
88    private final Counter metaHitCount = new Counter(0);
89    private final Counter rootIndexHitCount = new Counter(0);
90    private final Counter intermediateIndexHitCount = new Counter(0);
91    private final Counter fileInfoHitCount = new Counter(0);
92    private final Counter generalBloomMetaHitCount = new Counter(0);
93    private final Counter deleteFamilyBloomHitCount = new Counter(0);
94    private final Counter trailerHitCount = new Counter(0);
95  
96    /** The number of metrics periods to include in window */
97    private final int numPeriodsInWindow;
98    /** Hit counts for each period in window */
99    private final long [] hitCounts;
100   /** Caching hit counts for each period in window */
101   private final long [] hitCachingCounts;
102   /** Access counts for each period in window */
103   private final long [] requestCounts;
104   /** Caching access counts for each period in window */
105   private final long [] requestCachingCounts;
106   /** Last hit count read */
107   private long lastHitCount = 0;
108   /** Last hit caching count read */
109   private long lastHitCachingCount = 0;
110   /** Last request count read */
111   private long lastRequestCount = 0;
112   /** Last request caching count read */
113   private long lastRequestCachingCount = 0;
114   /** Current window index (next to be updated) */
115   private int windowIndex = 0;
116   /**
117    * Keep running age at eviction time
118    */
119   private FastLongHistogram ageAtEviction;
120   private long startTime = System.nanoTime();
121 
122   public CacheStats(final String name) {
123     this(name, DEFAULT_WINDOW_PERIODS);
124   }
125 
126   public CacheStats(final String name, int numPeriodsInWindow) {
127     this.numPeriodsInWindow = numPeriodsInWindow;
128     this.hitCounts = initializeZeros(numPeriodsInWindow);
129     this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
130     this.requestCounts = initializeZeros(numPeriodsInWindow);
131     this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
132     this.ageAtEviction = new FastLongHistogram();
133   }
134 
135   @Override
136   public String toString() {
137     AgeSnapshot snapshot = getAgeAtEvictionSnapshot();
138     return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() +
139       ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() +
140       ", evictionCount=" + getEvictionCount() +
141       ", evictedBlockCount=" + getEvictedCount() +
142       ", primaryMissCount=" + getPrimaryMissCount() +
143       ", primaryHitCount=" + getPrimaryHitCount() +
144       ", evictedAgeMean=" + snapshot.getMean();
145   }
146 
147 
148   public void miss(boolean caching, boolean primary, BlockType type) {
149     missCount.increment();
150     if (primary) primaryMissCount.increment();
151     if (caching) missCachingCount.increment();
152     if (type == null) {
153       return;
154     }
155     switch (type) {
156       case DATA:
157       case ENCODED_DATA:
158         dataMissCount.increment();
159         break;
160       case LEAF_INDEX:
161         leafIndexMissCount.increment();
162         break;
163       case BLOOM_CHUNK:
164         bloomChunkMissCount.increment();
165         break;
166       case META:
167         metaMissCount.increment();
168         break;
169       case INTERMEDIATE_INDEX:
170         intermediateIndexMissCount.increment();
171         break;
172       case ROOT_INDEX:
173         rootIndexMissCount.increment();
174         break;
175       case FILE_INFO:
176         fileInfoMissCount.increment();
177         break;
178       case GENERAL_BLOOM_META:
179         generalBloomMetaMissCount.increment();
180         break;
181       case DELETE_FAMILY_BLOOM_META:
182         deleteFamilyBloomMissCount.increment();
183         break;
184       case TRAILER:
185         trailerMissCount.increment();
186         break;
187       default:
188         // If there's a new type that's fine
189         // Ignore it for now. This is metrics don't exception.
190         break;
191     }
192   }
193 
194   public void hit(boolean caching, boolean primary, BlockType type) {
195     hitCount.increment();
196     if (primary) primaryHitCount.increment();
197     if (caching) hitCachingCount.increment();
198 
199 
200     if (type == null) {
201       return;
202     }
203     switch (type) {
204       case DATA:
205       case ENCODED_DATA:
206         dataHitCount.increment();
207         break;
208       case LEAF_INDEX:
209         leafIndexHitCount.increment();
210         break;
211       case BLOOM_CHUNK:
212         bloomChunkHitCount.increment();
213         break;
214       case META:
215         metaHitCount.increment();
216         break;
217       case INTERMEDIATE_INDEX:
218         intermediateIndexHitCount.increment();
219         break;
220       case ROOT_INDEX:
221         rootIndexHitCount.increment();
222         break;
223       case FILE_INFO:
224         fileInfoHitCount.increment();
225         break;
226       case GENERAL_BLOOM_META:
227         generalBloomMetaHitCount.increment();
228         break;
229       case DELETE_FAMILY_BLOOM_META:
230         deleteFamilyBloomHitCount.increment();
231         break;
232       case TRAILER:
233         trailerHitCount.increment();
234         break;
235       default:
236         // If there's a new type that's fine
237         // Ignore it for now. This is metrics don't exception.
238         break;
239     }
240   }
241 
242   public void evict() {
243     evictionCount.increment();
244   }
245 
246   public void evicted(final long t, boolean primary) {
247     if (t > this.startTime) this.ageAtEviction.add(t - this.startTime,1);
248     this.evictedBlockCount.increment();
249     if (primary) {
250       primaryEvictedBlockCount.increment();
251     }
252   }
253 
254   // All of the counts of misses and hits.
255   public long getDataMissCount() {
256     return dataMissCount.get();
257   }
258 
259   public long getLeafIndexMissCount() {
260     return leafIndexMissCount.get();
261   }
262 
263   public long getBloomChunkMissCount() {
264     return bloomChunkMissCount.get();
265   }
266 
267   public long getMetaMissCount() {
268     return metaMissCount.get();
269   }
270 
271   public long getRootIndexMissCount() {
272     return rootIndexMissCount.get();
273   }
274 
275   public long getIntermediateIndexMissCount() {
276     return intermediateIndexMissCount.get();
277   }
278 
279   public long getFileInfoMissCount() {
280     return fileInfoMissCount.get();
281   }
282 
283   public long getGeneralBloomMetaMissCount() {
284     return generalBloomMetaMissCount.get();
285   }
286 
287   public long getDeleteFamilyBloomMissCount() {
288     return deleteFamilyBloomMissCount.get();
289   }
290 
291   public long getTrailerMissCount() {
292     return trailerMissCount.get();
293   }
294 
295   public long getDataHitCount() {
296     return dataHitCount.get();
297   }
298 
299   public long getLeafIndexHitCount() {
300     return leafIndexHitCount.get();
301   }
302 
303   public long getBloomChunkHitCount() {
304     return bloomChunkHitCount.get();
305   }
306 
307   public long getMetaHitCount() {
308     return metaHitCount.get();
309   }
310 
311   public long getRootIndexHitCount() {
312     return rootIndexHitCount.get();
313   }
314 
315   public long getIntermediateIndexHitCount() {
316     return intermediateIndexHitCount.get();
317   }
318 
319   public long getFileInfoHitCount() {
320     return fileInfoHitCount.get();
321   }
322 
323   public long getGeneralBloomMetaHitCount() {
324     return generalBloomMetaHitCount.get();
325   }
326 
327   public long getDeleteFamilyBloomHitCount() {
328     return deleteFamilyBloomHitCount.get();
329   }
330 
331   public long getTrailerHitCount() {
332     return trailerHitCount.get();
333   }
334 
335   public long getRequestCount() {
336     return getHitCount() + getMissCount();
337   }
338 
339   public long getRequestCachingCount() {
340     return getHitCachingCount() + getMissCachingCount();
341   }
342 
343   public long getMissCount() {
344     return missCount.get();
345   }
346 
347   public long getPrimaryMissCount() {
348     return primaryMissCount.get();
349   }
350 
351   public long getMissCachingCount() {
352     return missCachingCount.get();
353   }
354 
355   public long getHitCount() {
356     return hitCount.get();
357   }
358 
359   public long getPrimaryHitCount() {
360     return primaryHitCount.get();
361   }
362 
363   public long getHitCachingCount() {
364     return hitCachingCount.get();
365   }
366 
367   public long getEvictionCount() {
368     return evictionCount.get();
369   }
370 
371   public long getEvictedCount() {
372     return this.evictedBlockCount.get();
373   }
374 
375   public long getPrimaryEvictedCount() {
376     return primaryEvictedBlockCount.get();
377   }
378 
379   public double getHitRatio() {
380     return ((float)getHitCount()/(float)getRequestCount());
381   }
382 
383   public double getHitCachingRatio() {
384     return ((float)getHitCachingCount()/(float)getRequestCachingCount());
385   }
386 
387   public double getMissRatio() {
388     return ((float)getMissCount()/(float)getRequestCount());
389   }
390 
391   public double getMissCachingRatio() {
392     return ((float)getMissCachingCount()/(float)getRequestCachingCount());
393   }
394 
395   public double evictedPerEviction() {
396     return ((float)getEvictedCount()/(float)getEvictionCount());
397   }
398 
399   public void rollMetricsPeriod() {
400     hitCounts[windowIndex] = getHitCount() - lastHitCount;
401     lastHitCount = getHitCount();
402     hitCachingCounts[windowIndex] =
403       getHitCachingCount() - lastHitCachingCount;
404     lastHitCachingCount = getHitCachingCount();
405     requestCounts[windowIndex] = getRequestCount() - lastRequestCount;
406     lastRequestCount = getRequestCount();
407     requestCachingCounts[windowIndex] =
408       getRequestCachingCount() - lastRequestCachingCount;
409     lastRequestCachingCount = getRequestCachingCount();
410     windowIndex = (windowIndex + 1) % numPeriodsInWindow;
411   }
412 
413   public long getSumHitCountsPastNPeriods() {
414     return sum(hitCounts);
415   }
416 
417   public long getSumRequestCountsPastNPeriods() {
418     return sum(requestCounts);
419   }
420 
421   public long getSumHitCachingCountsPastNPeriods() {
422     return sum(hitCachingCounts);
423   }
424 
425   public long getSumRequestCachingCountsPastNPeriods() {
426     return sum(requestCachingCounts);
427   }
428 
429   public double getHitRatioPastNPeriods() {
430     double ratio = ((double)sum(hitCounts)/(double)sum(requestCounts));
431     return Double.isNaN(ratio) ? 0 : ratio;
432   }
433 
434   public double getHitCachingRatioPastNPeriods() {
435     double ratio =
436       ((double)sum(hitCachingCounts)/(double)sum(requestCachingCounts));
437     return Double.isNaN(ratio) ? 0 : ratio;
438   }
439 
440   public AgeSnapshot getAgeAtEvictionSnapshot() {
441     return new AgeSnapshot(this.ageAtEviction);
442   }
443 
444   private static long sum(long [] counts) {
445     long sum = 0;
446     for (long count : counts) sum += count;
447     return sum;
448   }
449 
450   private static long [] initializeZeros(int n) {
451     long [] zeros = new long [n];
452     for (int i=0; i<n; i++) {
453       zeros[i] = 0L;
454     }
455     return zeros;
456   }
457 }