1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.util.NavigableMap;
22 import java.util.NavigableSet;
23 import java.util.concurrent.ConcurrentSkipListMap;
24 import java.util.concurrent.ConcurrentSkipListSet;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.util.FastLongHistogram;
29 import org.codehaus.jackson.JsonGenerationException;
30 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
31 import org.codehaus.jackson.map.JsonMappingException;
32 import org.codehaus.jackson.map.ObjectMapper;
33 import org.codehaus.jackson.map.SerializationConfig;
34
35
36
37
38
39 @InterfaceAudience.Private
40 public class BlockCacheUtil {
41
42
43
44 private static final ObjectMapper MAPPER = new ObjectMapper();
45 static {
46 MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
47 MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
48 MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
49 }
50
51
52
53
54
55 public static String toString(final CachedBlock cb, final long now) {
56 return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
57 }
58
59
60
61
62
63 static class CachedBlockCountsPerFile {
64 private int count = 0;
65 private long size = 0;
66 private int countData = 0;
67 private long sizeData = 0;
68 private final String filename;
69
70 CachedBlockCountsPerFile(final String filename) {
71 this.filename = filename;
72 }
73
74 public int getCount() {
75 return count;
76 }
77
78 public long getSize() {
79 return size;
80 }
81
82 public int getCountData() {
83 return countData;
84 }
85
86 public long getSizeData() {
87 return sizeData;
88 }
89
90 public String getFilename() {
91 return filename;
92 }
93 }
94
95
96
97
98
99
100
101
102
103 public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
104 throws JsonGenerationException, JsonMappingException, IOException {
105 CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
106 for (CachedBlock cb: blocks) {
107 counts.count++;
108 counts.size += cb.getSize();
109 BlockType bt = cb.getBlockType();
110 if (bt != null && bt.isData()) {
111 counts.countData++;
112 counts.sizeData += cb.getSize();
113 }
114 }
115 return MAPPER.writeValueAsString(counts);
116 }
117
118
119
120
121
122
123
124
125 public static String toJSON(final CachedBlocksByFile cbsbf)
126 throws JsonGenerationException, JsonMappingException, IOException {
127 return MAPPER.writeValueAsString(cbsbf);
128 }
129
130
131
132
133
134
135
136
137 public static String toJSON(final BlockCache bc)
138 throws JsonGenerationException, JsonMappingException, IOException {
139 return MAPPER.writeValueAsString(bc);
140 }
141
142
143
144
145
146 public static String toStringMinusFileName(final CachedBlock cb, final long now) {
147 return "offset=" + cb.getOffset() +
148 ", size=" + cb.getSize() +
149 ", age=" + (now - cb.getCachedTime()) +
150 ", type=" + cb.getBlockType() +
151 ", priority=" + cb.getBlockPriority();
152 }
153
154
155
156
157
158
159
160
161 public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
162 final BlockCache bc) {
163 CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
164 for (CachedBlock cb: bc) {
165 if (cbsbf.update(cb)) break;
166 }
167 return cbsbf;
168 }
169
170
171
172
173
174
175 @JsonIgnoreProperties({"cachedBlockStatsByFile"})
176 public static class CachedBlocksByFile {
177 private int count;
178 private int dataBlockCount;
179 private long size;
180 private long dataSize;
181 private final long now = System.nanoTime();
182 private final int max;
183 public static final int DEFAULT_MAX = 100000;
184
185 CachedBlocksByFile() {
186 this(null);
187 }
188
189 CachedBlocksByFile(final Configuration c) {
190 this.max = c == null? DEFAULT_MAX:
191 c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
192 }
193
194
195
196
197 private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
198 new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
199 FastLongHistogram hist = new FastLongHistogram();
200
201
202
203
204
205 public boolean update(final CachedBlock cb) {
206 if (isFull()) return true;
207 NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
208 if (set == null) {
209 set = new ConcurrentSkipListSet<CachedBlock>();
210 this.cachedBlockByFile.put(cb.getFilename(), set);
211 }
212 set.add(cb);
213 this.size += cb.getSize();
214 this.count++;
215 BlockType bt = cb.getBlockType();
216 if (bt != null && bt.isData()) {
217 this.dataBlockCount++;
218 this.dataSize += cb.getSize();
219 }
220 long age = this.now - cb.getCachedTime();
221 this.hist.add(age, 1);
222 return false;
223 }
224
225
226
227
228
229
230 public boolean isFull() {
231 return this.count >= this.max;
232 }
233
234 public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
235 return this.cachedBlockByFile;
236 }
237
238
239
240
241 public int getCount() {
242 return count;
243 }
244
245 public int getDataCount() {
246 return dataBlockCount;
247 }
248
249
250
251
252 public long getSize() {
253 return size;
254 }
255
256
257
258
259 public long getDataSize() {
260 return dataSize;
261 }
262
263 public AgeSnapshot getAgeInCacheSnapshot() {
264 return new AgeSnapshot(this.hist);
265 }
266
267 @Override
268 public String toString() {
269 AgeSnapshot snapshot = getAgeInCacheSnapshot();
270 return "count=" + count + ", dataBlockCount=" + dataBlockCount + ", size=" + size +
271 ", dataSize=" + getDataSize() +
272 ", mean age=" + snapshot.getMean() +
273 ", min age=" + snapshot.getMin() +
274 ", max age=" + snapshot.getMax() +
275 ", 75th percentile age=" + snapshot.get75thPercentile() +
276 ", 95th percentile age=" + snapshot.get95thPercentile() +
277 ", 98th percentile age=" + snapshot.get98thPercentile() +
278 ", 99th percentile age=" + snapshot.get99thPercentile() +
279 ", 99.9th percentile age=" + snapshot.get99thPercentile();
280 }
281 }
282 }