1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.util.NavigableMap;
22 import java.util.NavigableSet;
23 import java.util.concurrent.ConcurrentSkipListMap;
24 import java.util.concurrent.ConcurrentSkipListSet;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.codehaus.jackson.JsonGenerationException;
29 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
30 import org.codehaus.jackson.map.JsonMappingException;
31 import org.codehaus.jackson.map.ObjectMapper;
32 import org.codehaus.jackson.map.SerializationConfig;
33
34 import com.yammer.metrics.core.Histogram;
35 import com.yammer.metrics.core.MetricsRegistry;
36 import com.yammer.metrics.stats.Snapshot;
37
38
39
40
41
42 @InterfaceAudience.Private
43 public class BlockCacheUtil {
44
45
46
47 private static final MetricsRegistry METRICS = new MetricsRegistry();
48
49
50
51
52 private static final ObjectMapper MAPPER = new ObjectMapper();
53 static {
54 MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
55 MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
56 MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
57 }
58
59
60
61
62
63 public static String toString(final CachedBlock cb, final long now) {
64 return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
65 }
66
67
68
69
70
71 static class CachedBlockCountsPerFile {
72 private int count = 0;
73 private long size = 0;
74 private int countData = 0;
75 private long sizeData = 0;
76 private final String filename;
77
78 CachedBlockCountsPerFile(final String filename) {
79 this.filename = filename;
80 }
81
82 public int getCount() {
83 return count;
84 }
85
86 public long getSize() {
87 return size;
88 }
89
90 public int getCountData() {
91 return countData;
92 }
93
94 public long getSizeData() {
95 return sizeData;
96 }
97
98 public String getFilename() {
99 return filename;
100 }
101 }
102
103
104
105
106
107
108
109
110
111 public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
112 throws JsonGenerationException, JsonMappingException, IOException {
113 CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
114 for (CachedBlock cb: blocks) {
115 counts.count++;
116 counts.size += cb.getSize();
117 BlockType bt = cb.getBlockType();
118 if (bt != null && bt.isData()) {
119 counts.countData++;
120 counts.sizeData += cb.getSize();
121 }
122 }
123 return MAPPER.writeValueAsString(counts);
124 }
125
126
127
128
129
130
131
132
133 public static String toJSON(final CachedBlocksByFile cbsbf)
134 throws JsonGenerationException, JsonMappingException, IOException {
135 return MAPPER.writeValueAsString(cbsbf);
136 }
137
138
139
140
141
142
143
144
145 public static String toJSON(final BlockCache bc)
146 throws JsonGenerationException, JsonMappingException, IOException {
147 return MAPPER.writeValueAsString(bc);
148 }
149
150
151
152
153
154 public static String toStringMinusFileName(final CachedBlock cb, final long now) {
155 return "offset=" + cb.getOffset() +
156 ", size=" + cb.getSize() +
157 ", age=" + (now - cb.getCachedTime()) +
158 ", type=" + cb.getBlockType() +
159 ", priority=" + cb.getBlockPriority();
160 }
161
162
163
164
165
166
167
168
169 public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
170 final BlockCache bc) {
171 CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
172 for (CachedBlock cb: bc) {
173 if (cbsbf.update(cb)) break;
174 }
175 return cbsbf;
176 }
177
178
179
180
181
182
183 @JsonIgnoreProperties({"cachedBlockStatsByFile"})
184 public static class CachedBlocksByFile {
185 private int count;
186 private int dataBlockCount;
187 private long size;
188 private long dataSize;
189 private final long now = System.nanoTime();
190 private final int max;
191 public static final int DEFAULT_MAX = 100000;
192
193 CachedBlocksByFile() {
194 this(null);
195 }
196
197 CachedBlocksByFile(final Configuration c) {
198 this.max = c == null? DEFAULT_MAX:
199 c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
200 }
201
202
203
204
205 private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
206 new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
207 Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
208
209
210
211
212
213 public boolean update(final CachedBlock cb) {
214 if (isFull()) return true;
215 NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
216 if (set == null) {
217 set = new ConcurrentSkipListSet<CachedBlock>();
218 this.cachedBlockByFile.put(cb.getFilename(), set);
219 }
220 set.add(cb);
221 this.size += cb.getSize();
222 this.count++;
223 BlockType bt = cb.getBlockType();
224 if (bt != null && bt.isData()) {
225 this.dataBlockCount++;
226 this.dataSize += cb.getSize();
227 }
228 long age = this.now - cb.getCachedTime();
229 this.age.update(age);
230 return false;
231 }
232
233
234
235
236
237
238 public boolean isFull() {
239 return this.count >= this.max;
240 }
241
242 public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
243 return this.cachedBlockByFile;
244 }
245
246
247
248
249 public int getCount() {
250 return count;
251 }
252
253 public int getDataCount() {
254 return dataBlockCount;
255 }
256
257
258
259
260 public long getSize() {
261 return size;
262 }
263
264
265
266
267 public long getDataSize() {
268 return dataSize;
269 }
270
271 public AgeSnapshot getAgeInCacheSnapshot() {
272 return new AgeSnapshot(this.age);
273 }
274
275 @Override
276 public String toString() {
277 Snapshot snapshot = this.age.getSnapshot();
278 return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
279 ", dataSize=" + getDataSize() +
280 ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
281 ", min age=" + this.age.min() + ", max age=" + this.age.max() +
282 ", 95th percentile age=" + snapshot.get95thPercentile() +
283 ", 99th percentile age=" + snapshot.get99thPercentile();
284 }
285 }
286 }