1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.regex.Matcher;
25 import java.util.regex.Pattern;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.HFileLink;
37 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
38 import org.apache.hadoop.hbase.io.Reference;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 private static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53
54 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:(?:_SeqId_[0-9]+_)|(?:_del))?";
55
56
57 private static final Pattern HFILE_NAME_PATTERN =
58 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
59
60
61
62
63
64 public static final String DELFILE_NAME_REGEX = "[0-9a-f]+(?:_del)";
65
66
67 private static final Pattern DELFILE_NAME_PATTERN =
68 Pattern.compile("^(" + DELFILE_NAME_REGEX + ")");
69
70
71
72
73
74
75
76
77 private static final Pattern REF_NAME_PATTERN =
78 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
79 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
80
81
82 private Configuration conf;
83
84
85 private final FileSystem fs;
86
87
88 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
89
90
91 private final Reference reference;
92
93
94 private final HFileLink link;
95
96 private final Path initialPath;
97
98 private RegionCoprocessorHost coprocessorHost;
99
100
101
102
103
104
105
106 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath)
107 throws IOException {
108 assert fs != null;
109 assert initialPath != null;
110 assert conf != null;
111
112 this.fs = fs;
113 this.conf = conf;
114 this.initialPath = initialPath;
115 Path p = initialPath;
116 if (HFileLink.isHFileLink(p)) {
117
118 this.reference = null;
119 this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
120 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
121 } else if (isReference(p)) {
122 this.reference = Reference.read(fs, p);
123 Path referencePath = getReferredToFile(p);
124 if (HFileLink.isHFileLink(referencePath)) {
125
126 this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
127 } else {
128
129 this.link = null;
130 }
131 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
132 " reference to " + referencePath);
133 } else if (isHFile(p)) {
134
135 this.reference = null;
136 this.link = null;
137 } else {
138 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
139 }
140 }
141
142
143
144
145
146
147
148 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
149 throws IOException {
150 this(conf, fs, fileStatus.getPath());
151 }
152
153
154
155
156
157
158
159 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
160 final HFileLink link)
161 throws IOException {
162 this.fs = fs;
163 this.conf = conf;
164
165 this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
166
167 this.reference = null;
168 this.link = link;
169 }
170
171
172
173
174
175
176
177
178
179 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
180 final Reference reference)
181 throws IOException {
182 this.fs = fs;
183 this.conf = conf;
184 this.initialPath = fileStatus.getPath();
185 this.reference = reference;
186 this.link = null;
187 }
188
189
190
191
192
193 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
194 this.coprocessorHost = coprocessorHost;
195 }
196
197
198
199
200
201 public Reference getReference() {
202 return this.reference;
203 }
204
205
206 public boolean isReference() {
207 return this.reference != null;
208 }
209
210
211 public boolean isTopReference() {
212 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
213 }
214
215
216 public boolean isLink() {
217 return this.link != null && this.reference == null;
218 }
219
220
221 public HDFSBlocksDistribution getHDFSBlockDistribution() {
222 return this.hdfsBlocksDistribution;
223 }
224
225
226
227
228
229
230
231 public StoreFile.Reader open(final FileSystem fs,
232 final CacheConfig cacheConf, final boolean canUseDropBehind) throws IOException {
233 FSDataInputStreamWrapper in;
234 FileStatus status;
235
236 final boolean doDropBehind = canUseDropBehind && cacheConf.shouldDropBehindCompaction();
237 if (this.link != null) {
238
239 in = new FSDataInputStreamWrapper(fs, this.link, doDropBehind);
240 status = this.link.getFileStatus(fs);
241 } else if (this.reference != null) {
242
243 Path referencePath = getReferredToFile(this.getPath());
244 in = new FSDataInputStreamWrapper(fs, referencePath,
245 doDropBehind);
246 status = fs.getFileStatus(referencePath);
247 } else {
248 in = new FSDataInputStreamWrapper(fs, this.getPath(),
249 doDropBehind);
250 status = fs.getFileStatus(initialPath);
251 }
252 long length = status.getLen();
253 hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
254
255 StoreFile.Reader reader = null;
256 if (this.coprocessorHost != null) {
257 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
258 cacheConf, reference);
259 }
260 if (reader == null) {
261 if (this.reference != null) {
262 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
263 conf);
264 } else {
265 reader = new StoreFile.Reader(fs, status.getPath(), in, length, cacheConf, conf);
266 }
267 }
268 if (this.coprocessorHost != null) {
269 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
270 cacheConf, reference, reader);
271 }
272 return reader;
273 }
274
275
276
277
278 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
279 throws IOException {
280
281
282
283 if (this.link != null) {
284 FileNotFoundException exToThrow = null;
285 for (int i = 0; i < this.link.getLocations().length; i++) {
286 try {
287 return computeHDFSBlocksDistributionInternal(fs);
288 } catch (FileNotFoundException ex) {
289
290 exToThrow = ex;
291 }
292 }
293 throw exToThrow;
294 } else {
295 return computeHDFSBlocksDistributionInternal(fs);
296 }
297 }
298
299 private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
300 throws IOException {
301 FileStatus status = getReferencedFileStatus(fs);
302 if (this.reference != null) {
303 return computeRefFileHDFSBlockDistribution(fs, reference, status);
304 } else {
305 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
306 }
307 }
308
309
310
311
312
313
314 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
315 FileStatus status;
316 if (this.reference != null) {
317 if (this.link != null) {
318 FileNotFoundException exToThrow = null;
319 for (int i = 0; i < this.link.getLocations().length; i++) {
320
321 try {
322 return link.getFileStatus(fs);
323 } catch (FileNotFoundException ex) {
324
325 exToThrow = ex;
326 }
327 }
328 throw exToThrow;
329 } else {
330
331 Path referencePath = getReferredToFile(this.getPath());
332 status = fs.getFileStatus(referencePath);
333 }
334 } else {
335 if (this.link != null) {
336 FileNotFoundException exToThrow = null;
337 for (int i = 0; i < this.link.getLocations().length; i++) {
338
339 try {
340 return link.getFileStatus(fs);
341 } catch (FileNotFoundException ex) {
342
343 exToThrow = ex;
344 }
345 }
346 throw exToThrow;
347 } else {
348 status = fs.getFileStatus(initialPath);
349 }
350 }
351 return status;
352 }
353
354
355 public Path getPath() {
356 return initialPath;
357 }
358
359
360 public FileStatus getFileStatus() throws IOException {
361 return getReferencedFileStatus(fs);
362 }
363
364
365 public long getModificationTime() throws IOException {
366 return getFileStatus().getModificationTime();
367 }
368
369 @Override
370 public String toString() {
371 return this.getPath() +
372 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
373 }
374
375
376
377
378
379 public static boolean isHFile(final Path path) {
380 return isHFile(path.getName());
381 }
382
383 public static boolean isHFile(final String fileName) {
384 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
385 return m.matches() && m.groupCount() > 0;
386 }
387
388
389
390
391
392 public static boolean isDelFile(final Path path) {
393 return isDelFile(path.getName());
394 }
395
396
397
398
399
400 public static boolean isDelFile(final String fileName) {
401 Matcher m = DELFILE_NAME_PATTERN.matcher(fileName);
402 return m.matches() && m.groupCount() > 0;
403 }
404
405
406
407
408
409 public static boolean isReference(final Path path) {
410 return isReference(path.getName());
411 }
412
413
414
415
416
417 public static boolean isReference(final String name) {
418 Matcher m = REF_NAME_PATTERN.matcher(name);
419 return m.matches() && m.groupCount() > 1;
420 }
421
422
423
424
425
426
427
428
429 public static Path getReferredToFile(final Path p) {
430 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
431 if (m == null || !m.matches()) {
432 LOG.warn("Failed match of store file name " + p.toString());
433 throw new IllegalArgumentException("Failed match of store file name " +
434 p.toString());
435 }
436
437
438 String otherRegion = m.group(2);
439
440 Path tableDir = p.getParent().getParent().getParent();
441 String nameStrippedOfSuffix = m.group(1);
442 if (LOG.isDebugEnabled()) {
443 LOG.debug("reference '" + p + "' to region=" + otherRegion
444 + " hfile=" + nameStrippedOfSuffix);
445 }
446
447
448
449 return new Path(new Path(new Path(tableDir, otherRegion),
450 p.getParent().getName()), nameStrippedOfSuffix);
451 }
452
453
454
455
456
457
458 public static boolean validateStoreFileName(final String fileName) {
459 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
460 return(true);
461 return !fileName.contains("-");
462 }
463
464
465
466
467
468
469 public static boolean isValid(final FileStatus fileStatus)
470 throws IOException {
471 final Path p = fileStatus.getPath();
472
473 if (fileStatus.isDirectory())
474 return false;
475
476
477
478
479 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
480 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
481 return false;
482 }
483
484 return validateStoreFileName(p.getName());
485 }
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
501 final FileSystem fs, final Reference reference, final FileStatus status)
502 throws IOException {
503 if (status == null) {
504 return null;
505 }
506
507 long start = 0;
508 long length = 0;
509
510 if (Reference.isTopFileRegion(reference.getFileRegion())) {
511 start = status.getLen()/2;
512 length = status.getLen() - status.getLen()/2;
513 } else {
514 start = 0;
515 length = status.getLen()/2;
516 }
517 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
518 }
519
520 @Override
521 public boolean equals(Object that) {
522 if (this == that) return true;
523 if (that == null) return false;
524
525 if (!(that instanceof StoreFileInfo)) return false;
526
527 StoreFileInfo o = (StoreFileInfo)that;
528 if (initialPath != null && o.initialPath == null) return false;
529 if (initialPath == null && o.initialPath != null) return false;
530 if (initialPath != o.initialPath && initialPath != null
531 && !initialPath.equals(o.initialPath)) return false;
532
533 if (reference != null && o.reference == null) return false;
534 if (reference == null && o.reference != null) return false;
535 if (reference != o.reference && reference != null
536 && !reference.equals(o.reference)) return false;
537
538 if (link != null && o.link == null) return false;
539 if (link == null && o.link != null) return false;
540 if (link != o.link && link != null && !link.equals(o.link)) return false;
541
542 return true;
543 };
544
545
546 @Override
547 public int hashCode() {
548 int hash = 17;
549 hash = hash * 31 + ((reference == null) ? 0 : reference.hashCode());
550 hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
551 hash = hash * 31 + ((link == null) ? 0 : link.hashCode());
552 return hash;
553 }
554 }