1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.Comparator;
24 import java.util.HashSet;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicInteger;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.MetaTableAccessor;
39 import org.apache.hadoop.hbase.ScheduledChore;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.backup.HFileArchiver;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.client.Connection;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.FSUtils;
49 import org.apache.hadoop.hbase.util.Pair;
50 import org.apache.hadoop.hbase.util.PairOfSameType;
51 import org.apache.hadoop.hbase.util.Triple;
52
53
54
55
56
57 @InterfaceAudience.Private
58 public class CatalogJanitor extends ScheduledChore {
59 private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
60 private final Server server;
61 private final MasterServices services;
62 private AtomicBoolean enabled = new AtomicBoolean(true);
63 private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
64 private final Connection connection;
65
66 CatalogJanitor(final Server server, final MasterServices services) {
67 super("CatalogJanitor-" + server.getServerName().toShortString(), server, server
68 .getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
69 this.server = server;
70 this.services = services;
71 this.connection = server.getConnection();
72 }
73
74 @Override
75 protected boolean initialChore() {
76 try {
77 if (this.enabled.get()) scan();
78 } catch (IOException e) {
79 LOG.warn("Failed initial scan of catalog table", e);
80 return false;
81 }
82 return true;
83 }
84
85
86
87
88 public boolean setEnabled(final boolean enabled) {
89 return this.enabled.getAndSet(enabled);
90 }
91
92 boolean getEnabled() {
93 return this.enabled.get();
94 }
95
96 @Override
97 protected void chore() {
98 try {
99 if (this.enabled.get()) {
100 scan();
101 } else {
102 LOG.warn("CatalogJanitor disabled! Not running scan.");
103 }
104 } catch (IOException e) {
105 LOG.warn("Failed scan of catalog table", e);
106 }
107 }
108
109
110
111
112
113
114
115
116 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>
117 getMergedRegionsAndSplitParents() throws IOException {
118 return getMergedRegionsAndSplitParents(null);
119 }
120
121
122
123
124
125
126
127
128
129
130
131 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>
132 getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {
133 final boolean isTableSpecified = (tableName != null);
134
135 final AtomicInteger count = new AtomicInteger(0);
136
137
138 final Map<HRegionInfo, Result> splitParents =
139 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
140 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
141
142
143 MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
144 @Override
145 public boolean visit(Result r) throws IOException {
146 if (r == null || r.isEmpty()) return true;
147 count.incrementAndGet();
148 HRegionInfo info = MetaTableAccessor.getHRegionInfo(r);
149 if (info == null) return true;
150 if (isTableSpecified
151 && info.getTable().compareTo(tableName) > 0) {
152
153 return false;
154 }
155 if (info.isSplitParent()) splitParents.put(info, r);
156 if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
157 mergedRegions.put(info, r);
158 }
159
160 return true;
161 }
162 };
163
164
165
166 MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);
167
168 return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
169 count.get(), mergedRegions, splitParents);
170 }
171
172
173
174
175
176
177
178
179
180
181
182 boolean cleanMergeRegion(final HRegionInfo mergedRegion,
183 final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
184 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
185 Path rootdir = this.services.getMasterFileSystem().getRootDir();
186 Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
187 HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
188 HRegionFileSystem regionFs = null;
189 try {
190 regionFs = HRegionFileSystem.openRegionFromFileSystem(
191 this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
192 } catch (IOException e) {
193 LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
194 }
195 if (regionFs == null || !regionFs.hasReferences(htd)) {
196 LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
197 + regionB.getRegionNameAsString()
198 + " from fs because merged region no longer holds references");
199 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
200 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
201 MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
202 mergedRegion);
203 return true;
204 }
205 return false;
206 }
207
208
209
210
211
212
213
214 int scan() throws IOException {
215 try {
216 if (!alreadyRunning.compareAndSet(false, true)) {
217 return 0;
218 }
219 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
220 getMergedRegionsAndSplitParents();
221 int count = scanTriple.getFirst();
222
223
224
225 int mergeCleaned = 0;
226 Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
227 for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
228 PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(e.getValue());
229 HRegionInfo regionA = p.getFirst();
230 HRegionInfo regionB = p.getSecond();
231 if (regionA == null || regionB == null) {
232 LOG.warn("Unexpected references regionA="
233 + (regionA == null ? "null" : regionA.getRegionNameAsString())
234 + ",regionB="
235 + (regionB == null ? "null" : regionB.getRegionNameAsString())
236 + " in merged region " + e.getKey().getRegionNameAsString());
237 } else {
238 if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
239 mergeCleaned++;
240 }
241 }
242 }
243
244
245
246 Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
247
248
249 int splitCleaned = 0;
250
251 HashSet<String> parentNotCleaned = new HashSet<String>();
252 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
253 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
254 cleanParent(e.getKey(), e.getValue())) {
255 splitCleaned++;
256 } else {
257
258
259 PairOfSameType<HRegionInfo> daughters =
260 MetaTableAccessor.getDaughterRegions(e.getValue());
261 parentNotCleaned.add(daughters.getFirst().getEncodedName());
262 parentNotCleaned.add(daughters.getSecond().getEncodedName());
263 }
264 }
265 if ((mergeCleaned + splitCleaned) != 0) {
266 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
267 + " unreferenced merged region(s) and " + splitCleaned
268 + " unreferenced parent region(s)");
269 } else if (LOG.isTraceEnabled()) {
270 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
271 + " unreferenced merged region(s) and " + splitCleaned
272 + " unreferenced parent region(s)");
273 }
274 return mergeCleaned + splitCleaned;
275 } finally {
276 alreadyRunning.set(false);
277 }
278 }
279
280
281
282
283
284 static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
285 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
286 @Override
287 public int compare(HRegionInfo left, HRegionInfo right) {
288
289
290 if (left == null) return -1;
291 if (right == null) return 1;
292
293 int result = left.getTable().compareTo(right.getTable());
294 if (result != 0) return result;
295
296 result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
297 if (result != 0) return result;
298
299 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
300
301 return result;
302 }
303 }
304
305
306
307
308
309
310
311
312
313
314 boolean cleanParent(final HRegionInfo parent, Result rowContent)
315 throws IOException {
316 boolean result = false;
317
318
319
320 if (rowContent.getValue(HConstants.CATALOG_FAMILY,
321 HConstants.MERGEA_QUALIFIER) != null) {
322
323 return result;
324 }
325
326 PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
327 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
328 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
329 if (hasNoReferences(a) && hasNoReferences(b)) {
330 LOG.debug("Deleting region " + parent.getRegionNameAsString() +
331 " because daughter splits no longer hold references");
332 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
333 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
334 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
335 MetaTableAccessor.deleteRegion(this.connection, parent);
336 result = true;
337 }
338 return result;
339 }
340
341
342
343
344
345
346
347 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
348 return !p.getFirst() || !p.getSecond();
349 }
350
351
352
353
354
355
356
357
358
359
360
361 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
362 throws IOException {
363 if (daughter == null) {
364 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
365 }
366
367 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
368 Path rootdir = this.services.getMasterFileSystem().getRootDir();
369 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
370
371 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
372
373 HRegionFileSystem regionFs = null;
374
375 try {
376 if (!FSUtils.isExists(fs, daughterRegionDir)) {
377 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
378 }
379 } catch (IOException ioe) {
380 LOG.warn("Error trying to determine if daughter region exists, " +
381 "assuming exists and has references", ioe);
382 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
383 }
384
385 try {
386 regionFs = HRegionFileSystem.openRegionFromFileSystem(
387 this.services.getConfiguration(), fs, tabledir, daughter, true);
388 } catch (IOException e) {
389 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
390 + ", to: " + parent.getEncodedName() + " assuming has references", e);
391 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
392 }
393
394 boolean references = false;
395 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
396 for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
397 if ((references = regionFs.hasReferences(family.getNameAsString()))) {
398 break;
399 }
400 }
401 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
402 }
403
404 private HTableDescriptor getTableDescriptor(final TableName tableName)
405 throws FileNotFoundException, IOException {
406 return this.services.getTableDescriptors().get(tableName);
407 }
408
409
410
411
412
413
414
415
416 public boolean cleanMergeQualifier(final HRegionInfo region)
417 throws IOException {
418
419
420 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
421 .getRegionsFromMergeQualifier(this.services.getConnection(),
422 region.getRegionName());
423 if (mergeRegions == null
424 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
425
426 return true;
427 }
428
429 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
430 LOG.error("Merged region " + region.getRegionNameAsString()
431 + " has only one merge qualifier in META.");
432 return false;
433 }
434 return cleanMergeRegion(region, mergeRegions.getFirst(),
435 mergeRegions.getSecond());
436 }
437 }