1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.Comparator;
24 import java.util.HashSet;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicInteger;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.MetaTableAccessor;
39 import org.apache.hadoop.hbase.ScheduledChore;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.backup.HFileArchiver;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.client.Connection;
45 import org.apache.hadoop.hbase.client.MetaScanner;
46 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Pair;
52 import org.apache.hadoop.hbase.util.PairOfSameType;
53 import org.apache.hadoop.hbase.util.Threads;
54 import org.apache.hadoop.hbase.util.Triple;
55
56
57
58
59
60 @InterfaceAudience.Private
61 public class CatalogJanitor extends ScheduledChore {
62 private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
63 private final Server server;
64 private final MasterServices services;
65 private AtomicBoolean enabled = new AtomicBoolean(true);
66 private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
67 private final Connection connection;
68
69 CatalogJanitor(final Server server, final MasterServices services) {
70 super("CatalogJanitor-" + server.getServerName().toShortString(), server, server
71 .getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
72 this.server = server;
73 this.services = services;
74 this.connection = server.getConnection();
75 }
76
77 @Override
78 protected boolean initialChore() {
79 try {
80 if (this.enabled.get()) scan();
81 } catch (IOException e) {
82 LOG.warn("Failed initial scan of catalog table", e);
83 return false;
84 }
85 return true;
86 }
87
88
89
90
91 public boolean setEnabled(final boolean enabled) {
92 boolean alreadyEnabled = this.enabled.getAndSet(enabled);
93
94
95
96
97 if (!enabled && alreadyEnabled) {
98 while (alreadyRunning.get()) {
99 Threads.sleepWithoutInterrupt(100);
100 }
101 }
102 return alreadyEnabled;
103 }
104
105 boolean getEnabled() {
106 return this.enabled.get();
107 }
108
109 @Override
110 protected void chore() {
111 try {
112 AssignmentManager am = this.services.getAssignmentManager();
113 if (this.enabled.get()
114 && !this.services.isInMaintenanceMode()
115 && am != null
116 && am.isFailoverCleanupDone()
117 && am.getRegionStates().getRegionsInTransition().size() == 0) {
118 scan();
119 } else {
120 LOG.warn("CatalogJanitor disabled! Not running scan.");
121 }
122 } catch (IOException e) {
123 LOG.warn("Failed scan of catalog table", e);
124 }
125 }
126
127
128
129
130
131
132
133
134 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents()
135 throws IOException {
136 return getMergedRegionsAndSplitParents(null);
137 }
138
139
140
141
142
143
144
145
146
147
148
149 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents(
150 final TableName tableName) throws IOException {
151 final boolean isTableSpecified = (tableName != null);
152
153 final AtomicInteger count = new AtomicInteger(0);
154
155
156 final Map<HRegionInfo, Result> splitParents =
157 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
158 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
159
160
161 MetaScannerVisitor visitor = new MetaScanner.MetaScannerVisitorBase() {
162 @Override
163 public boolean processRow(Result r) throws IOException {
164 if (r == null || r.isEmpty()) return true;
165 count.incrementAndGet();
166 HRegionInfo info = HRegionInfo.getHRegionInfo(r);
167 if (info == null) return true;
168 if (isTableSpecified
169 && info.getTable().compareTo(tableName) > 0) {
170
171 return false;
172 }
173 if (info.isSplitParent()) splitParents.put(info, r);
174 if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
175 mergedRegions.put(info, r);
176 }
177
178 return true;
179 }
180 };
181
182
183
184 MetaScanner.metaScan(this.connection, visitor, tableName);
185
186 return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
187 count.get(), mergedRegions, splitParents);
188 }
189
190
191
192
193
194
195
196
197
198
199
200 boolean cleanMergeRegion(final HRegionInfo mergedRegion,
201 final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
202 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
203 Path rootdir = this.services.getMasterFileSystem().getRootDir();
204 Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
205 HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
206 HRegionFileSystem regionFs = null;
207 try {
208 regionFs = HRegionFileSystem.openRegionFromFileSystem(
209 this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
210 } catch (IOException e) {
211 LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
212 }
213 if (regionFs == null || !regionFs.hasReferences(htd)) {
214 LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
215 + regionB.getRegionNameAsString()
216 + " from fs because merged region no longer holds references");
217 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
218 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
219 MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
220 services.getServerManager().removeRegion(regionA);
221 services.getServerManager().removeRegion(regionB);
222 return true;
223 }
224 return false;
225 }
226
227
228
229
230
231
232
233 int scan() throws IOException {
234 try {
235 if (!alreadyRunning.compareAndSet(false, true)) {
236 return 0;
237 }
238 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
239 getMergedRegionsAndSplitParents();
240 int count = scanTriple.getFirst();
241
242
243
244 int mergeCleaned = 0;
245 Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
246 for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
247 if (this.services.isInMaintenanceMode()) {
248
249 break;
250 }
251
252 HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(),
253 HConstants.MERGEA_QUALIFIER);
254 HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(),
255 HConstants.MERGEB_QUALIFIER);
256 if (regionA == null || regionB == null) {
257 LOG.warn("Unexpected references regionA="
258 + (regionA == null ? "null" : regionA.getRegionNameAsString())
259 + ",regionB="
260 + (regionB == null ? "null" : regionB.getRegionNameAsString())
261 + " in merged region " + e.getKey().getRegionNameAsString());
262 } else {
263 if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
264 mergeCleaned++;
265 }
266 }
267 }
268
269
270
271 Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
272
273
274 int splitCleaned = 0;
275
276 HashSet<String> parentNotCleaned = new HashSet<String>();
277 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
278 if (this.services.isInMaintenanceMode()) {
279
280 break;
281 }
282
283 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
284 cleanParent(e.getKey(), e.getValue())) {
285 splitCleaned++;
286 } else {
287
288 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue());
289 parentNotCleaned.add(daughters.getFirst().getEncodedName());
290 parentNotCleaned.add(daughters.getSecond().getEncodedName());
291 }
292 }
293 if ((mergeCleaned + splitCleaned) != 0) {
294 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
295 + " unreferenced merged region(s) and " + splitCleaned
296 + " unreferenced parent region(s)");
297 } else if (LOG.isTraceEnabled()) {
298 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
299 + " unreferenced merged region(s) and " + splitCleaned
300 + " unreferenced parent region(s)");
301 }
302 return mergeCleaned + splitCleaned;
303 } finally {
304 alreadyRunning.set(false);
305 }
306 }
307
308
309
310
311
312 static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
313 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
314 @Override
315 public int compare(HRegionInfo left, HRegionInfo right) {
316
317
318 if (left == null) return -1;
319 if (right == null) return 1;
320
321 int result = left.getTable().compareTo(right.getTable());
322 if (result != 0) return result;
323
324 result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
325 if (result != 0) return result;
326
327 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
328
329 return result;
330 }
331 }
332
333
334
335
336
337
338
339
340
341
342 boolean cleanParent(final HRegionInfo parent, Result rowContent)
343 throws IOException {
344 boolean result = false;
345
346
347
348 if (rowContent.getValue(HConstants.CATALOG_FAMILY,
349 HConstants.MERGEA_QUALIFIER) != null) {
350
351 return result;
352 }
353
354 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent);
355 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
356 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
357 if (hasNoReferences(a) && hasNoReferences(b)) {
358 LOG.debug("Deleting region " + parent.getRegionNameAsString() +
359 " because daughter splits no longer hold references");
360 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
361 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
362 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
363 MetaTableAccessor.deleteRegion(this.connection, parent);
364 services.getServerManager().removeRegion(parent);
365 result = true;
366 }
367 return result;
368 }
369
370
371
372
373
374
375
376 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
377 return !p.getFirst() || !p.getSecond();
378 }
379
380
381
382
383
384
385
386
387
388
389
390 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
391 throws IOException {
392 if (daughter == null) {
393 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
394 }
395
396 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
397 Path rootdir = this.services.getMasterFileSystem().getRootDir();
398 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
399
400 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
401
402 HRegionFileSystem regionFs = null;
403
404 try {
405 if (!FSUtils.isExists(fs, daughterRegionDir)) {
406 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
407 }
408 } catch (IOException ioe) {
409 LOG.warn("Error trying to determine if daughter region exists, " +
410 "assuming exists and has references", ioe);
411 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
412 }
413
414 try {
415 regionFs = HRegionFileSystem.openRegionFromFileSystem(
416 this.services.getConfiguration(), fs, tabledir, daughter, true);
417 } catch (IOException e) {
418 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
419 + ", to: " + parent.getEncodedName() + " assuming has references", e);
420 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
421 }
422
423 boolean references = false;
424 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
425 for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
426 if ((references = regionFs.hasReferences(family.getNameAsString()))) {
427 break;
428 }
429 }
430 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
431 }
432
433 private HTableDescriptor getTableDescriptor(final TableName tableName)
434 throws FileNotFoundException, IOException {
435 return this.services.getTableDescriptors().get(tableName);
436 }
437
438
439
440
441
442
443
444
445 public boolean cleanMergeQualifier(final HRegionInfo region)
446 throws IOException {
447
448
449 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
450 .getRegionsFromMergeQualifier(this.services.getConnection(),
451 region.getRegionName());
452 if (mergeRegions == null
453 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
454
455 return true;
456 }
457
458 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
459 LOG.error("Merged region " + region.getRegionNameAsString()
460 + " has only one merge qualifier in META.");
461 return false;
462 }
463 return cleanMergeRegion(region, mergeRegions.getFirst(),
464 mergeRegions.getSecond());
465 }
466 }