View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to you under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.hadoop.hbase.quotas;
18  
19  import java.io.IOException;
20  import java.util.Collections;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Objects;
25  import java.util.Set;
26  import java.util.concurrent.ConcurrentHashMap;
27  import java.util.concurrent.TimeUnit;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.ScheduledChore;
34  import org.apache.hadoop.hbase.Stoppable;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.client.Connection;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.master.HMaster;
39  import org.apache.hadoop.hbase.quotas.QuotaSnapshotStore.ViolationState;
40  import org.apache.hadoop.hbase.master.MetricsMaster;
41  import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
42  import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
43  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
44  import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceQuota;
45  
46  import com.google.common.annotations.VisibleForTesting;
47  import com.google.common.collect.HashMultimap;
48  import com.google.common.collect.Iterables;
49  import com.google.common.collect.Multimap;
50  
51  /**
52   * Reads the currently received Region filesystem-space use reports and acts on those which
53   * violate a defined quota.
54   */
55  public class QuotaObserverChore extends ScheduledChore {
56    private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
57    static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
58        "hbase.master.quotas.observer.chore.period";
59    static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1 minutes in millis
60  
61    static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
62        "hbase.master.quotas.observer.chore.delay";
63    static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15 seconds in millis
64  
65    static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
66        "hbase.master.quotas.observer.chore.timeunit";
67    static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT = TimeUnit.MILLISECONDS.name();
68  
69    static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
70        "hbase.master.quotas.observer.report.percent";
71    static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
72  
73    static final String REGION_REPORT_RETENTION_DURATION_KEY = 
74        "hbase.master.quotas.region.report.retention.millis";
75    static final long REGION_REPORT_RETENTION_DURATION_DEFAULT =
76        1000 * 60 * 10; // 10 minutes
77  
78  
79    private final Connection conn;
80    private final Configuration conf;
81    private final MasterQuotaManager quotaManager;
82    private final MetricsMaster metrics;
83    /*
84     * Callback that changes in quota violation are passed to.
85     */
86    private final SpaceQuotaSnapshotNotifier snapshotNotifier;
87  
88    /*
89     * Preserves the state of quota violations for tables and namespaces
90     */
91    private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
92    private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
93    private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
94    private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
95  
96    // The time, in millis, that region reports should be kept by the master
97    private final long regionReportLifetimeMillis;
98  
99    /*
100    * Encapsulates logic for moving tables/namespaces into or out of quota violation
101    */
102   private QuotaSnapshotStore<TableName> tableSnapshotStore;
103   private QuotaSnapshotStore<String> namespaceSnapshotStore;
104 
105   public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
106     this(
107         master.getConnection(), master.getConfiguration(),
108         master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
109         master, metrics);
110   }
111 
112   QuotaObserverChore(
113       Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
114       MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
115     super(
116         QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
117         getInitialDelay(conf), getTimeUnit(conf));
118     this.conn = conn;
119     this.conf = conf;
120     this.metrics = metrics;
121     this.quotaManager = quotaManager;
122     this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
123     this.tableQuotaSnapshots = new ConcurrentHashMap<>();
124     this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
125     this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
126     this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
127     this.regionReportLifetimeMillis = conf.getLong(
128         REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
129   }
130 
131   @Override
132   protected void chore() {
133     try {
134       if (LOG.isTraceEnabled()) {
135         LOG.trace("Refreshing space quotas in RegionServer");
136       }
137       long start = System.nanoTime();
138       _chore();
139       if (metrics != null) {
140         metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
141       }
142     } catch (IOException e) {
143       LOG.warn("Failed to process quota reports and update quota violation state. Will retry.", e);
144     }
145   }
146 
147   void _chore() throws IOException {
148     // Get the total set of tables that have quotas defined. Includes table quotas
149     // and tables included by namespace quotas.
150     TablesWithQuotas tablesWithQuotas = fetchAllTablesWithQuotasDefined();
151     if (LOG.isTraceEnabled()) {
152       LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
153     }
154 
155     if (metrics != null) {
156       // Set the number of namespaces and tables with quotas defined
157       metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
158           + tablesWithQuotas.getNamespacesWithQuotas().size());
159     }
160 
161     // The current "view" of region space use. Used henceforth.
162     final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
163     if (LOG.isTraceEnabled()) {
164       LOG.trace("Using " + reportedRegionSpaceUse.size() + " region space use reports");
165     }
166 
167     // Remove the "old" region reports
168     pruneOldRegionReports();
169 
170     // Create the stores to track table and namespace snapshots
171     initializeSnapshotStores(reportedRegionSpaceUse);
172     // Report the number of (non-expired) region size reports
173     if (metrics != null) {
174       metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
175     }
176 
177     // Filter out tables for which we don't have adequate regionspace reports yet.
178     // Important that we do this after we instantiate the stores above.
179     // This gives us a set of Tables which may or may not be violating their quota.
180     // To be save, we want to make sure that these are not in violation.
181     Set<TableName> tablesInLimbo = tablesWithQuotas.filterInsufficientlyReportedTables(
182         tableSnapshotStore);
183 
184     if (LOG.isTraceEnabled()) {
185       LOG.trace("Filtered insufficiently reported tables, left with " +
186           reportedRegionSpaceUse.size() + " regions reported");
187     }
188 
189     for (TableName tableInLimbo : tablesInLimbo) {
190       final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(tableInLimbo);
191       if (currentSnapshot.getQuotaStatus().isInViolation()) {
192         if (LOG.isTraceEnabled()) {
193           LOG.trace("Moving " + tableInLimbo + " out of violation because fewer region sizes were"
194               + " reported than required.");
195         }
196         SpaceQuotaSnapshot targetSnapshot = new SpaceQuotaSnapshot(
197             SpaceQuotaStatus.notInViolation(), currentSnapshot.getUsage(),
198             currentSnapshot.getLimit());
199         this.snapshotNotifier.transitionTable(tableInLimbo, targetSnapshot);
200         // Update it in the Table QuotaStore so that memory is consistent with no violation.
201         tableSnapshotStore.setCurrentState(tableInLimbo, targetSnapshot);
202       }
203     }
204 
205     // Transition each table to/from quota violation based on the current and target state.
206     // Only table quotas are enacted.
207     final Set<TableName> tablesWithTableQuotas = tablesWithQuotas.getTableQuotaTables();
208     processTablesWithQuotas(tablesWithTableQuotas);
209 
210     // For each Namespace quota, transition each table in the namespace in or out of violation
211     // only if a table quota violation policy has not already been applied.
212     final Set<String> namespacesWithQuotas = tablesWithQuotas.getNamespacesWithQuotas();
213     final Multimap<String,TableName> tablesByNamespace = tablesWithQuotas.getTablesByNamespace();
214     processNamespacesWithQuotas(namespacesWithQuotas, tablesByNamespace);
215   }
216 
217   void initializeSnapshotStores(Map<HRegionInfo,Long> regionSizes) {
218     Map<HRegionInfo,Long> immutableRegionSpaceUse = Collections.unmodifiableMap(regionSizes);
219     if (tableSnapshotStore == null) {
220       tableSnapshotStore = new TableQuotaSnapshotStore(conn, this, immutableRegionSpaceUse);
221     } else {
222       tableSnapshotStore.setRegionUsage(immutableRegionSpaceUse);
223     }
224     if (namespaceSnapshotStore == null) {
225       namespaceSnapshotStore = new NamespaceQuotaSnapshotStore(
226           conn, this, immutableRegionSpaceUse);
227     } else {
228       namespaceSnapshotStore.setRegionUsage(immutableRegionSpaceUse);
229     }
230   }
231 
232   /**
233    * Processes each {@code TableName} which has a quota defined and moves it in or out of
234    * violation based on the space use.
235    *
236    * @param tablesWithTableQuotas The HBase tables which have quotas defined
237    */
238   void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
239     long numTablesInViolation = 0L;
240     for (TableName table : tablesWithTableQuotas) {
241       final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
242       if (spaceQuota == null) {
243         if (LOG.isDebugEnabled()) {
244           LOG.debug("Unexpectedly did not find a space quota for " + table
245               + ", maybe it was recently deleted.");
246         }
247         continue;
248       }
249       final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
250       final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
251       if (LOG.isTraceEnabled()) {
252         LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
253             + targetSnapshot);
254       }
255       updateTableQuota(table, currentSnapshot, targetSnapshot);
256 
257       if (targetSnapshot.getQuotaStatus().isInViolation()) {
258         numTablesInViolation++;
259       }
260     }
261     // Report the number of tables in violation
262     if (metrics != null) {
263       metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
264     }
265   }
266 
267   /**
268    * Processes each namespace which has a quota defined and moves all of the tables contained
269    * in that namespace into or out of violation of the quota. Tables which are already in
270    * violation of a quota at the table level which <em>also</em> have a reside in a namespace
271    * with a violated quota will not have the namespace quota enacted. The table quota takes
272    * priority over the namespace quota.
273    *
274    * @param namespacesWithQuotas The set of namespaces that have quotas defined
275    * @param tablesByNamespace A mapping of namespaces and the tables contained in those namespaces
276    */
277   void processNamespacesWithQuotas(
278       final Set<String> namespacesWithQuotas,
279       final Multimap<String,TableName> tablesByNamespace) throws IOException {
280     long numNamespacesInViolation = 0L;
281     for (String namespace : namespacesWithQuotas) {
282       // Get the quota definition for the namespace
283       final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
284       if (spaceQuota == null) {
285         if (LOG.isDebugEnabled()) {
286           LOG.debug("Could not get Namespace space quota for " + namespace
287               + ", maybe it was recently deleted.");
288         }
289         continue;
290       }
291       final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
292       final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
293           namespace, spaceQuota);
294       if (LOG.isTraceEnabled()) {
295         LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
296             + targetSnapshot);
297       }
298       updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
299 
300       if (targetSnapshot.getQuotaStatus().isInViolation()) {
301         numNamespacesInViolation++;
302       }
303     }
304 
305     // Report the number of namespaces in violation
306     if (metrics != null) {
307       metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
308     }
309   }
310 
311   /**
312    * Updates the hbase:quota table with the new quota policy for this <code>table</code>
313    * if necessary.
314    *
315    * @param table The table being checked
316    * @param currentSnapshot The state of the quota on this table from the previous invocation.
317    * @param targetSnapshot The state the quota should be in for this table.
318    */
319   void updateTableQuota(
320       TableName table, SpaceQuotaSnapshot currentSnapshot, SpaceQuotaSnapshot targetSnapshot)
321           throws IOException {
322     final SpaceQuotaStatus currentStatus = currentSnapshot.getQuotaStatus();
323     final SpaceQuotaStatus targetStatus = targetSnapshot.getQuotaStatus();
324 
325     // If we're changing something, log it.
326     if (!currentSnapshot.equals(targetSnapshot)) {
327       // If the target is none, we're moving out of violation. Update the hbase:quota table
328       if (!targetStatus.isInViolation()) {
329         if (LOG.isDebugEnabled()) {
330           LOG.debug(table + " moving into observance of table space quota.");
331         }
332       } else if (LOG.isDebugEnabled()) {
333         // We're either moving into violation or changing violation policies
334         LOG.debug(table + " moving into violation of table space quota with policy of "
335             + targetStatus.getPolicy());
336       }
337 
338       this.snapshotNotifier.transitionTable(table, targetSnapshot);
339       // Update it in memory
340       tableSnapshotStore.setCurrentState(table, targetSnapshot);
341     } else if (LOG.isTraceEnabled()) {
342       // Policies are the same, so we have nothing to do except log this. Don't need to re-update
343       // the quota table
344       if (!currentStatus.isInViolation()) {
345         LOG.trace(table + " remains in observance of quota.");
346       } else {
347         LOG.trace(table + " remains in violation of quota.");
348       }
349     }
350   }
351       
352   /**
353    * Updates the hbase:quota table with the target quota policy for this <code>namespace</code>
354    * if necessary.
355    *
356    * @param namespace The namespace being checked
357    * @param currentSnapshot The state of the quota on this namespace from the previous invocation
358    * @param targetSnapshot The state the quota should be in for this namespace
359    * @param tablesByNamespace A mapping of tables in namespaces.
360    */
361   void updateNamespaceQuota(
362       String namespace, SpaceQuotaSnapshot currentSnapshot, SpaceQuotaSnapshot targetSnapshot,
363       final Multimap<String,TableName> tablesByNamespace) throws IOException {
364     final SpaceQuotaStatus targetStatus = targetSnapshot.getQuotaStatus();
365 
366     // When the policies differ, we need to move into or out of violatino
367     if (!currentSnapshot.equals(targetSnapshot)) {
368       // We want to have a policy of "NONE", moving out of violation
369       if (!targetStatus.isInViolation()) {
370         for (TableName tableInNS : tablesByNamespace.get(namespace)) {
371           // If there is a quota on this table in violation
372           if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
373             // Table-level quota violation policy is being applied here.
374             if (LOG.isTraceEnabled()) {
375               LOG.trace("Not activating Namespace violation policy because a Table violation"
376                   + " policy is already in effect for " + tableInNS);
377             }
378           } else {
379             LOG.info(tableInNS + " moving into observance of namespace space quota");
380             this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
381           }
382         }
383       // Want to move into violation at the NS level
384       } else {
385         // Moving tables in the namespace into violation or to a different violation policy
386         for (TableName tableInNS : tablesByNamespace.get(namespace)) {
387           final SpaceQuotaSnapshot tableQuotaSnapshot =
388               tableSnapshotStore.getCurrentState(tableInNS);
389           final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA != tableQuotaSnapshot;
390           if (hasTableQuota && tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
391             // Table-level quota violation policy is being applied here.
392             if (LOG.isTraceEnabled()) {
393               LOG.trace("Not activating Namespace violation policy because a Table violation"
394                   + " policy is already in effect for " + tableInNS);
395             }
396           } else {
397             // No table quota present or a table quota present that is not in violation
398             LOG.info(tableInNS + " moving into violation of namespace space quota with policy "
399                 + targetStatus.getPolicy());
400             this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
401           }
402         }
403       }
404       // Update the new state in memory for this namespace
405       namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
406     } else {
407       // Policies are the same
408       if (!targetStatus.isInViolation()) {
409         // Both are NONE, so we remain in observance
410         if (LOG.isTraceEnabled()) {
411           LOG.trace(namespace + " remains in observance of quota.");
412         }
413       } else {
414         // Namespace quota is still in violation, need to enact if the table quota is not
415         // taking priority.
416         for (TableName tableInNS : tablesByNamespace.get(namespace)) {
417           // Does a table policy exist
418           if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
419             // Table-level quota violation policy is being applied here.
420             if (LOG.isTraceEnabled()) {
421               LOG.trace("Not activating Namespace violation policy because Table violation"
422                   + " policy is already in effect for " + tableInNS);
423             }
424           } else {
425             // No table policy, so enact namespace policy
426             LOG.info(tableInNS + " moving into violation of namespace space quota");
427             this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
428           }
429         }
430       }
431     }
432   }
433 
434   /**
435    * Removes region reports over a certain age.
436    */
437   void pruneOldRegionReports() {
438     long now = EnvironmentEdgeManager.currentTime();
439     long pruneTime = now - regionReportLifetimeMillis;
440     if (LOG.isTraceEnabled()) {
441       LOG.trace("Pruning Region size reports older than " + pruneTime);
442     }
443     int numRemoved = quotaManager.pruneEntriesOlderThan(pruneTime);
444     if (LOG.isTraceEnabled()) {
445       LOG.trace("Removed " + numRemoved + " old region size reports.");
446     }
447   }
448 
449   void initializeViolationStores(Map<HRegionInfo,Long> regionSizes) {
450     Map<HRegionInfo,Long> immutableRegionSpaceUse = Collections.unmodifiableMap(regionSizes);
451     tableSnapshotStore = new TableQuotaSnapshotStore(conn, this, immutableRegionSpaceUse);
452     namespaceSnapshotStore = new NamespaceQuotaSnapshotStore(conn, this, immutableRegionSpaceUse);
453   }
454 
455   /**
456    * Computes the set of all tables that have quotas defined. This includes tables with quotas
457    * explicitly set on them, in addition to tables that exist namespaces which have a quota
458    * defined.
459    */
460   TablesWithQuotas fetchAllTablesWithQuotasDefined() throws IOException {
461     final Scan scan = QuotaTableUtil.makeScan(null);
462     final TablesWithQuotas tablesWithQuotas = new TablesWithQuotas(conn, conf);
463     try (final QuotaRetriever scanner = new QuotaRetriever()) {
464       scanner.init(conn, scan);
465       for (QuotaSettings quotaSettings : scanner) {
466         // Only one of namespace and tablename should be 'null'
467         final String namespace = quotaSettings.getNamespace();
468         final TableName tableName = quotaSettings.getTableName();
469         if (QuotaType.SPACE != quotaSettings.getQuotaType()) {
470           continue;
471         }
472 
473         if (namespace != null) {
474           assert tableName == null;
475           // Collect all of the tables in the namespace
476           TableName[] tablesInNS = conn.getAdmin().listTableNamesByNamespace(namespace);
477           for (TableName tableUnderNs : tablesInNS) {
478             if (LOG.isTraceEnabled()) {
479               LOG.trace("Adding " + tableUnderNs + " under " +  namespace
480                   + " as having a namespace quota");
481             }
482             tablesWithQuotas.addNamespaceQuotaTable(tableUnderNs);
483           }
484         } else {
485           assert tableName != null;
486           if (LOG.isTraceEnabled()) {
487             LOG.trace("Adding " + tableName + " as having table quota.");
488           }
489           // namespace is already null, must be a non-null tableName
490           tablesWithQuotas.addTableQuotaTable(tableName);
491         }
492       }
493       return tablesWithQuotas;
494     }
495   }
496 
497   @VisibleForTesting
498   QuotaSnapshotStore<TableName> getTableSnapshotStore() {
499     return tableSnapshotStore;
500   }
501 
502   @VisibleForTesting
503   QuotaSnapshotStore<String> getNamespaceSnapshotStore() {
504     return namespaceSnapshotStore;
505   }
506 
507   /**
508    * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
509    * for each HBase table with a quota defined.
510    */
511   public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
512     return readOnlyTableQuotaSnapshots;
513   }
514 
515   /**
516    * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
517    * for each HBase namespace with a quota defined.
518    */
519   public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
520     return readOnlyNamespaceSnapshots;
521   }
522 
523   /**
524    * Fetches the {@link SpaceQuotaSnapshot} for the given table.
525    */
526   SpaceQuotaSnapshot getTableQuotaSnapshot(TableName table) {
527     SpaceQuotaSnapshot state = this.tableQuotaSnapshots.get(table);
528     if (state == null) {
529       // No tracked state implies observance.
530       return QuotaSnapshotStore.NO_QUOTA;
531     }
532     return state;
533   }
534 
535   /**
536    * Stores the quota state for the given table.
537    */
538   void setTableQuotaSnapshot(TableName table, SpaceQuotaSnapshot snapshot) {
539     this.tableQuotaSnapshots.put(table, snapshot);
540   }
541 
542   /**
543    * Fetches the {@link SpaceQuotaSnapshot} for the given namespace from this chore.
544    */
545   SpaceQuotaSnapshot getNamespaceQuotaSnapshot(String namespace) {
546     SpaceQuotaSnapshot state = this.namespaceQuotaSnapshots.get(namespace);
547     if (state == null) {
548       // No tracked state implies observance.
549       return QuotaSnapshotStore.NO_QUOTA;
550     }
551     return state;
552   }
553 
554   /**
555    * Stores the given {@code snapshot} for the given {@code namespace} in this chore.
556    */
557   void setNamespaceQuotaViolation(String namespace, SpaceQuotaSnapshot snapshot) {
558     this.namespaceQuotaSnapshots.put(namespace, snapshot);
559   }
560 
561   /**
562    * Extracts the period for the chore from the configuration.
563    *
564    * @param conf The configuration object.
565    * @return The configured chore period or the default value in the given timeunit.
566    * @see #getTimeUnit(Configuration)
567    */
568   static int getPeriod(Configuration conf) {
569     return conf.getInt(QUOTA_OBSERVER_CHORE_PERIOD_KEY,
570         QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT);
571   }
572 
573   /**
574    * Extracts the initial delay for the chore from the configuration.
575    *
576    * @param conf The configuration object.
577    * @return The configured chore initial delay or the default value in the given timeunit.
578    * @see #getTimeUnit(Configuration)
579    */
580   static long getInitialDelay(Configuration conf) {
581     return conf.getLong(QUOTA_OBSERVER_CHORE_DELAY_KEY,
582         QUOTA_OBSERVER_CHORE_DELAY_DEFAULT);
583   }
584 
585   /**
586    * Extracts the time unit for the chore period and initial delay from the configuration. The
587    * configuration value for {@link #QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY} must correspond to
588    * a {@link TimeUnit} value.
589    *
590    * @param conf The configuration object.
591    * @return The configured time unit for the chore period and initial delay or the default value.
592    */
593   static TimeUnit getTimeUnit(Configuration conf) {
594     return TimeUnit.valueOf(conf.get(QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY,
595         QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT));
596   }
597 
598   /**
599    * Extracts the percent of Regions for a table to have been reported to enable quota violation
600    * state change.
601    *
602    * @param conf The configuration object.
603    * @return The percent of regions reported to use.
604    */
605   static Double getRegionReportPercent(Configuration conf) {
606     return conf.getDouble(QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY,
607         QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT);
608   }
609 
610   /**
611    * A container which encapsulates the tables that have either a table quota or are contained in a
612    * namespace which have a namespace quota.
613    */
614   static class TablesWithQuotas {
615     private final Set<TableName> tablesWithTableQuotas = new HashSet<>();
616     private final Set<TableName> tablesWithNamespaceQuotas = new HashSet<>();
617     private final Connection conn;
618     private final Configuration conf;
619 
620     public TablesWithQuotas(Connection conn, Configuration conf) {
621       this.conn = Objects.requireNonNull(conn);
622       this.conf = Objects.requireNonNull(conf);
623     }
624 
625     Configuration getConfiguration() {
626       return conf;
627     }
628 
629     /**
630      * Adds a table with a table quota.
631      */
632     public void addTableQuotaTable(TableName tn) {
633       tablesWithTableQuotas.add(tn);
634     }
635 
636     /**
637      * Adds a table with a namespace quota.
638      */
639     public void addNamespaceQuotaTable(TableName tn) {
640       tablesWithNamespaceQuotas.add(tn);
641     }
642 
643     /**
644      * Returns true if the given table has a table quota.
645      */
646     public boolean hasTableQuota(TableName tn) {
647       return tablesWithTableQuotas.contains(tn);
648     }
649 
650     /**
651      * Returns true if the table exists in a namespace with a namespace quota.
652      */
653     public boolean hasNamespaceQuota(TableName tn) {
654       return tablesWithNamespaceQuotas.contains(tn);
655     }
656 
657     /**
658      * Returns an unmodifiable view of all tables with table quotas.
659      */
660     public Set<TableName> getTableQuotaTables() {
661       return Collections.unmodifiableSet(tablesWithTableQuotas);
662     }
663 
664     /**
665      * Returns an unmodifiable view of all tables in namespaces that have
666      * namespace quotas.
667      */
668     public Set<TableName> getNamespaceQuotaTables() {
669       return Collections.unmodifiableSet(tablesWithNamespaceQuotas);
670     }
671 
672     public Set<String> getNamespacesWithQuotas() {
673       Set<String> namespaces = new HashSet<>();
674       for (TableName tn : tablesWithNamespaceQuotas) {
675         namespaces.add(tn.getNamespaceAsString());
676       }
677       return namespaces;
678     }
679 
680     /**
681      * Returns a view of all tables that reside in a namespace with a namespace
682      * quota, grouped by the namespace itself.
683      */
684     public Multimap<String,TableName> getTablesByNamespace() {
685       Multimap<String,TableName> tablesByNS = HashMultimap.create();
686       for (TableName tn : tablesWithNamespaceQuotas) {
687         tablesByNS.put(tn.getNamespaceAsString(), tn);
688       }
689       return tablesByNS;
690     }
691 
692     /**
693      * Filters out all tables for which the Master currently doesn't have enough region space
694      * reports received from RegionServers yet.
695      */
696     public Set<TableName> filterInsufficientlyReportedTables(QuotaSnapshotStore<TableName> tableStore)
697         throws IOException {
698       final double percentRegionsReportedThreshold = getRegionReportPercent(getConfiguration());
699       Set<TableName> tablesToRemove = new HashSet<>();
700       for (TableName table : Iterables.concat(tablesWithTableQuotas, tablesWithNamespaceQuotas)) {
701         // Don't recompute a table we've already computed
702         if (tablesToRemove.contains(table)) {
703           continue;
704         }
705         final int numRegionsInTable = getNumRegions(table);
706         // If the table doesn't exist (no regions), bail out.
707         if (numRegionsInTable == 0) {
708           if (LOG.isTraceEnabled()) {
709             LOG.trace("Filtering " + table + " because no regions were reported.");
710           }
711           tablesToRemove.add(table);
712           continue;
713         }
714         final int reportedRegionsInQuota = getNumReportedRegions(table, tableStore);
715         final double ratioReported = ((double) reportedRegionsInQuota) / numRegionsInTable;
716         if (ratioReported < percentRegionsReportedThreshold) {
717           if (LOG.isTraceEnabled()) {
718             LOG.trace("Filtering " + table + " because " + reportedRegionsInQuota  + " of " +
719                 numRegionsInTable + " regions were reported.");
720           }
721           tablesToRemove.add(table);
722         } else if (LOG.isTraceEnabled()) {
723           LOG.trace("Retaining " + table + " because " + reportedRegionsInQuota  + " of " +
724               numRegionsInTable + " regions were reported.");
725         }
726       }
727       for (TableName tableToRemove : tablesToRemove) {
728         tablesWithTableQuotas.remove(tableToRemove);
729         tablesWithNamespaceQuotas.remove(tableToRemove);
730       }
731       return tablesToRemove;
732     }
733 
734     /**
735      * Computes the total number of regions in a table.
736      */
737     int getNumRegions(TableName table) throws IOException {
738       List<HRegionInfo> regions = this.conn.getAdmin().getTableRegions(table);
739       if (regions == null) {
740         return 0;
741       }
742       return regions.size();
743     }
744 
745     /**
746      * Computes the number of regions reported for a table.
747      */
748     int getNumReportedRegions(TableName table, QuotaSnapshotStore<TableName> tableStore)
749         throws IOException {
750       return Iterables.size(tableStore.filterBySubject(table));
751     }
752 
753     @Override
754     public String toString() {
755       final StringBuilder sb = new StringBuilder(32);
756       sb.append(getClass().getSimpleName())
757           .append(": tablesWithTableQuotas=")
758           .append(this.tablesWithTableQuotas)
759           .append(", tablesWithNamespaceQuotas=")
760           .append(this.tablesWithNamespaceQuotas);
761       return sb.toString();
762     }
763   }
764 }