View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import java.io.IOException;
21  import java.lang.reflect.InvocationTargetException;
22  import java.lang.reflect.Method;
23  import java.util.Map;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.classification.InterfaceStability;
30  import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
31  import org.apache.hadoop.hbase.util.VersionInfo;
32  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
33  
34  /**
35   * Adds HBase configuration files to a Configuration
36   */
37  @InterfaceAudience.Public
38  @InterfaceStability.Stable
39  public class HBaseConfiguration extends Configuration {
40  
41    private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
42  
43    /**
44     * Instantinating HBaseConfiguration() is deprecated. Please use
45     * HBaseConfiguration#create() to construct a plain Configuration
46     */
47    @Deprecated
48    public HBaseConfiguration() {
49      //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration
50      super();
51      addHbaseResources(this);
52      LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use"
53          + " HBaseConfiguration#create() to construct a plain Configuration");
54    }
55  
56    /**
57     * Instantiating HBaseConfiguration() is deprecated. Please use
58     * HBaseConfiguration#create(conf) to construct a plain Configuration
59     */
60    @Deprecated
61    public HBaseConfiguration(final Configuration c) {
62      //TODO:replace with private constructor
63      this();
64      merge(this, c);
65    }
66  
67    private static void checkDefaultsVersion(Configuration conf) {
68      if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
69      String defaultsVersion = conf.get("hbase.defaults.for.version");
70      String thisVersion = VersionInfo.getVersion();
71      if (!thisVersion.equals(defaultsVersion)) {
72        throw new RuntimeException(
73          "hbase-default.xml file seems to be for an older version of HBase (" +
74          defaultsVersion + "), this version is " + thisVersion);
75      }
76    }
77  
78    public static Configuration addHbaseResources(Configuration conf) {
79      conf.addResource("hbase-default.xml");
80      conf.addResource("hbase-site.xml");
81  
82      checkDefaultsVersion(conf);
83      HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
84      return conf;
85    }
86  
87    /**
88     * Creates a Configuration with HBase resources
89     * @return a Configuration with HBase resources
90     */
91    public static Configuration create() {
92      Configuration conf = new Configuration();
93      // In case HBaseConfiguration is loaded from a different classloader than
94      // Configuration, conf needs to be set with appropriate class loader to resolve
95      // HBase resources.
96      conf.setClassLoader(HBaseConfiguration.class.getClassLoader());
97      return addHbaseResources(conf);
98    }
99  
100   /**
101    * @param that Configuration to clone.
102    * @return a Configuration created with the hbase-*.xml files plus
103    * the given configuration.
104    */
105   public static Configuration create(final Configuration that) {
106     Configuration conf = create();
107     merge(conf, that);
108     return conf;
109   }
110 
111   /**
112    * Merge two configurations.
113    * @param destConf the configuration that will be overwritten with items
114    *                 from the srcConf
115    * @param srcConf the source configuration
116    **/
117   public static void merge(Configuration destConf, Configuration srcConf) {
118     for (Map.Entry<String, String> e : srcConf) {
119       destConf.set(e.getKey(), e.getValue());
120     }
121   }
122 
123   /**
124    * Returns a subset of the configuration properties, matching the given key prefix.
125    * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix",
126    * the entry "myprefix.key1 = value1" would be returned as "key1 = value1".  If an entry's
127    * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be
128    * included in the results, since it would show up as an entry with an empty key.
129    */
130   public static Configuration subset(Configuration srcConf, String prefix) {
131     Configuration newConf = new Configuration(false);
132     for (Map.Entry<String, String> entry : srcConf) {
133       if (entry.getKey().startsWith(prefix)) {
134         String newKey = entry.getKey().substring(prefix.length());
135         // avoid entries that would produce an empty key
136         if (!newKey.isEmpty()) {
137           newConf.set(newKey, entry.getValue());
138         }
139       }
140     }
141     return newConf;
142   }
143 
144   /**
145    * Sets all the entries in the provided {@code Map<String, String>} as properties in the
146    * given {@code Configuration}.  Each property will have the specified prefix prepended,
147    * so that the configuration entries are keyed by {@code prefix + entry.getKey()}.
148    */
149   public static void setWithPrefix(Configuration conf, String prefix,
150                                    Iterable<Map.Entry<String, String>> properties) {
151     for (Map.Entry<String, String> entry : properties) {
152       conf.set(prefix + entry.getKey(), entry.getValue());
153     }
154   }
155 
156   /**
157    * @return whether to show HBase Configuration in servlet
158    */
159   public static boolean isShowConfInServlet() {
160     boolean isShowConf = false;
161     try {
162       if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) {
163         isShowConf = true;
164       }
165     } catch (LinkageError e) {
166        // should we handle it more aggressively in addition to log the error?
167        LOG.warn("Error thrown: ", e);
168     } catch (ClassNotFoundException ce) {
169       LOG.debug("ClassNotFound: ConfServlet");
170       // ignore
171     }
172     return isShowConf;
173   }
174 
175   /**
176    * Get the value of the <code>name</code> property as an <code>int</code>, possibly
177    * referring to the deprecated name of the configuration property.
178    * If no such property exists, the provided default value is returned,
179    * or if the specified value is not a valid <code>int</code>,
180    * then an error is thrown.
181    *
182    * @param name property name.
183    * @param deprecatedName a deprecatedName for the property to use
184    * if non-deprecated name is not used
185    * @param defaultValue default value.
186    * @throws NumberFormatException when the value is invalid
187    * @return property value as an <code>int</code>,
188    *         or <code>defaultValue</code>.
189    */
190   // TODO: developer note: This duplicates the functionality of deprecated
191   // property support in Configuration in Hadoop 2. But since Hadoop-1 does not
192   // contain these changes, we will do our own as usual. Replace these when H2 is default.
193   public static int getInt(Configuration conf, String name,
194       String deprecatedName, int defaultValue) {
195     if (conf.get(deprecatedName) != null) {
196       LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\""
197         , deprecatedName, name));
198       return conf.getInt(deprecatedName, defaultValue);
199     } else {
200       return conf.getInt(name, defaultValue);
201     }
202   }
203 
204   /**
205    * Get the password from the Configuration instance using the
206    * getPassword method if it exists. If not, then fall back to the
207    * general get method for configuration elements.
208    * @param conf configuration instance for accessing the passwords
209    * @param alias the name of the password element
210    * @param defPass the default password
211    * @return String password or default password
212    * @throws IOException
213    */
214   public static String getPassword(Configuration conf, String alias,
215       String defPass) throws IOException {
216     String passwd = null;
217     try {
218       Method m = Configuration.class.getMethod("getPassword", String.class);
219       char[] p = (char[]) m.invoke(conf, alias);
220       if (p != null) {
221         LOG.debug(String.format("Config option \"%s\" was found through" +
222             " the Configuration getPassword method.", alias));
223         passwd = new String(p);
224       }
225       else {
226         LOG.debug(String.format(
227             "Config option \"%s\" was not found. Using provided default value",
228             alias));
229         passwd = defPass;
230       }
231     } catch (NoSuchMethodException e) {
232       // this is a version of Hadoop where the credential
233       //provider API doesn't exist yet
234       LOG.debug(String.format(
235           "Credential.getPassword method is not available." +
236           " Falling back to configuration."));
237       passwd = conf.get(alias, defPass);
238     } catch (SecurityException e) {
239       throw new IOException(e.getMessage(), e);
240     } catch (IllegalAccessException e) {
241       throw new IOException(e.getMessage(), e);
242     } catch (IllegalArgumentException e) {
243       throw new IOException(e.getMessage(), e);
244     } catch (InvocationTargetException e) {
245       throw new IOException(e.getMessage(), e);
246     }
247     return passwd;
248   }
249 
250   /**
251    * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key
252    * to the base Configuration.  Note that additional configuration properties may be needed
253    * for a remote cluster, so it is preferable to use
254    * {@link #createClusterConf(Configuration, String, String)}.
255    *
256    * @param baseConf the base configuration to use, containing prefixed override properties
257    * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
258    *
259    * @return the merged configuration with override properties and cluster key applied
260    *
261    * @see #createClusterConf(Configuration, String, String)
262    */
263   public static Configuration createClusterConf(Configuration baseConf, String clusterKey)
264       throws IOException {
265     return createClusterConf(baseConf, clusterKey, null);
266   }
267 
268   /**
269    * Generates a {@link Configuration} instance by applying property overrides prefixed by
270    * a cluster profile key to the base Configuration.  Override properties are extracted by
271    * the {@link #subset(Configuration, String)} method, then the merged on top of the base
272    * Configuration and returned.
273    *
274    * @param baseConf the base configuration to use, containing prefixed override properties
275    * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
276    * @param overridePrefix the property key prefix to match for override properties,
277    *     or {@code null} if none
278    * @return the merged configuration with override properties and cluster key applied
279    */
280   public static Configuration createClusterConf(Configuration baseConf, String clusterKey,
281                                                 String overridePrefix) throws IOException {
282     Configuration clusterConf = HBaseConfiguration.create(baseConf);
283     if (clusterKey != null && !clusterKey.isEmpty()) {
284       applyClusterKeyToConf(clusterConf, clusterKey);
285     }
286 
287     if (overridePrefix != null && !overridePrefix.isEmpty()) {
288       Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix);
289       HBaseConfiguration.merge(clusterConf, clusterSubset);
290     }
291     return clusterConf;
292   }
293 
294   /**
295    * Apply the settings in the given key to the given configuration, this is
296    * used to communicate with distant clusters
297    * @param conf configuration object to configure
298    * @param key string that contains the 3 required configuratins
299    * @throws IOException
300    */
301   private static void applyClusterKeyToConf(Configuration conf, String key)
302       throws IOException{
303     ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key);
304     conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString());
305     conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort());
306     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent());
307   }
308 
309   /**
310    * For debugging.  Dump configurations to system output as xml format.
311    * Master and RS configurations can also be dumped using
312    * http services. e.g. "curl http://master:16010/dump"
313    */
314   public static void main(String[] args) throws Exception {
315     HBaseConfiguration.create().writeXml(System.out);
316   }
317 }