001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.io.IOException; 021import java.lang.reflect.InvocationTargetException; 022import java.lang.reflect.Method; 023import java.util.Map; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.util.VersionInfo; 027import org.apache.hadoop.hbase.zookeeper.ZKConfig; 028import org.apache.yetus.audience.InterfaceAudience; 029import org.slf4j.Logger; 030import org.slf4j.LoggerFactory; 031 032/** 033 * Adds HBase configuration files to a Configuration 034 */ 035@InterfaceAudience.Public 036public class HBaseConfiguration extends Configuration { 037 private static final Logger LOG = LoggerFactory.getLogger(HBaseConfiguration.class); 038 039 /** 040 * Instantiating HBaseConfiguration() is deprecated. Please use 041 * HBaseConfiguration#create() to construct a plain Configuration 042 * @deprecated Please use create() instead. 043 */ 044 @Deprecated 045 public HBaseConfiguration() { 046 //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration 047 super(); 048 addHbaseResources(this); 049 LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use" 050 + " HBaseConfiguration#create() to construct a plain Configuration"); 051 } 052 053 /** 054 * Instantiating HBaseConfiguration() is deprecated. Please use 055 * HBaseConfiguration#create(conf) to construct a plain Configuration 056 * @deprecated Please user create(conf) instead. 057 */ 058 @Deprecated 059 public HBaseConfiguration(final Configuration c) { 060 //TODO:replace with private constructor 061 this(); 062 merge(this, c); 063 } 064 065 private static void checkDefaultsVersion(Configuration conf) { 066 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; 067 String defaultsVersion = conf.get("hbase.defaults.for.version"); 068 String thisVersion = VersionInfo.getVersion(); 069 if (!thisVersion.equals(defaultsVersion)) { 070 throw new RuntimeException( 071 "hbase-default.xml file seems to be for an older version of HBase (" + 072 defaultsVersion + "), this version is " + thisVersion); 073 } 074 } 075 076 public static Configuration addHbaseResources(Configuration conf) { 077 conf.addResource("hbase-default.xml"); 078 conf.addResource("hbase-site.xml"); 079 080 checkDefaultsVersion(conf); 081 return conf; 082 } 083 084 /** 085 * Creates a Configuration with HBase resources 086 * @return a Configuration with HBase resources 087 */ 088 public static Configuration create() { 089 Configuration conf = new Configuration(); 090 // In case HBaseConfiguration is loaded from a different classloader than 091 // Configuration, conf needs to be set with appropriate class loader to resolve 092 // HBase resources. 093 conf.setClassLoader(HBaseConfiguration.class.getClassLoader()); 094 return addHbaseResources(conf); 095 } 096 097 /** 098 * @param that Configuration to clone. 099 * @return a Configuration created with the hbase-*.xml files plus 100 * the given configuration. 101 */ 102 public static Configuration create(final Configuration that) { 103 Configuration conf = create(); 104 merge(conf, that); 105 return conf; 106 } 107 108 /** 109 * Merge two configurations. 110 * @param destConf the configuration that will be overwritten with items 111 * from the srcConf 112 * @param srcConf the source configuration 113 **/ 114 public static void merge(Configuration destConf, Configuration srcConf) { 115 for (Map.Entry<String, String> e : srcConf) { 116 destConf.set(e.getKey(), e.getValue()); 117 } 118 } 119 120 /** 121 * Returns a subset of the configuration properties, matching the given key prefix. 122 * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix", 123 * the entry "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's 124 * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be 125 * included in the results, since it would show up as an entry with an empty key. 126 */ 127 public static Configuration subset(Configuration srcConf, String prefix) { 128 Configuration newConf = new Configuration(false); 129 for (Map.Entry<String, String> entry : srcConf) { 130 if (entry.getKey().startsWith(prefix)) { 131 String newKey = entry.getKey().substring(prefix.length()); 132 // avoid entries that would produce an empty key 133 if (!newKey.isEmpty()) { 134 newConf.set(newKey, entry.getValue()); 135 } 136 } 137 } 138 return newConf; 139 } 140 141 /** 142 * Sets all the entries in the provided {@code Map<String, String>} as properties in the 143 * given {@code Configuration}. Each property will have the specified prefix prepended, 144 * so that the configuration entries are keyed by {@code prefix + entry.getKey()}. 145 */ 146 public static void setWithPrefix(Configuration conf, String prefix, 147 Iterable<Map.Entry<String, String>> properties) { 148 for (Map.Entry<String, String> entry : properties) { 149 conf.set(prefix + entry.getKey(), entry.getValue()); 150 } 151 } 152 153 /** 154 * @return whether to show HBase Configuration in servlet 155 */ 156 public static boolean isShowConfInServlet() { 157 boolean isShowConf = false; 158 try { 159 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { 160 isShowConf = true; 161 } 162 } catch (LinkageError e) { 163 // should we handle it more aggressively in addition to log the error? 164 LOG.warn("Error thrown: ", e); 165 } catch (ClassNotFoundException ce) { 166 LOG.debug("ClassNotFound: ConfServlet"); 167 // ignore 168 } 169 return isShowConf; 170 } 171 172 /** 173 * Get the value of the <code>name</code> property as an <code>int</code>, possibly referring to 174 * the deprecated name of the configuration property. If no such property exists, the provided 175 * default value is returned, or if the specified value is not a valid <code>int</code>, then an 176 * error is thrown. 177 * @param name property name. 178 * @param deprecatedName a deprecatedName for the property to use if non-deprecated name is not 179 * used 180 * @param defaultValue default value. 181 * @throws NumberFormatException when the value is invalid 182 * @return property value as an <code>int</code>, or <code>defaultValue</code>. 183 * @deprecated it will be removed in 3.0.0. Use 184 * {@link Configuration#addDeprecation(String, String)} instead. 185 */ 186 @Deprecated 187 public static int getInt(Configuration conf, String name, 188 String deprecatedName, int defaultValue) { 189 if (conf.get(deprecatedName) != null) { 190 LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\"" 191 , deprecatedName, name)); 192 return conf.getInt(deprecatedName, defaultValue); 193 } else { 194 return conf.getInt(name, defaultValue); 195 } 196 } 197 198 /** 199 * Get the password from the Configuration instance using the 200 * getPassword method if it exists. If not, then fall back to the 201 * general get method for configuration elements. 202 * 203 * @param conf configuration instance for accessing the passwords 204 * @param alias the name of the password element 205 * @param defPass the default password 206 * @return String password or default password 207 * @throws IOException 208 */ 209 public static String getPassword(Configuration conf, String alias, 210 String defPass) throws IOException { 211 String passwd = null; 212 try { 213 Method m = Configuration.class.getMethod("getPassword", String.class); 214 char[] p = (char[]) m.invoke(conf, alias); 215 if (p != null) { 216 LOG.debug(String.format("Config option \"%s\" was found through" + 217 " the Configuration getPassword method.", alias)); 218 passwd = new String(p); 219 } else { 220 LOG.debug(String.format( 221 "Config option \"%s\" was not found. Using provided default value", 222 alias)); 223 passwd = defPass; 224 } 225 } catch (NoSuchMethodException e) { 226 // this is a version of Hadoop where the credential 227 //provider API doesn't exist yet 228 LOG.debug(String.format( 229 "Credential.getPassword method is not available." + 230 " Falling back to configuration.")); 231 passwd = conf.get(alias, defPass); 232 } catch (SecurityException e) { 233 throw new IOException(e.getMessage(), e); 234 } catch (IllegalAccessException e) { 235 throw new IOException(e.getMessage(), e); 236 } catch (IllegalArgumentException e) { 237 throw new IOException(e.getMessage(), e); 238 } catch (InvocationTargetException e) { 239 throw new IOException(e.getMessage(), e); 240 } 241 return passwd; 242 } 243 244 /** 245 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key 246 * to the base Configuration. Note that additional configuration properties may be needed 247 * for a remote cluster, so it is preferable to use 248 * {@link #createClusterConf(Configuration, String, String)}. 249 * 250 * @param baseConf the base configuration to use, containing prefixed override properties 251 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 252 * 253 * @return the merged configuration with override properties and cluster key applied 254 * 255 * @see #createClusterConf(Configuration, String, String) 256 */ 257 public static Configuration createClusterConf(Configuration baseConf, String clusterKey) 258 throws IOException { 259 return createClusterConf(baseConf, clusterKey, null); 260 } 261 262 /** 263 * Generates a {@link Configuration} instance by applying property overrides prefixed by 264 * a cluster profile key to the base Configuration. Override properties are extracted by 265 * the {@link #subset(Configuration, String)} method, then the merged on top of the base 266 * Configuration and returned. 267 * 268 * @param baseConf the base configuration to use, containing prefixed override properties 269 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 270 * @param overridePrefix the property key prefix to match for override properties, 271 * or {@code null} if none 272 * @return the merged configuration with override properties and cluster key applied 273 */ 274 public static Configuration createClusterConf(Configuration baseConf, String clusterKey, 275 String overridePrefix) throws IOException { 276 Configuration clusterConf = HBaseConfiguration.create(baseConf); 277 if (clusterKey != null && !clusterKey.isEmpty()) { 278 applyClusterKeyToConf(clusterConf, clusterKey); 279 } 280 281 if (overridePrefix != null && !overridePrefix.isEmpty()) { 282 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix); 283 HBaseConfiguration.merge(clusterConf, clusterSubset); 284 } 285 return clusterConf; 286 } 287 288 /** 289 * Apply the settings in the given key to the given configuration, this is 290 * used to communicate with distant clusters 291 * @param conf configuration object to configure 292 * @param key string that contains the 3 required configuratins 293 * @throws IOException 294 */ 295 private static void applyClusterKeyToConf(Configuration conf, String key) 296 throws IOException{ 297 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); 298 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString()); 299 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort()); 300 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent()); 301 } 302 303 /** 304 * For debugging. Dump configurations to system output as xml format. 305 * Master and RS configurations can also be dumped using 306 * http services. e.g. "curl http://master:16010/dump" 307 */ 308 public static void main(String[] args) throws Exception { 309 HBaseConfiguration.create().writeXml(System.out); 310 } 311}