001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.service; 020 021import org.apache.hadoop.conf.Configuration; 022import org.apache.oozie.util.ConfigUtils; 023import org.apache.oozie.util.Instrumentable; 024import org.apache.oozie.util.Instrumentation; 025import org.apache.oozie.util.XLog; 026import org.apache.oozie.util.XConfiguration; 027import org.apache.oozie.ErrorCode; 028 029import java.io.File; 030import java.io.FileInputStream; 031import java.io.IOException; 032import java.io.InputStream; 033import java.io.StringWriter; 034import java.lang.reflect.InvocationTargetException; 035import java.lang.reflect.Method; 036import java.util.HashMap; 037import java.util.HashSet; 038import java.util.Map; 039import java.util.Set; 040import java.util.Arrays; 041 042import org.apache.oozie.util.ZKUtils; 043 044import com.google.common.annotations.VisibleForTesting; 045 046/** 047 * Built in service that initializes the services configuration. 048 * <p> 049 * The configuration loading sequence is identical to Hadoop configuration loading sequence. 050 * <p> 051 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values 052 * are loaded from a site configuration file from the Oozie configuration directory. 053 * <p> 054 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME</code> environment variable as 055 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME</code> environment variable is not defined the 056 * initialization of the <code>ConfigurationService</code> fails. 057 * <p> 058 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory. 059 * <p> 060 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment 061 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory. 062 * <p> 063 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values. 064 * <p> 065 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode 066 * via system properties settings. 067 */ 068public class ConfigurationService implements Service, Instrumentable { 069 private static final String INSTRUMENTATION_GROUP = "configuration"; 070 071 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService."; 072 073 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties"; 074 075 public static final String CONF_VERIFY_AVAILABLE_PROPS = CONF_PREFIX + "verify.available.properties"; 076 077 /** 078 * System property that indicates the configuration directory. 079 */ 080 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir"; 081 082 083 /** 084 * System property that indicates the data directory. 085 */ 086 public static final String OOZIE_DATA_DIR = "oozie.data.dir"; 087 088 /** 089 * System property that indicates the name of the site configuration file to load. 090 */ 091 public static final String OOZIE_CONFIG_FILE = "oozie.config.file"; 092 093 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); 094 private static final Set<String> CONF_SYS_PROPS = new HashSet<String>(); 095 096 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; 097 private static final Set<String> MASK_PROPS = new HashSet<String>(); 098 private static Map<String,String> defaultConfigs = new HashMap<String,String>(); 099 100 private static Method getPasswordMethod; 101 102 static { 103 104 //all this properties are seeded as system properties, no need to log changes 105 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); 106 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR); 107 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR); 108 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE); 109 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR); 110 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR); 111 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE); 112 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD); 113 114 CONF_SYS_PROPS.add("oozie.http.hostname"); 115 CONF_SYS_PROPS.add("oozie.http.port"); 116 CONF_SYS_PROPS.add("oozie.https.port"); 117 CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID); 118 119 // These properties should be masked when displayed because they contain sensitive info (e.g. password) 120 MASK_PROPS.add(JPAService.CONF_PASSWORD); 121 MASK_PROPS.add("oozie.authentication.signature.secret"); 122 123 try { 124 // Only supported in Hadoop 2.6.0+ 125 getPasswordMethod = Configuration.class.getMethod("getPassword", String.class); 126 } catch (NoSuchMethodException e) { 127 // Not supported 128 getPasswordMethod = null; 129 } 130 } 131 132 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; 133 public static final String SITE_CONFIG_FILE = "oozie-site.xml"; 134 135 private static XLog log = XLog.getLog(ConfigurationService.class); 136 137 private String configDir; 138 private String configFile; 139 140 private LogChangesConfiguration configuration; 141 142 public ConfigurationService() { 143 log = XLog.getLog(ConfigurationService.class); 144 } 145 146 /** 147 * Initialize the log service. 148 * 149 * @param services services instance. 150 * @throws ServiceException thrown if the log service could not be initialized. 151 */ 152 @Override 153 public void init(Services services) throws ServiceException { 154 configDir = getConfigurationDirectory(); 155 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE); 156 if (configFile.contains("/")) { 157 throw new ServiceException(ErrorCode.E0022, configFile); 158 } 159 log.info("Oozie home dir [{0}]", Services.getOozieHome()); 160 log.info("Oozie conf dir [{0}]", configDir); 161 log.info("Oozie conf file [{0}]", configFile); 162 configFile = new File(configDir, configFile).toString(); 163 configuration = loadConf(); 164 if (configuration.getBoolean(CONF_VERIFY_AVAILABLE_PROPS, false)) { 165 verifyConfigurationName(); 166 } 167 } 168 169 public static String getConfigurationDirectory() throws ServiceException { 170 String oozieHome = Services.getOozieHome(); 171 String configDir = System.getProperty(OOZIE_CONFIG_DIR, null); 172 File file = configDir == null 173 ? new File(oozieHome, "conf") 174 : new File(configDir); 175 if (!file.exists()) { 176 throw new ServiceException(ErrorCode.E0024, configDir); 177 } 178 return file.getPath(); 179 } 180 181 /** 182 * Destroy the configuration service. 183 */ 184 @Override 185 public void destroy() { 186 configuration = null; 187 } 188 189 /** 190 * Return the public interface for configuration service. 191 * 192 * @return {@link ConfigurationService}. 193 */ 194 @Override 195 public Class<? extends Service> getInterface() { 196 return ConfigurationService.class; 197 } 198 199 /** 200 * Return the services configuration. 201 * 202 * @return the services configuration. 203 */ 204 public Configuration getConf() { 205 if (configuration == null) { 206 throw new IllegalStateException("Not initialized"); 207 } 208 return configuration; 209 } 210 211 /** 212 * Return Oozie configuration directory. 213 * 214 * @return Oozie configuration directory. 215 */ 216 public String getConfigDir() { 217 return configDir; 218 } 219 220 private InputStream getDefaultConfiguration() throws ServiceException, IOException { 221 ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 222 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE); 223 if (inputStream == null) { 224 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE); 225 } 226 return inputStream; 227 } 228 229 private LogChangesConfiguration loadConf() throws ServiceException { 230 XConfiguration configuration; 231 try { 232 InputStream inputStream = getDefaultConfiguration(); 233 configuration = loadConfig(inputStream, true); 234 File file = new File(configFile); 235 if (!file.exists()) { 236 log.info("Missing site configuration file [{0}]", configFile); 237 } 238 else { 239 inputStream = new FileInputStream(configFile); 240 XConfiguration siteConfiguration = loadConfig(inputStream, false); 241 XConfiguration.injectDefaults(configuration, siteConfiguration); 242 configuration = siteConfiguration; 243 } 244 } 245 catch (IOException ex) { 246 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex); 247 } 248 249 if (log.isTraceEnabled()) { 250 try { 251 StringWriter writer = new StringWriter(); 252 for (Map.Entry<String, String> entry : configuration) { 253 String value = getValue(configuration, entry.getKey()); 254 writer.write(" " + entry.getKey() + " = " + value + "\n"); 255 } 256 writer.close(); 257 log.trace("Configuration:\n{0}---", writer.toString()); 258 } 259 catch (IOException ex) { 260 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex); 261 } 262 } 263 264 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS); 265 if (ignoreSysProps != null) { 266 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps)); 267 } 268 269 for (Map.Entry<String, String> entry : configuration) { 270 String sysValue = System.getProperty(entry.getKey()); 271 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) { 272 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue); 273 configuration.set(entry.getKey(), sysValue); 274 } 275 } 276 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { 277 String name = (String) entry.getKey(); 278 if (!IGNORE_SYS_PROPS.contains(name)) { 279 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) { 280 if (configuration.get(name) == null) { 281 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name); 282 } 283 } 284 } 285 } 286 287 //Backward compatible, we should still support -Dparam. 288 for (String key : CONF_SYS_PROPS) { 289 String sysValue = System.getProperty(key); 290 if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) { 291 log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue); 292 configuration.set(key, sysValue); 293 } 294 } 295 296 return new LogChangesConfiguration(configuration); 297 } 298 299 private XConfiguration loadConfig(InputStream inputStream, boolean defaultConfig) throws IOException, ServiceException { 300 XConfiguration configuration; 301 configuration = new XConfiguration(inputStream); 302 for(Map.Entry<String,String> entry: configuration) { 303 if (defaultConfig) { 304 defaultConfigs.put(entry.getKey(), entry.getValue()); 305 } 306 else { 307 log.debug("Overriding configuration with oozie-site, [{0}]", entry.getKey()); 308 } 309 } 310 return configuration; 311 } 312 313 private class LogChangesConfiguration extends XConfiguration { 314 315 public LogChangesConfiguration(Configuration conf) { 316 for (Map.Entry<String, String> entry : conf) { 317 if (get(entry.getKey()) == null) { 318 setValue(entry.getKey(), entry.getValue()); 319 } 320 } 321 } 322 323 @Override 324 public String[] getStrings(String name) { 325 String s = get(name); 326 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0]; 327 } 328 329 @Override 330 public String[] getStrings(String name, String[] defaultValue) { 331 String s = get(name); 332 if (s == null) { 333 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, 334 Arrays.asList(defaultValue).toString()); 335 } 336 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : defaultValue; 337 } 338 339 @Override 340 public String get(String name, String defaultValue) { 341 String value = get(name); 342 if (value == null) { 343 boolean maskValue = MASK_PROPS.contains(name); 344 value = defaultValue; 345 String logValue = (maskValue) ? "**MASKED**" : defaultValue; 346 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, logValue); 347 } 348 return value; 349 } 350 351 @Override 352 public void set(String name, String value) { 353 setValue(name, value); 354 boolean maskValue = MASK_PROPS.contains(name); 355 value = (maskValue) ? "**MASKED**" : value; 356 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value); 357 } 358 359 @Override 360 public boolean getBoolean(String name, boolean defaultValue) { 361 String value = get(name); 362 if (value == null) { 363 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 364 } 365 return super.getBoolean(name, defaultValue); 366 } 367 368 @Override 369 public int getInt(String name, int defaultValue) { 370 String value = get(name); 371 if (value == null) { 372 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 373 } 374 return super.getInt(name, defaultValue); 375 } 376 377 @Override 378 public long getLong(String name, long defaultValue) { 379 String value = get(name); 380 if (value == null) { 381 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 382 } 383 return super.getLong(name, defaultValue); 384 } 385 386 @Override 387 public float getFloat(String name, float defaultValue) { 388 String value = get(name); 389 if (value == null) { 390 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 391 } 392 return super.getFloat(name, defaultValue); 393 } 394 395 @Override 396 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { 397 String value = get(name); 398 if (value == null) { 399 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 400 } 401 return super.getClasses(name, defaultValue); 402 } 403 404 @Override 405 public Class<?> getClass(String name, Class<?> defaultValue) { 406 String value = get(name); 407 if (value == null) { 408 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 409 return defaultValue; 410 } 411 try { 412 return getClassByName(value); 413 } catch (ClassNotFoundException e) { 414 throw new RuntimeException(e); 415 } 416 } 417 418 private void setValue(String name, String value) { 419 super.set(name, value); 420 } 421 422 } 423 424 /** 425 * Instruments the configuration service. <p> It sets instrumentation variables indicating the config dir and 426 * config file used. 427 * 428 * @param instr instrumentation to use. 429 */ 430 @Override 431 public void instrument(Instrumentation instr) { 432 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() { 433 @Override 434 public String getValue() { 435 return configDir; 436 } 437 }); 438 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 439 @Override 440 public String getValue() { 441 return configFile; 442 } 443 }); 444 } 445 446 /** 447 * Return a configuration with all sensitive values masked. 448 * 449 * @return masked configuration. 450 */ 451 public Configuration getMaskedConfiguration() { 452 XConfiguration maskedConf = new XConfiguration(); 453 Configuration conf = getConf(); 454 for (Map.Entry<String, String> entry : conf) { 455 String name = entry.getKey(); 456 String value = getValue(conf, name); 457 maskedConf.set(name, value); 458 } 459 return maskedConf; 460 } 461 462 private String getValue(Configuration config, String key) { 463 String value; 464 if (MASK_PROPS.contains(key)) { 465 value = "**MASKED**"; 466 } 467 else { 468 value = config.get(key); 469 } 470 return value; 471 } 472 473 474 /** 475 * Gets the oozie configuration value in oozie-default. 476 * @param name 477 * @return the configuration value of the <code>name</code> otherwise null 478 */ 479 private String getDefaultOozieConfig(String name) { 480 return defaultConfigs.get(name); 481 } 482 483 /** 484 * Verify the configuration is in oozie-default 485 */ 486 public void verifyConfigurationName() { 487 for (Map.Entry<String, String> entry: configuration) { 488 if (getDefaultOozieConfig(entry.getKey()) == null) { 489 log.warn("Invalid configuration defined, [{0}] ", entry.getKey()); 490 } 491 } 492 } 493 494 @VisibleForTesting 495 public static void set(String name, String value) { 496 Configuration conf = Services.get().getConf(); 497 conf.set(name, value); 498 } 499 500 @VisibleForTesting 501 public static void setBoolean(String name, boolean value) { 502 Configuration conf = Services.get().getConf(); 503 conf.setBoolean(name, value); 504 } 505 506 public static String get(String name) { 507 Configuration conf = Services.get().getConf(); 508 return get(conf, name); 509 } 510 511 public static String get(Configuration conf, String name) { 512 return conf.get(name, ConfigUtils.STRING_DEFAULT); 513 } 514 515 public static String[] getStrings(String name) { 516 Configuration conf = Services.get().getConf(); 517 return getStrings(conf, name); 518 } 519 520 public static String[] getStrings(Configuration conf, String name) { 521 return conf.getStrings(name, new String[0]); 522 } 523 524 public static boolean getBoolean(String name) { 525 Configuration conf = Services.get().getConf(); 526 return getBoolean(conf, name); 527 } 528 529 public static boolean getBoolean(Configuration conf, String name) { 530 return conf.getBoolean(name, ConfigUtils.BOOLEAN_DEFAULT); 531 } 532 533 public static int getInt(String name) { 534 Configuration conf = Services.get().getConf(); 535 return getInt(conf, name); 536 } 537 538 public static int getInt(String name, int defaultValue) { 539 Configuration conf = Services.get().getConf(); 540 return conf.getInt(name, defaultValue); 541 } 542 543 public static int getInt(Configuration conf, String name) { 544 return conf.getInt(name, ConfigUtils.INT_DEFAULT); 545 } 546 547 public static float getFloat(String name) { 548 Configuration conf = Services.get().getConf(); 549 return conf.getFloat(name, ConfigUtils.FLOAT_DEFAULT); 550 } 551 552 public static long getLong(String name) { 553 return getLong(name, ConfigUtils.LONG_DEFAULT); 554 } 555 556 public static long getLong(String name, long defultValue) { 557 Configuration conf = Services.get().getConf(); 558 return getLong(conf, name, defultValue); 559 } 560 561 public static long getLong(Configuration conf, String name) { 562 return getLong(conf, name, ConfigUtils.LONG_DEFAULT); 563 } 564 public static long getLong(Configuration conf, String name, long defultValue) { 565 return conf.getLong(name, defultValue); 566 } 567 568 public static Class<?>[] getClasses(String name) { 569 Configuration conf = Services.get().getConf(); 570 return getClasses(conf, name); 571 } 572 573 public static Class<?>[] getClasses(Configuration conf, String name) { 574 return conf.getClasses(name); 575 } 576 577 public static Class<?> getClass(Configuration conf, String name) { 578 return conf.getClass(name, Object.class); 579 } 580 581 public static String getPassword(Configuration conf, String name) { 582 return getPassword(conf, name, null); 583 } 584 585 public static String getPassword(Configuration conf, String name, String defaultValue) { 586 if (getPasswordMethod != null) { 587 try { 588 char[] pass = (char[]) getPasswordMethod.invoke(conf, name); 589 return pass == null ? defaultValue : new String(pass); 590 } catch (IllegalAccessException e) { 591 log.error(e); 592 throw new IllegalArgumentException("Could not load password for [" + name + "]", e); 593 } catch (InvocationTargetException e) { 594 log.error(e); 595 throw new IllegalArgumentException("Could not load password for [" + name + "]", e); 596 } 597 } else { 598 return conf.get(name); 599 } 600 } 601 602 public static String getPassword(String name, String defaultValue) { 603 Configuration conf = Services.get().getConf(); 604 return getPassword(conf, name, defaultValue); 605 } 606 607}