001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.service; 020 021import com.google.common.base.Strings; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.oozie.util.ConfigUtils; 024import org.apache.oozie.util.Instrumentable; 025import org.apache.oozie.util.Instrumentation; 026import org.apache.oozie.util.XLog; 027import org.apache.oozie.util.XConfiguration; 028import org.apache.oozie.ErrorCode; 029 030import java.io.File; 031import java.io.FileInputStream; 032import java.io.IOException; 033import java.io.InputStream; 034import java.io.StringWriter; 035import java.lang.reflect.InvocationTargetException; 036import java.lang.reflect.Method; 037import java.util.HashMap; 038import java.util.HashSet; 039import java.util.Map; 040import java.util.Set; 041import java.util.Arrays; 042 043import org.apache.oozie.util.ZKUtils; 044 045import com.google.common.annotations.VisibleForTesting; 046 047import javax.xml.parsers.DocumentBuilderFactory; 048 049/** 050 * Built in service that initializes the services configuration. 051 * <p> 052 * The configuration loading sequence is identical to Hadoop configuration loading sequence. 053 * <p> 054 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values 055 * are loaded from a site configuration file from the Oozie configuration directory. 056 * <p> 057 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME</code> environment variable as 058 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME</code> environment variable is not defined the 059 * initialization of the <code>ConfigurationService</code> fails. 060 * <p> 061 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory. 062 * <p> 063 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment 064 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory. 065 * <p> 066 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values. 067 * <p> 068 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode 069 * via system properties settings. 070 */ 071public class ConfigurationService implements Service, Instrumentable { 072 private static final String INSTRUMENTATION_GROUP = "configuration"; 073 074 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService."; 075 076 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties"; 077 078 public static final String CONF_VERIFY_AVAILABLE_PROPS = CONF_PREFIX + "verify.available.properties"; 079 080 public static final String CONF_JAVAX_XML_PARSERS_DOCUMENTBUILDERFACTORY = "oozie.javax.xml.parsers.DocumentBuilderFactory"; 081 082 /** 083 * System property that indicates the configuration directory. 084 */ 085 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir"; 086 087 088 /** 089 * System property that indicates the data directory. 090 */ 091 public static final String OOZIE_DATA_DIR = "oozie.data.dir"; 092 093 /** 094 * System property that indicates the name of the site configuration file to load. 095 */ 096 public static final String OOZIE_CONFIG_FILE = "oozie.config.file"; 097 098 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); 099 private static final Set<String> CONF_SYS_PROPS = new HashSet<String>(); 100 101 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; 102 private static final Set<String> MASK_PROPS = new HashSet<String>(); 103 private static Map<String,String> defaultConfigs = new HashMap<String,String>(); 104 105 static { 106 107 //all this properties are seeded as system properties, no need to log changes 108 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); 109 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR); 110 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR); 111 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE); 112 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR); 113 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR); 114 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE); 115 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD); 116 117 CONF_SYS_PROPS.add("oozie.http.hostname"); 118 CONF_SYS_PROPS.add("oozie.http.port"); 119 CONF_SYS_PROPS.add("oozie.https.port"); 120 CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID); 121 122 // These properties should be masked when displayed because they contain sensitive info (e.g. password) 123 MASK_PROPS.add(JPAService.CONF_PASSWORD); 124 MASK_PROPS.add("oozie.authentication.signature.secret"); 125 } 126 127 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; 128 public static final String SITE_CONFIG_FILE = "oozie-site.xml"; 129 130 private static XLog log = XLog.getLog(ConfigurationService.class); 131 132 private String configDir; 133 private String configFile; 134 135 private LogChangesConfiguration configuration; 136 137 public ConfigurationService() { 138 log = XLog.getLog(ConfigurationService.class); 139 } 140 141 /** 142 * Initialize the log service. 143 * 144 * @param services services instance. 145 * @throws ServiceException thrown if the log service could not be initialized. 146 */ 147 @Override 148 public void init(Services services) throws ServiceException { 149 configDir = getConfigurationDirectory(); 150 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE); 151 if (configFile.contains("/")) { 152 throw new ServiceException(ErrorCode.E0022, configFile); 153 } 154 log.info("Oozie home dir [{0}]", Services.getOozieHome()); 155 log.info("Oozie conf dir [{0}]", configDir); 156 log.info("Oozie conf file [{0}]", configFile); 157 configFile = new File(configDir, configFile).toString(); 158 configuration = loadConf(); 159 if (configuration.getBoolean(CONF_VERIFY_AVAILABLE_PROPS, false)) { 160 verifyConfigurationName(); 161 } 162 163 // Set the javax.xml.parsers.DocumentBuilderFactory property, which should make finding these classes faster, as the JVM 164 // doesn't have to do expensive searching. This happens quite frequently in Oozie. 165 String docFac = configuration.get(CONF_JAVAX_XML_PARSERS_DOCUMENTBUILDERFACTORY); 166 if (docFac != null && !docFac.trim().isEmpty()) { 167 System.setProperty("javax.xml.parsers.DocumentBuilderFactory", docFac.trim()); 168 Class<?> dbfClass = DocumentBuilderFactory.newInstance().getClass(); 169 log.debug("Using javax.xml.parsers.DocumentBuilderFactory: {0}", dbfClass.getName()); 170 } 171 } 172 173 public static String getConfigurationDirectory() throws ServiceException { 174 String oozieHome = Services.getOozieHome(); 175 String configDir = System.getProperty(OOZIE_CONFIG_DIR, null); 176 File file = configDir == null 177 ? new File(oozieHome, "conf") 178 : new File(configDir); 179 if (!file.exists()) { 180 throw new ServiceException(ErrorCode.E0024, configDir); 181 } 182 return file.getPath(); 183 } 184 185 /** 186 * Destroy the configuration service. 187 */ 188 @Override 189 public void destroy() { 190 configuration = null; 191 } 192 193 /** 194 * Return the public interface for configuration service. 195 * 196 * @return {@link ConfigurationService}. 197 */ 198 @Override 199 public Class<? extends Service> getInterface() { 200 return ConfigurationService.class; 201 } 202 203 /** 204 * Return the services configuration. 205 * 206 * @return the services configuration. 207 */ 208 public Configuration getConf() { 209 if (configuration == null) { 210 throw new IllegalStateException("Not initialized"); 211 } 212 return configuration; 213 } 214 215 /** 216 * Return Oozie configuration directory. 217 * 218 * @return Oozie configuration directory. 219 */ 220 public String getConfigDir() { 221 return configDir; 222 } 223 224 private InputStream getDefaultConfiguration() throws ServiceException, IOException { 225 ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 226 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE); 227 if (inputStream == null) { 228 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE); 229 } 230 return inputStream; 231 } 232 233 private LogChangesConfiguration loadConf() throws ServiceException { 234 XConfiguration configuration; 235 try { 236 InputStream inputStream = getDefaultConfiguration(); 237 configuration = loadConfig(inputStream, true); 238 File file = new File(configFile); 239 if (!file.exists()) { 240 log.info("Missing site configuration file [{0}]", configFile); 241 } 242 else { 243 inputStream = new FileInputStream(configFile); 244 XConfiguration siteConfiguration = loadConfig(inputStream, false); 245 XConfiguration.injectDefaults(configuration, siteConfiguration); 246 configuration = siteConfiguration; 247 } 248 } 249 catch (IOException ex) { 250 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex); 251 } 252 253 if (log.isTraceEnabled()) { 254 try { 255 StringWriter writer = new StringWriter(); 256 for (Map.Entry<String, String> entry : configuration) { 257 String value = getValue(configuration, entry.getKey()); 258 writer.write(" " + entry.getKey() + " = " + value + "\n"); 259 } 260 writer.close(); 261 log.trace("Configuration:\n{0}---", writer.toString()); 262 } 263 catch (IOException ex) { 264 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex); 265 } 266 } 267 268 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS); 269 if (ignoreSysProps != null) { 270 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps)); 271 } 272 273 for (Map.Entry<String, String> entry : configuration) { 274 String sysValue = System.getProperty(entry.getKey()); 275 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) { 276 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue); 277 configuration.set(entry.getKey(), sysValue); 278 } 279 } 280 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { 281 String name = (String) entry.getKey(); 282 if (!IGNORE_SYS_PROPS.contains(name)) { 283 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) { 284 if (configuration.get(name) == null) { 285 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name); 286 } 287 } 288 } 289 } 290 291 //Backward compatible, we should still support -Dparam. 292 for (String key : CONF_SYS_PROPS) { 293 String sysValue = System.getProperty(key); 294 if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) { 295 log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue); 296 configuration.set(key, sysValue); 297 } 298 } 299 300 return new LogChangesConfiguration(configuration); 301 } 302 303 private XConfiguration loadConfig(InputStream inputStream, boolean defaultConfig) throws IOException, ServiceException { 304 XConfiguration configuration; 305 configuration = new XConfiguration(inputStream); 306 for(Map.Entry<String,String> entry: configuration) { 307 if (defaultConfig) { 308 defaultConfigs.put(entry.getKey(), entry.getValue()); 309 } 310 else { 311 log.debug("Overriding configuration with oozie-site, [{0}]", entry.getKey()); 312 } 313 } 314 return configuration; 315 } 316 317 private class LogChangesConfiguration extends XConfiguration { 318 319 public LogChangesConfiguration(Configuration conf) { 320 for (Map.Entry<String, String> entry : conf) { 321 if (get(entry.getKey()) == null) { 322 setValue(entry.getKey(), entry.getValue()); 323 } 324 } 325 } 326 327 @Override 328 public String[] getStrings(String name) { 329 String s = get(name); 330 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0]; 331 } 332 333 @Override 334 public String[] getStrings(String name, String[] defaultValue) { 335 String s = get(name); 336 if (s == null) { 337 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, 338 Arrays.asList(defaultValue).toString()); 339 } 340 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : defaultValue; 341 } 342 343 @Override 344 public String get(String name, String defaultValue) { 345 String value = get(name); 346 if (value == null) { 347 boolean maskValue = MASK_PROPS.contains(name); 348 value = defaultValue; 349 String logValue = (maskValue) ? "**MASKED**" : defaultValue; 350 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, logValue); 351 } 352 return value; 353 } 354 355 @Override 356 public void set(String name, String value) { 357 setValue(name, value); 358 boolean maskValue = MASK_PROPS.contains(name); 359 value = (maskValue) ? "**MASKED**" : value; 360 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value); 361 } 362 363 @Override 364 public boolean getBoolean(String name, boolean defaultValue) { 365 String value = get(name); 366 if (value == null) { 367 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 368 } 369 return super.getBoolean(name, defaultValue); 370 } 371 372 @Override 373 public int getInt(String name, int defaultValue) { 374 String value = get(name); 375 if (value == null) { 376 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 377 } 378 return super.getInt(name, defaultValue); 379 } 380 381 @Override 382 public long getLong(String name, long defaultValue) { 383 String value = get(name); 384 if (value == null) { 385 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 386 } 387 return super.getLong(name, defaultValue); 388 } 389 390 @Override 391 public float getFloat(String name, float defaultValue) { 392 String value = get(name); 393 if (value == null) { 394 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 395 } 396 return super.getFloat(name, defaultValue); 397 } 398 399 @Override 400 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { 401 String value = get(name); 402 if (value == null) { 403 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 404 } 405 return super.getClasses(name, defaultValue); 406 } 407 408 @Override 409 public Class<?> getClass(String name, Class<?> defaultValue) { 410 String value = get(name); 411 if (value == null) { 412 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 413 return defaultValue; 414 } 415 try { 416 return getClassByName(value); 417 } catch (ClassNotFoundException e) { 418 throw new RuntimeException(e); 419 } 420 } 421 422 private void setValue(String name, String value) { 423 super.set(name, value); 424 } 425 426 } 427 428 /** 429 * Instruments the configuration service. <p> It sets instrumentation variables indicating the config dir and 430 * config file used. 431 * 432 * @param instr instrumentation to use. 433 */ 434 @Override 435 public void instrument(Instrumentation instr) { 436 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() { 437 @Override 438 public String getValue() { 439 return configDir; 440 } 441 }); 442 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 443 @Override 444 public String getValue() { 445 return configFile; 446 } 447 }); 448 } 449 450 /** 451 * Return a configuration with all sensitive values masked. 452 * 453 * @return masked configuration. 454 */ 455 public Configuration getMaskedConfiguration() { 456 XConfiguration maskedConf = new XConfiguration(); 457 Configuration conf = getConf(); 458 for (Map.Entry<String, String> entry : conf) { 459 String name = entry.getKey(); 460 String value = getValue(conf, name); 461 maskedConf.set(name, value); 462 } 463 return maskedConf; 464 } 465 466 private String getValue(Configuration config, String key) { 467 String value; 468 if (MASK_PROPS.contains(key)) { 469 value = "**MASKED**"; 470 } 471 else { 472 value = config.get(key); 473 } 474 return value; 475 } 476 477 478 /** 479 * Gets the oozie configuration value in oozie-default. 480 * @param name 481 * @return the configuration value of the <code>name</code> otherwise null 482 */ 483 private String getDefaultOozieConfig(String name) { 484 return defaultConfigs.get(name); 485 } 486 487 /** 488 * Verify the configuration is in oozie-default 489 */ 490 public void verifyConfigurationName() { 491 for (Map.Entry<String, String> entry: configuration) { 492 if (getDefaultOozieConfig(entry.getKey()) == null) { 493 log.warn("Invalid configuration defined, [{0}] ", entry.getKey()); 494 } 495 } 496 } 497 498 @VisibleForTesting 499 public static void set(String name, String value) { 500 Configuration conf = Services.get().getConf(); 501 conf.set(name, value); 502 } 503 504 @VisibleForTesting 505 public static void setBoolean(String name, boolean value) { 506 Configuration conf = Services.get().getConf(); 507 conf.setBoolean(name, value); 508 } 509 510 public static String get(String name) { 511 Configuration conf = Services.get().getConf(); 512 return get(conf, name); 513 } 514 515 public static String get(Configuration conf, String name) { 516 return conf.get(name, ConfigUtils.STRING_DEFAULT); 517 } 518 519 public static String[] getStrings(String name) { 520 Configuration conf = Services.get().getConf(); 521 return getStrings(conf, name); 522 } 523 524 public static String[] getStrings(Configuration conf, String name) { 525 return conf.getStrings(name, new String[0]); 526 } 527 528 public static boolean getBoolean(String name) { 529 Configuration conf = Services.get().getConf(); 530 return getBoolean(conf, name); 531 } 532 533 public static boolean getBoolean(String name, boolean defaultValue) { 534 return Services.get().getConf().getBoolean(name, defaultValue); 535 } 536 537 public static boolean getBoolean(Configuration conf, String name) { 538 return conf.getBoolean(name, ConfigUtils.BOOLEAN_DEFAULT); 539 } 540 541 /** 542 * Get the {@code boolean} value for {@code name} from {@code conf}, or the default {@link Configuration} coming from 543 * {@code oozie-site.xml}, or {@code defaultValue}, if no previous occurrences present. 544 * 545 * @param conf the {@link Configuration} for primary lookup 546 * @param name name of the parameter to look up 547 * @param defaultValue default value to return when every other possibility is exhausted 548 * @return a {@code boolean} given above lookup order 549 */ 550 public static boolean getBooleanOrDefault(final Configuration conf, final String name, final boolean defaultValue) { 551 if (Strings.isNullOrEmpty(conf.get(name))) { 552 final Configuration defaultConf = Services.get().getConf(); 553 return defaultConf.getBoolean(name, defaultValue); 554 } 555 556 return conf.getBoolean(name, defaultValue); 557 } 558 559 public static int getInt(String name) { 560 Configuration conf = Services.get().getConf(); 561 return getInt(conf, name); 562 } 563 564 public static int getInt(String name, int defaultValue) { 565 Configuration conf = Services.get().getConf(); 566 return conf.getInt(name, defaultValue); 567 } 568 569 public static int getInt(Configuration conf, String name) { 570 return conf.getInt(name, ConfigUtils.INT_DEFAULT); 571 } 572 573 public static float getFloat(String name) { 574 Configuration conf = Services.get().getConf(); 575 return conf.getFloat(name, ConfigUtils.FLOAT_DEFAULT); 576 } 577 578 public static long getLong(String name) { 579 return getLong(name, ConfigUtils.LONG_DEFAULT); 580 } 581 582 public static long getLong(String name, long defultValue) { 583 Configuration conf = Services.get().getConf(); 584 return getLong(conf, name, defultValue); 585 } 586 587 public static long getLong(Configuration conf, String name) { 588 return getLong(conf, name, ConfigUtils.LONG_DEFAULT); 589 } 590 591 public static long getLong(Configuration conf, String name, long defultValue) { 592 return conf.getLong(name, defultValue); 593 } 594 595 public static Class<?>[] getClasses(String name) { 596 Configuration conf = Services.get().getConf(); 597 return getClasses(conf, name); 598 } 599 600 public static Class<?>[] getClasses(Configuration conf, String name) { 601 return conf.getClasses(name); 602 } 603 604 public static Class<?> getClass(Configuration conf, String name) { 605 return conf.getClass(name, Object.class); 606 } 607 608 public static String getPassword(Configuration conf, String name) { 609 return getPassword(conf, name, null); 610 } 611 612 public static String getPassword(Configuration conf, String name, String defaultValue) { 613 try { 614 char[] pass = conf.getPassword(name); 615 return pass == null ? defaultValue : new String(pass); 616 } catch (IOException e) { 617 log.error(e); 618 throw new IllegalArgumentException("Could not load password for [" + name + "]", e); 619 } 620 } 621 622 public static String getPassword(String name, String defaultValue) { 623 Configuration conf = Services.get().getConf(); 624 return getPassword(conf, name, defaultValue); 625 } 626 627 public static Map<String, String> getValByRegex(final String regex) { 628 final Configuration conf = Services.get().getConf(); 629 return conf.getValByRegex(regex); 630 } 631}