001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.service; 020 021import com.google.common.annotations.VisibleForTesting; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.oozie.ErrorCode; 024import org.apache.oozie.util.ConfigUtils; 025import org.apache.oozie.util.Instrumentable; 026import org.apache.oozie.util.Instrumentation; 027import org.apache.oozie.util.XConfiguration; 028import org.apache.oozie.util.XLog; 029import org.apache.oozie.util.ZKUtils; 030 031import java.io.File; 032import java.io.FileInputStream; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.StringWriter; 036import java.lang.reflect.InvocationTargetException; 037import java.lang.reflect.Method; 038import java.util.Arrays; 039import java.util.HashMap; 040import java.util.HashSet; 041import java.util.Map; 042import java.util.Set; 043 044/** 045 * Built in service that initializes the services configuration. 046 * <p> 047 * The configuration loading sequence is identical to Hadoop configuration loading sequence. 048 * <p> 049 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values 050 * are loaded from a site configuration file from the Oozie configuration directory. 051 * <p> 052 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME</code> environment variable as 053 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME</code> environment variable is not defined the 054 * initialization of the <code>ConfigurationService</code> fails. 055 * <p> 056 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory. 057 * <p> 058 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment 059 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory. 060 * <p> 061 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values. 062 * <p> 063 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode 064 * via system properties settings. 065 */ 066public class ConfigurationService implements Service, Instrumentable { 067 private static final String INSTRUMENTATION_GROUP = "configuration"; 068 069 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService."; 070 071 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties"; 072 073 public static final String CONF_VERIFY_AVAILABLE_PROPS = CONF_PREFIX + "verify.available.properties"; 074 075 /** 076 * System property that indicates the configuration directory. 077 */ 078 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir"; 079 080 081 /** 082 * System property that indicates the data directory. 083 */ 084 public static final String OOZIE_DATA_DIR = "oozie.data.dir"; 085 086 /** 087 * System property that indicates the name of the site configuration file to load. 088 */ 089 public static final String OOZIE_CONFIG_FILE = "oozie.config.file"; 090 091 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>(); 092 private static final Set<String> CONF_SYS_PROPS = new HashSet<String>(); 093 094 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test."; 095 private static final Set<String> MASK_PROPS = new HashSet<String>(); 096 private static Map<String,String> defaultConfigs = new HashMap<String,String>(); 097 098 private static Method getPasswordMethod; 099 100 static { 101 102 //all this properties are seeded as system properties, no need to log changes 103 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS); 104 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR); 105 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR); 106 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE); 107 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR); 108 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR); 109 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE); 110 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD); 111 112 CONF_SYS_PROPS.add("oozie.http.hostname"); 113 CONF_SYS_PROPS.add("oozie.http.port"); 114 CONF_SYS_PROPS.add("oozie.https.port"); 115 CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID); 116 117 // These properties should be masked when displayed because they contain sensitive info (e.g. password) 118 MASK_PROPS.add(JPAService.CONF_PASSWORD); 119 MASK_PROPS.add("oozie.authentication.signature.secret"); 120 121 try { 122 // Only supported in Hadoop 2.6.0+ 123 getPasswordMethod = Configuration.class.getMethod("getPassword", String.class); 124 } catch (NoSuchMethodException e) { 125 // Not supported 126 getPasswordMethod = null; 127 } 128 129 try { 130 Method method = Configuration.class.getDeclaredMethod("setRestrictSystemPropertiesDefault", boolean 131 .class); 132 method.invoke(null, true); 133 } catch( NoSuchMethodException | InvocationTargetException | IllegalAccessException ignore) { 134 } 135 } 136 137 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; 138 public static final String SITE_CONFIG_FILE = "oozie-site.xml"; 139 140 private static XLog log = XLog.getLog(ConfigurationService.class); 141 142 private String configDir; 143 private String configFile; 144 145 private LogChangesConfiguration configuration; 146 147 public ConfigurationService() { 148 log = XLog.getLog(ConfigurationService.class); 149 } 150 151 /** 152 * Initialize the log service. 153 * 154 * @param services services instance. 155 * @throws ServiceException thrown if the log service could not be initialized. 156 */ 157 @Override 158 public void init(Services services) throws ServiceException { 159 configDir = getConfigurationDirectory(); 160 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE); 161 if (configFile.contains("/")) { 162 throw new ServiceException(ErrorCode.E0022, configFile); 163 } 164 log.info("Oozie home dir [{0}]", Services.getOozieHome()); 165 log.info("Oozie conf dir [{0}]", configDir); 166 log.info("Oozie conf file [{0}]", configFile); 167 configFile = new File(configDir, configFile).toString(); 168 configuration = loadConf(); 169 if (configuration.getBoolean(CONF_VERIFY_AVAILABLE_PROPS, false)) { 170 verifyConfigurationName(); 171 } 172 } 173 174 public static String getConfigurationDirectory() throws ServiceException { 175 String oozieHome = Services.getOozieHome(); 176 String configDir = System.getProperty(OOZIE_CONFIG_DIR, null); 177 File file = configDir == null 178 ? new File(oozieHome, "conf") 179 : new File(configDir); 180 if (!file.exists()) { 181 throw new ServiceException(ErrorCode.E0024, configDir); 182 } 183 return file.getPath(); 184 } 185 186 /** 187 * Destroy the configuration service. 188 */ 189 @Override 190 public void destroy() { 191 configuration = null; 192 } 193 194 /** 195 * Return the public interface for configuration service. 196 * 197 * @return {@link ConfigurationService}. 198 */ 199 @Override 200 public Class<? extends Service> getInterface() { 201 return ConfigurationService.class; 202 } 203 204 /** 205 * Return the services configuration. 206 * 207 * @return the services configuration. 208 */ 209 public Configuration getConf() { 210 if (configuration == null) { 211 throw new IllegalStateException("Not initialized"); 212 } 213 return configuration; 214 } 215 216 /** 217 * Return Oozie configuration directory. 218 * 219 * @return Oozie configuration directory. 220 */ 221 public String getConfigDir() { 222 return configDir; 223 } 224 225 private InputStream getDefaultConfiguration() throws ServiceException, IOException { 226 ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 227 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE); 228 if (inputStream == null) { 229 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE); 230 } 231 return inputStream; 232 } 233 234 private LogChangesConfiguration loadConf() throws ServiceException { 235 XConfiguration configuration; 236 try { 237 InputStream inputStream = getDefaultConfiguration(); 238 configuration = loadConfig(inputStream, true); 239 File file = new File(configFile); 240 if (!file.exists()) { 241 log.info("Missing site configuration file [{0}]", configFile); 242 } 243 else { 244 inputStream = new FileInputStream(configFile); 245 XConfiguration siteConfiguration = loadConfig(inputStream, false); 246 XConfiguration.injectDefaults(configuration, siteConfiguration); 247 configuration = siteConfiguration; 248 } 249 } 250 catch (IOException ex) { 251 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex); 252 } 253 254 if (log.isTraceEnabled()) { 255 try { 256 StringWriter writer = new StringWriter(); 257 for (Map.Entry<String, String> entry : configuration) { 258 String value = getValue(configuration, entry.getKey()); 259 writer.write(" " + entry.getKey() + " = " + value + "\n"); 260 } 261 writer.close(); 262 log.trace("Configuration:\n{0}---", writer.toString()); 263 } 264 catch (IOException ex) { 265 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex); 266 } 267 } 268 269 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS); 270 if (ignoreSysProps != null) { 271 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps)); 272 } 273 274 for (Map.Entry<String, String> entry : configuration) { 275 String sysValue = System.getProperty(entry.getKey()); 276 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) { 277 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue); 278 configuration.set(entry.getKey(), sysValue); 279 } 280 } 281 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { 282 String name = (String) entry.getKey(); 283 if (!IGNORE_SYS_PROPS.contains(name)) { 284 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) { 285 if (configuration.get(name) == null) { 286 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name); 287 } 288 } 289 } 290 } 291 292 //Backward compatible, we should still support -Dparam. 293 for (String key : CONF_SYS_PROPS) { 294 String sysValue = System.getProperty(key); 295 if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) { 296 log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue); 297 configuration.set(key, sysValue); 298 } 299 } 300 301 return new LogChangesConfiguration(configuration); 302 } 303 304 private XConfiguration loadConfig(InputStream inputStream, boolean defaultConfig) throws IOException, ServiceException { 305 XConfiguration configuration; 306 configuration = new XConfiguration(inputStream); 307 configuration.setRestrictSystemProperties(false); 308 for(Map.Entry<String,String> entry: configuration) { 309 if (defaultConfig) { 310 defaultConfigs.put(entry.getKey(), entry.getValue()); 311 } 312 else { 313 log.debug("Overriding configuration with oozie-site, [{0}]", entry.getKey()); 314 } 315 } 316 return configuration; 317 } 318 319 private class LogChangesConfiguration extends XConfiguration { 320 321 public LogChangesConfiguration(Configuration conf) { 322 for (Map.Entry<String, String> entry : conf) { 323 if (get(entry.getKey()) == null) { 324 setValue(entry.getKey(), entry.getValue()); 325 } 326 } 327 if(conf instanceof XConfiguration) { 328 this.setRestrictParser(((XConfiguration)conf).getRestrictParser()); 329 this.setRestrictSystemProperties(((XConfiguration)conf).getRestrictSystemProperties()); 330 } 331 } 332 333 @Override 334 public String[] getStrings(String name) { 335 String s = get(name); 336 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0]; 337 } 338 339 @Override 340 public String[] getStrings(String name, String[] defaultValue) { 341 String s = get(name); 342 if (s == null) { 343 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, 344 Arrays.asList(defaultValue).toString()); 345 } 346 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : defaultValue; 347 } 348 349 @Override 350 public String get(String name, String defaultValue) { 351 String value = get(name); 352 if (value == null) { 353 boolean maskValue = MASK_PROPS.contains(name); 354 value = defaultValue; 355 String logValue = (maskValue) ? "**MASKED**" : defaultValue; 356 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, logValue); 357 } 358 return value; 359 } 360 361 @Override 362 public void set(String name, String value) { 363 setValue(name, value); 364 boolean maskValue = MASK_PROPS.contains(name); 365 value = (maskValue) ? "**MASKED**" : value; 366 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value); 367 } 368 369 @Override 370 public boolean getBoolean(String name, boolean defaultValue) { 371 String value = get(name); 372 if (value == null) { 373 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 374 } 375 return super.getBoolean(name, defaultValue); 376 } 377 378 @Override 379 public int getInt(String name, int defaultValue) { 380 String value = get(name); 381 if (value == null) { 382 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 383 } 384 return super.getInt(name, defaultValue); 385 } 386 387 @Override 388 public long getLong(String name, long defaultValue) { 389 String value = get(name); 390 if (value == null) { 391 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 392 } 393 return super.getLong(name, defaultValue); 394 } 395 396 @Override 397 public float getFloat(String name, float defaultValue) { 398 String value = get(name); 399 if (value == null) { 400 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 401 } 402 return super.getFloat(name, defaultValue); 403 } 404 405 @Override 406 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { 407 String value = get(name); 408 if (value == null) { 409 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 410 } 411 return super.getClasses(name, defaultValue); 412 } 413 414 @Override 415 public Class<?> getClass(String name, Class<?> defaultValue) { 416 String value = get(name); 417 if (value == null) { 418 log.debug(XLog.OPS, "Configuration property [{0}] not found, use given value [{1}]", name, defaultValue); 419 return defaultValue; 420 } 421 try { 422 return getClassByName(value); 423 } catch (ClassNotFoundException e) { 424 throw new RuntimeException(e); 425 } 426 } 427 428 private void setValue(String name, String value) { 429 super.set(name, value); 430 } 431 432 } 433 434 /** 435 * Instruments the configuration service. <p> It sets instrumentation variables indicating the config dir and 436 * config file used. 437 * 438 * @param instr instrumentation to use. 439 */ 440 @Override 441 public void instrument(Instrumentation instr) { 442 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() { 443 @Override 444 public String getValue() { 445 return configDir; 446 } 447 }); 448 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 449 @Override 450 public String getValue() { 451 return configFile; 452 } 453 }); 454 } 455 456 /** 457 * Return a configuration with all sensitive values masked. 458 * 459 * @return masked configuration. 460 */ 461 public Configuration getMaskedConfiguration() { 462 XConfiguration maskedConf = new XConfiguration(); 463 Configuration conf = getConf(); 464 for (Map.Entry<String, String> entry : conf) { 465 String name = entry.getKey(); 466 String value = getValue(conf, name); 467 maskedConf.set(name, value); 468 } 469 return maskedConf; 470 } 471 472 private String getValue(Configuration config, String key) { 473 String value; 474 if (MASK_PROPS.contains(key)) { 475 value = "**MASKED**"; 476 } 477 else { 478 value = config.get(key); 479 } 480 return value; 481 } 482 483 484 /** 485 * Gets the oozie configuration value in oozie-default. 486 * @param name 487 * @return the configuration value of the <code>name</code> otherwise null 488 */ 489 private String getDefaultOozieConfig(String name) { 490 return defaultConfigs.get(name); 491 } 492 493 /** 494 * Verify the configuration is in oozie-default 495 */ 496 public void verifyConfigurationName() { 497 for (Map.Entry<String, String> entry: configuration) { 498 if (getDefaultOozieConfig(entry.getKey()) == null) { 499 log.warn("Invalid configuration defined, [{0}] ", entry.getKey()); 500 } 501 } 502 } 503 504 @VisibleForTesting 505 public static void set(String name, String value) { 506 Configuration conf = Services.get().getConf(); 507 conf.set(name, value); 508 } 509 510 @VisibleForTesting 511 public static void setBoolean(String name, boolean value) { 512 Configuration conf = Services.get().getConf(); 513 conf.setBoolean(name, value); 514 } 515 516 public static String get(String name) { 517 Configuration conf = Services.get().getConf(); 518 return get(conf, name); 519 } 520 521 public static String get(Configuration conf, String name) { 522 return conf.get(name, ConfigUtils.STRING_DEFAULT); 523 } 524 525 public static String[] getStrings(String name) { 526 Configuration conf = Services.get().getConf(); 527 return getStrings(conf, name); 528 } 529 530 public static String[] getStrings(Configuration conf, String name) { 531 return conf.getStrings(name, new String[0]); 532 } 533 534 public static boolean getBoolean(String name) { 535 Configuration conf = Services.get().getConf(); 536 return getBoolean(conf, name); 537 } 538 539 public static boolean getBoolean(Configuration conf, String name) { 540 return conf.getBoolean(name, ConfigUtils.BOOLEAN_DEFAULT); 541 } 542 543 public static int getInt(String name) { 544 Configuration conf = Services.get().getConf(); 545 return getInt(conf, name); 546 } 547 548 public static int getInt(String name, int defaultValue) { 549 Configuration conf = Services.get().getConf(); 550 return conf.getInt(name, defaultValue); 551 } 552 553 public static int getInt(Configuration conf, String name) { 554 return conf.getInt(name, ConfigUtils.INT_DEFAULT); 555 } 556 557 public static float getFloat(String name) { 558 Configuration conf = Services.get().getConf(); 559 return conf.getFloat(name, ConfigUtils.FLOAT_DEFAULT); 560 } 561 562 public static long getLong(String name) { 563 return getLong(name, ConfigUtils.LONG_DEFAULT); 564 } 565 566 public static long getLong(String name, long defultValue) { 567 Configuration conf = Services.get().getConf(); 568 return getLong(conf, name, defultValue); 569 } 570 571 public static long getLong(Configuration conf, String name) { 572 return getLong(conf, name, ConfigUtils.LONG_DEFAULT); 573 } 574 public static long getLong(Configuration conf, String name, long defultValue) { 575 return conf.getLong(name, defultValue); 576 } 577 578 public static Class<?>[] getClasses(String name) { 579 Configuration conf = Services.get().getConf(); 580 return getClasses(conf, name); 581 } 582 583 public static Class<?>[] getClasses(Configuration conf, String name) { 584 return conf.getClasses(name); 585 } 586 587 public static Class<?> getClass(Configuration conf, String name) { 588 return conf.getClass(name, Object.class); 589 } 590 591 public static String getPassword(Configuration conf, String name) { 592 return getPassword(conf, name, null); 593 } 594 595 public static String getPassword(Configuration conf, String name, String defaultValue) { 596 if (getPasswordMethod != null) { 597 try { 598 char[] pass = (char[]) getPasswordMethod.invoke(conf, name); 599 return pass == null ? defaultValue : new String(pass); 600 } catch (IllegalAccessException e) { 601 log.error(e); 602 throw new IllegalArgumentException("Could not load password for [" + name + "]", e); 603 } catch (InvocationTargetException e) { 604 log.error(e); 605 throw new IllegalArgumentException("Could not load password for [" + name + "]", e); 606 } 607 } else { 608 return conf.get(name); 609 } 610 } 611 612 public static String getPassword(String name, String defaultValue) { 613 Configuration conf = Services.get().getConf(); 614 return getPassword(conf, name, defaultValue); 615 } 616 617}