001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import org.apache.hadoop.conf.Configuration;
021 import org.apache.oozie.util.Instrumentable;
022 import org.apache.oozie.util.Instrumentation;
023 import org.apache.oozie.util.XLog;
024 import org.apache.oozie.util.XConfiguration;
025 import org.apache.oozie.ErrorCode;
026
027 import java.io.File;
028 import java.io.FileInputStream;
029 import java.io.IOException;
030 import java.io.InputStream;
031 import java.io.StringWriter;
032 import java.util.HashSet;
033 import java.util.Map;
034 import java.util.Set;
035 import java.util.Arrays;
036
037 /**
038 * Built in service that initializes the services configuration.
039 * <p/>
040 * The configuration loading sequence is identical to Hadoop configuration loading sequence.
041 * <p/>
042 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
043 * are loaded from a site configuration file from the Oozie configuration directory.
044 * <p/>
045 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
046 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
047 * initialization of the <code>ConfigurationService</code> fails.
048 * <p/>
049 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
050 * <p/>
051 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
052 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
053 * <p/>
054 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
055 * <p/>
056 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
057 * via system properties settings.
058 */
059 public class ConfigurationService implements Service, Instrumentable {
060 private static final String INSTRUMENTATION_GROUP = "configuration";
061
062 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
063
064 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
065
066 /**
067 * System property that indicates the configuration directory.
068 */
069 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
070
071
072 /**
073 * System property that indicates the data directory.
074 */
075 public static final String OOZIE_DATA_DIR = "oozie.data.dir";
076
077 /**
078 * System property that indicates the name of the site configuration file to load.
079 */
080 public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
081
082 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
083 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
084
085 private static final String PASSWORD_PROPERTY_END = ".password";
086
087 static {
088 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
089
090 //all this properties are seeded as system properties, no need to log changes
091 IGNORE_SYS_PROPS.add("oozie.http.hostname");
092 IGNORE_SYS_PROPS.add("oozie.http.port");
093
094 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
095 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
096 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
097 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
098
099 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
100 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
101 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
102 }
103
104 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
105 public static final String SITE_CONFIG_FILE = "oozie-site.xml";
106
107 private static XLog log = XLog.getLog(ConfigurationService.class);
108
109 private String configDir;
110 private String configFile;
111
112 private LogChangesConfiguration configuration;
113
114 public ConfigurationService() {
115 log = XLog.getLog(ConfigurationService.class);
116 }
117
118 /**
119 * Initialize the log service.
120 *
121 * @param services services instance.
122 * @throws ServiceException thrown if the log service could not be initialized.
123 */
124 public void init(Services services) throws ServiceException {
125 configDir = getConfigurationDirectory();
126 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
127 if (configFile.contains("/")) {
128 throw new ServiceException(ErrorCode.E0022, configFile);
129 }
130 log.info("Oozie home dir [{0}]", Services.getOozieHome());
131 log.info("Oozie conf dir [{0}]", configDir);
132 log.info("Oozie conf file [{0}]", configFile);
133 configFile = new File(configDir, configFile).toString();
134 configuration = loadConf();
135 }
136
137 public static String getConfigurationDirectory() throws ServiceException {
138 String oozieHome = Services.getOozieHome();
139 String configDir = System.getProperty(OOZIE_CONFIG_DIR, oozieHome + "/conf");
140 File file = new File(configDir);
141 if (!file.exists()) {
142 throw new ServiceException(ErrorCode.E0024, configDir);
143 }
144 return configDir;
145 }
146
147 /**
148 * Destroy the configuration service.
149 */
150 public void destroy() {
151 configuration = null;
152 }
153
154 /**
155 * Return the public interface for configuration service.
156 *
157 * @return {@link ConfigurationService}.
158 */
159 public Class<? extends Service> getInterface() {
160 return ConfigurationService.class;
161 }
162
163 /**
164 * Return the services configuration.
165 *
166 * @return the services configuration.
167 */
168 public Configuration getConf() {
169 if (configuration == null) {
170 throw new IllegalStateException("Not initialized");
171 }
172 return configuration;
173 }
174
175 /**
176 * Return Oozie configuration directory.
177 *
178 * @return Oozie configuration directory.
179 */
180 public String getConfigDir() {
181 return configDir;
182 }
183
184 private InputStream getDefaultConfiguration() throws ServiceException, IOException {
185 ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
186 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
187 if (inputStream == null) {
188 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
189 }
190 return inputStream;
191 }
192
193 private LogChangesConfiguration loadConf() throws ServiceException {
194 XConfiguration configuration;
195 try {
196 InputStream inputStream = getDefaultConfiguration();
197 configuration = new XConfiguration(inputStream);
198 File file = new File(configFile);
199 if (!file.exists()) {
200 log.info("Missing site configuration file [{0}]", configFile);
201 }
202 else {
203 inputStream = new FileInputStream(configFile);
204 XConfiguration siteConfiguration = new XConfiguration(inputStream);
205 XConfiguration.injectDefaults(configuration, siteConfiguration);
206 configuration = siteConfiguration;
207 }
208 }
209 catch (IOException ex) {
210 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
211 }
212
213 if (log.isTraceEnabled()) {
214 try {
215 StringWriter writer = new StringWriter();
216 for (Map.Entry<String, String> entry : configuration) {
217 boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
218 String value = (maskValue) ? "**MASKED**" : entry.getValue();
219 writer.write(" " + entry.getKey() + " = " + value + "\n");
220 }
221 writer.close();
222 log.trace("Configuration:\n{0}---", writer.toString());
223 }
224 catch (IOException ex) {
225 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
226 }
227 }
228
229 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
230 if (ignoreSysProps != null) {
231 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
232 }
233
234 for (Map.Entry<String, String> entry : configuration) {
235 String sysValue = System.getProperty(entry.getKey());
236 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
237 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
238 configuration.set(entry.getKey(), sysValue);
239 }
240 }
241 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
242 String name = (String) entry.getKey();
243 if (!IGNORE_SYS_PROPS.contains(name)) {
244 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
245 if (configuration.get(name) == null) {
246 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
247 }
248 }
249 }
250 }
251
252 return new LogChangesConfiguration(configuration);
253 }
254
255 private class LogChangesConfiguration extends XConfiguration {
256
257 public LogChangesConfiguration(Configuration conf) {
258 for (Map.Entry<String, String> entry : conf) {
259 if (get(entry.getKey()) == null) {
260 setValue(entry.getKey(), entry.getValue());
261 }
262 }
263 }
264
265 public String[] getStrings(String name) {
266 String s = get(name);
267 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
268 }
269
270 public String get(String name, String defaultValue) {
271 String value = get(name);
272 if (value == null) {
273 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
274 value = defaultValue;
275 String logValue = (maskValue) ? "**MASKED**" : defaultValue;
276 log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
277 }
278 return value;
279 }
280
281 public void set(String name, String value) {
282 setValue(name, value);
283 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
284 value = (maskValue) ? "**MASKED**" : value;
285 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
286 }
287
288 private void setValue(String name, String value) {
289 super.set(name, value);
290 }
291
292 }
293
294 /**
295 * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
296 * config file used.
297 *
298 * @param instr instrumentation to use.
299 */
300 public void instrument(Instrumentation instr) {
301 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
302 public String getValue() {
303 return configDir;
304 }
305 });
306 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
307 public String getValue() {
308 return configFile;
309 }
310 });
311 instr.setConfiguration(configuration);
312 }
313
314 /**
315 * Return a configuration with all sensitive values masked.
316 *
317 * @param conf configuration to mask.
318 * @return masked configuration.
319 */
320 public static Configuration maskPasswords(Configuration conf) {
321 XConfiguration maskedConf = new XConfiguration();
322 for (Map.Entry<String, String> entry : conf) {
323 String name = entry.getKey();
324 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
325 String value = (maskValue) ? "**MASKED**" : entry.getValue();
326 maskedConf.set(name, value);
327 }
328 return maskedConf;
329 }
330
331 }