Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * 
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 * 
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.service;
019
020import org.apache.hadoop.conf.Configuration;
021import org.apache.oozie.util.Instrumentable;
022import org.apache.oozie.util.Instrumentation;
023import org.apache.oozie.util.XLog;
024import org.apache.oozie.util.XConfiguration;
025import org.apache.oozie.ErrorCode;
026
027import java.io.File;
028import java.io.FileInputStream;
029import java.io.IOException;
030import java.io.InputStream;
031import java.io.StringWriter;
032import java.util.HashSet;
033import java.util.Map;
034import java.util.Set;
035import java.util.Arrays;
036import org.apache.oozie.util.ZKUtils;
037
038/**
039 * Built in service that initializes the services configuration.
040 * <p/>
041 * The configuration loading sequence is identical to Hadoop configuration loading sequence.
042 * <p/>
043 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
044 * are loaded from a site configuration file from the Oozie configuration directory.
045 * <p/>
046 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
047 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
048 * initialization of the <code>ConfigurationService</code> fails.
049 * <p/>
050 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
051 * <p/>
052 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
053 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
054 * <p/>
055 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
056 * <p/>
057 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
058 * via system properties settings.
059 */
060public class ConfigurationService implements Service, Instrumentable {
061    private static final String INSTRUMENTATION_GROUP = "configuration";
062
063    public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
064
065    public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
066
067    /**
068     * System property that indicates the configuration directory.
069     */
070    public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
071
072
073    /**
074     * System property that indicates the data directory.
075     */
076    public static final String OOZIE_DATA_DIR = "oozie.data.dir";
077
078    /**
079     * System property that indicates the name of the site configuration file to load.
080     */
081    public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
082
083    private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
084    private static final Set<String> CONF_SYS_PROPS = new HashSet<String>();
085
086    private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
087    private static final Set<String> MASK_PROPS = new HashSet<String>();
088
089    static {
090
091        //all this properties are seeded as system properties, no need to log changes
092        IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
093        IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
094        IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
095        IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
096        IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
097        IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
098        IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
099        IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
100
101        CONF_SYS_PROPS.add("oozie.http.hostname");
102        CONF_SYS_PROPS.add("oozie.http.port");
103        CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
104
105        // These properties should be masked when displayed because they contain sensitive info (e.g. password)
106        MASK_PROPS.add(JPAService.CONF_PASSWORD);
107        MASK_PROPS.add("oozie.authentication.signature.secret");
108    }
109
110    public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
111    public static final String SITE_CONFIG_FILE = "oozie-site.xml";
112
113    private static XLog log = XLog.getLog(ConfigurationService.class);
114
115    private String configDir;
116    private String configFile;
117
118    private LogChangesConfiguration configuration;
119
120    public ConfigurationService() {
121        log = XLog.getLog(ConfigurationService.class);
122    }
123
124    /**
125     * Initialize the log service.
126     *
127     * @param services services instance.
128     * @throws ServiceException thrown if the log service could not be initialized.
129     */
130    public void init(Services services) throws ServiceException {
131        configDir = getConfigurationDirectory();
132        configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
133        if (configFile.contains("/")) {
134            throw new ServiceException(ErrorCode.E0022, configFile);
135        }
136        log.info("Oozie home dir  [{0}]", Services.getOozieHome());
137        log.info("Oozie conf dir  [{0}]", configDir);
138        log.info("Oozie conf file [{0}]", configFile);
139        configFile = new File(configDir, configFile).toString();
140        configuration = loadConf();
141    }
142
143    public static String getConfigurationDirectory() throws ServiceException {
144        String oozieHome = Services.getOozieHome();
145        String configDir = System.getProperty(OOZIE_CONFIG_DIR, null);
146        File file = configDir == null
147                ? new File(oozieHome, "conf")
148                : new File(configDir);
149        if (!file.exists()) {
150            throw new ServiceException(ErrorCode.E0024, configDir);
151        }
152        return file.getPath();
153    }
154
155    /**
156     * Destroy the configuration service.
157     */
158    public void destroy() {
159        configuration = null;
160    }
161
162    /**
163     * Return the public interface for configuration service.
164     *
165     * @return {@link ConfigurationService}.
166     */
167    public Class<? extends Service> getInterface() {
168        return ConfigurationService.class;
169    }
170
171    /**
172     * Return the services configuration.
173     *
174     * @return the services configuration.
175     */
176    public Configuration getConf() {
177        if (configuration == null) {
178            throw new IllegalStateException("Not initialized");
179        }
180        return configuration;
181    }
182
183    /**
184     * Return Oozie configuration directory.
185     *
186     * @return Oozie configuration directory.
187     */
188    public String getConfigDir() {
189        return configDir;
190    }
191
192    private InputStream getDefaultConfiguration() throws ServiceException, IOException {
193        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
194        InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
195        if (inputStream == null) {
196            throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
197        }
198        return inputStream;
199    }
200
201    private LogChangesConfiguration loadConf() throws ServiceException {
202        XConfiguration configuration;
203        try {
204            InputStream inputStream = getDefaultConfiguration();
205            configuration = new XConfiguration(inputStream);
206            File file = new File(configFile);
207            if (!file.exists()) {
208                log.info("Missing site configuration file [{0}]", configFile);
209            }
210            else {
211                inputStream = new FileInputStream(configFile);
212                XConfiguration siteConfiguration = new XConfiguration(inputStream);
213                XConfiguration.injectDefaults(configuration, siteConfiguration);
214                configuration = siteConfiguration;
215            }
216        }
217        catch (IOException ex) {
218            throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
219        }
220
221        if (log.isTraceEnabled()) {
222            try {
223                StringWriter writer = new StringWriter();
224                for (Map.Entry<String, String> entry : configuration) {
225                    String value = getValue(configuration, entry.getKey());
226                    writer.write(" " + entry.getKey() + " = " + value + "\n");
227                }
228                writer.close();
229                log.trace("Configuration:\n{0}---", writer.toString());
230            }
231            catch (IOException ex) {
232                throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
233            }
234        }
235
236        String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
237        if (ignoreSysProps != null) {
238            IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
239        }
240
241        for (Map.Entry<String, String> entry : configuration) {
242            String sysValue = System.getProperty(entry.getKey());
243            if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
244                log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
245                configuration.set(entry.getKey(), sysValue);
246            }
247        }
248        for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
249            String name = (String) entry.getKey();
250            if (!IGNORE_SYS_PROPS.contains(name)) {
251                if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
252                    if (configuration.get(name) == null) {
253                        log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
254                    }
255                }
256            }
257        }
258
259        //Backward compatible, we should still support -Dparam.
260        for (String key : CONF_SYS_PROPS) {
261            String sysValue = System.getProperty(key);
262            if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
263                log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue);
264                configuration.set(key, sysValue);
265            }
266        }
267
268        return new LogChangesConfiguration(configuration);
269    }
270
271    private class LogChangesConfiguration extends XConfiguration {
272
273        public LogChangesConfiguration(Configuration conf) {
274            for (Map.Entry<String, String> entry : conf) {
275                if (get(entry.getKey()) == null) {
276                    setValue(entry.getKey(), entry.getValue());
277                }
278            }
279        }
280
281        public String[] getStrings(String name) {
282            String s = get(name);
283            return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
284        }
285
286        public String get(String name, String defaultValue) {
287            String value = get(name);
288            if (value == null) {
289                boolean maskValue = MASK_PROPS.contains(name);
290                value = defaultValue;
291                String logValue = (maskValue) ? "**MASKED**" : defaultValue;
292                log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
293            }
294            return value;
295        }
296
297        public void set(String name, String value) {
298            setValue(name, value);
299            boolean maskValue = MASK_PROPS.contains(name);
300            value = (maskValue) ? "**MASKED**" : value;
301            log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
302        }
303
304        private void setValue(String name, String value) {
305            super.set(name, value);
306        }
307
308    }
309
310    /**
311     * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
312     * config file used.
313     *
314     * @param instr instrumentation to use.
315     */
316    public void instrument(Instrumentation instr) {
317        instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
318            public String getValue() {
319                return configDir;
320            }
321        });
322        instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
323            public String getValue() {
324                return configFile;
325            }
326        });
327    }
328
329    /**
330     * Return a configuration with all sensitive values masked.
331     *
332     * @return masked configuration.
333     */
334    public Configuration getMaskedConfiguration() {
335        XConfiguration maskedConf = new XConfiguration();
336        Configuration conf = getConf();
337        for (Map.Entry<String, String> entry : conf) {
338            String name = entry.getKey();
339            String value = getValue(conf, name);
340            maskedConf.set(name, value);
341        }
342        return maskedConf;
343    }
344
345    private String getValue(Configuration config, String key) {
346        String value;
347        if (MASK_PROPS.contains(key)) {
348            value = "**MASKED**";
349        }
350        else {
351            value = config.get(key);
352        }
353        return value;
354    }
355}