001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.util.Instrumentable;
022    import org.apache.oozie.util.Instrumentation;
023    import org.apache.oozie.util.XLog;
024    import org.apache.oozie.util.XConfiguration;
025    import org.apache.oozie.ErrorCode;
026    
027    import java.io.File;
028    import java.io.FileInputStream;
029    import java.io.IOException;
030    import java.io.InputStream;
031    import java.io.StringWriter;
032    import java.util.HashSet;
033    import java.util.Map;
034    import java.util.Set;
035    import java.util.Arrays;
036    
037    /**
038     * Built in service that initializes the services configuration.
039     * <p/>
040     * The configuration loading sequence is identical to Hadoop configuration loading sequence.
041     * <p/>
042     * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
043     * are loaded from a site configuration file from the Oozie configuration directory.
044     * <p/>
045     * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
046     * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
047     * initialization of the <code>ConfigurationService</code> fails.
048     * <p/>
049     * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
050     * <p/>
051     * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
052     * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
053     * <p/>
054     * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
055     * <p/>
056     * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
057     * via system properties settings.
058     */
059    public class ConfigurationService implements Service, Instrumentable {
060        private static final String INSTRUMENTATION_GROUP = "configuration";
061    
062        public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
063    
064        public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
065    
066        /**
067         * System property that indicates the configuration directory.
068         */
069        public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
070    
071    
072        /**
073         * System property that indicates the data directory.
074         */
075        public static final String OOZIE_DATA_DIR = "oozie.data.dir";
076    
077        /**
078         * System property that indicates the name of the site configuration file to load.
079         */
080        public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
081    
082        private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
083        private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
084    
085        private static final String PASSWORD_PROPERTY_END = ".password";
086    
087        static {
088            IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
089    
090            //all this properties are seeded as system properties, no need to log changes
091            IGNORE_SYS_PROPS.add("oozie.http.hostname");
092            IGNORE_SYS_PROPS.add("oozie.http.port");
093    
094            IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
095            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
096            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
097            IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
098    
099            IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
100            IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
101            IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
102        }
103    
104        public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
105        public static final String SITE_CONFIG_FILE = "oozie-site.xml";
106    
107        private static XLog log = XLog.getLog(ConfigurationService.class);
108    
109        private String configDir;
110        private String configFile;
111    
112        private LogChangesConfiguration configuration;
113    
114        public ConfigurationService() {
115            log = XLog.getLog(ConfigurationService.class);
116        }
117    
118        /**
119         * Initialize the log service.
120         *
121         * @param services services instance.
122         * @throws ServiceException thrown if the log service could not be initialized.
123         */
124        public void init(Services services) throws ServiceException {
125            configDir = getConfigurationDirectory();
126            configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
127            if (configFile.contains("/")) {
128                throw new ServiceException(ErrorCode.E0022, configFile);
129            }
130            log.info("Oozie home dir  [{0}]", Services.getOozieHome());
131            log.info("Oozie conf dir  [{0}]", configDir);
132            log.info("Oozie conf file [{0}]", configFile);
133            configFile = new File(configDir, configFile).toString();
134            configuration = loadConf();
135        }
136    
137        public static String getConfigurationDirectory() throws ServiceException {
138            String oozieHome = Services.getOozieHome();
139            String configDir = System.getProperty(OOZIE_CONFIG_DIR, oozieHome + "/conf");
140            File file = new File(configDir);
141            if (!file.exists()) {
142                throw new ServiceException(ErrorCode.E0024, configDir);
143            }
144            return configDir;
145        }
146    
147        /**
148         * Destroy the configuration service.
149         */
150        public void destroy() {
151            configuration = null;
152        }
153    
154        /**
155         * Return the public interface for configuration service.
156         *
157         * @return {@link ConfigurationService}.
158         */
159        public Class<? extends Service> getInterface() {
160            return ConfigurationService.class;
161        }
162    
163        /**
164         * Return the services configuration.
165         *
166         * @return the services configuration.
167         */
168        public Configuration getConf() {
169            if (configuration == null) {
170                throw new IllegalStateException("Not initialized");
171            }
172            return configuration;
173        }
174    
175        /**
176         * Return Oozie configuration directory.
177         *
178         * @return Oozie configuration directory.
179         */
180        public String getConfigDir() {
181            return configDir;
182        }
183    
184        private InputStream getDefaultConfiguration() throws ServiceException, IOException {
185            ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
186            InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
187            if (inputStream == null) {
188                throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
189            }
190            return inputStream;
191        }
192    
193        private LogChangesConfiguration loadConf() throws ServiceException {
194            XConfiguration configuration;
195            try {
196                InputStream inputStream = getDefaultConfiguration();
197                configuration = new XConfiguration(inputStream);
198                File file = new File(configFile);
199                if (!file.exists()) {
200                    log.info("Missing site configuration file [{0}]", configFile);
201                }
202                else {
203                    inputStream = new FileInputStream(configFile);
204                    XConfiguration siteConfiguration = new XConfiguration(inputStream);
205                    XConfiguration.injectDefaults(configuration, siteConfiguration);
206                    configuration = siteConfiguration;
207                }
208            }
209            catch (IOException ex) {
210                throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
211            }
212    
213            if (log.isTraceEnabled()) {
214                try {
215                    StringWriter writer = new StringWriter();
216                    for (Map.Entry<String, String> entry : configuration) {
217                        boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
218                        String value = (maskValue) ? "**MASKED**" : entry.getValue();
219                        writer.write(" " + entry.getKey() + " = " + value + "\n");
220                    }
221                    writer.close();
222                    log.trace("Configuration:\n{0}---", writer.toString());
223                }
224                catch (IOException ex) {
225                    throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
226                }
227            }
228    
229            String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
230            if (ignoreSysProps != null) {
231                IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
232            }
233    
234            for (Map.Entry<String, String> entry : configuration) {
235                String sysValue = System.getProperty(entry.getKey());
236                if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
237                    log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
238                    configuration.set(entry.getKey(), sysValue);
239                }
240            }
241            for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
242                String name = (String) entry.getKey();
243                if (!IGNORE_SYS_PROPS.contains(name)) {
244                    if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
245                        if (configuration.get(name) == null) {
246                            log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
247                        }
248                    }
249                }
250            }
251    
252            return new LogChangesConfiguration(configuration);
253        }
254    
255        private class LogChangesConfiguration extends XConfiguration {
256    
257            public LogChangesConfiguration(Configuration conf) {
258                for (Map.Entry<String, String> entry : conf) {
259                    if (get(entry.getKey()) == null) {
260                        setValue(entry.getKey(), entry.getValue());
261                    }
262                }
263            }
264    
265            public String[] getStrings(String name) {
266                String s = get(name);
267                return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
268            }
269    
270            public String get(String name, String defaultValue) {
271                String value = get(name);
272                if (value == null) {
273                    boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
274                    value = defaultValue;
275                    String logValue = (maskValue) ? "**MASKED**" : defaultValue;
276                    log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
277                }
278                return value;
279            }
280    
281            public void set(String name, String value) {
282                setValue(name, value);
283                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
284                value = (maskValue) ? "**MASKED**" : value;
285                log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
286            }
287    
288            private void setValue(String name, String value) {
289                super.set(name, value);
290            }
291    
292        }
293    
294        /**
295         * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
296         * config file used.
297         *
298         * @param instr instrumentation to use.
299         */
300        public void instrument(Instrumentation instr) {
301            instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
302                public String getValue() {
303                    return configDir;
304                }
305            });
306            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
307                public String getValue() {
308                    return configFile;
309                }
310            });
311            instr.setConfiguration(configuration);
312        }
313    
314        /**
315         * Return a configuration with all sensitive values masked.
316         *
317         * @param conf configuration to mask.
318         * @return masked configuration.
319         */
320        public static Configuration maskPasswords(Configuration conf) {
321            XConfiguration maskedConf = new XConfiguration();
322            for (Map.Entry<String, String> entry : conf) {
323                String name = entry.getKey();
324                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
325                String value = (maskValue) ? "**MASKED**" : entry.getValue();
326                maskedConf.set(name, value);
327            }
328            return maskedConf;
329        }
330    
331    }