001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.commons.logging.LogFactory;
021    import org.apache.log4j.LogManager;
022    import org.apache.log4j.PropertyConfigurator;
023    import org.apache.oozie.util.Instrumentable;
024    import org.apache.oozie.util.Instrumentation;
025    import org.apache.oozie.util.XLog;
026    import org.apache.oozie.util.XLogStreamer;
027    import org.apache.oozie.util.XConfiguration;
028    import org.apache.oozie.BuildInfo;
029    import org.apache.oozie.ErrorCode;
030    import org.apache.hadoop.conf.Configuration;
031    
032    import java.io.File;
033    import java.io.FileInputStream;
034    import java.io.IOException;
035    import java.io.InputStream;
036    import java.io.Writer;
037    import java.net.URL;
038    import java.util.Properties;
039    import java.util.Map;
040    import java.util.Date;
041    
042    /**
043     * Built-in service that initializes and manages Logging via Log4j.
044     * <p/>
045     * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
046     * <p/>
047     * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
048     * <p/>
049     * The Log4j configuration files must be a properties file.
050     * <p/>
051     * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
052     * If the file is not found there, it is looked in the classpath.
053     * <p/>
054     * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
055     * <p/>
056     * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
057     * <p/>
058     * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default
059     * value is 10 seconds.
060     */
061    public class XLogService implements Service, Instrumentable {
062        private static final String INSTRUMENTATION_GROUP = "logging";
063    
064        /**
065         * System property that indicates the logs directory.
066         */
067        public static final String OOZIE_LOG_DIR = "oozie.log.dir";
068    
069        /**
070         * System property that indicates the log4j configuration file to load.
071         */
072        public static final String LOG4J_FILE = "oozie.log4j.file";
073    
074        /**
075         * System property that indicates the reload interval of the configuration file.
076         */
077        public static final String LOG4J_RELOAD = "oozie.log4j.reload";
078    
079        /**
080         * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set.
081         */
082        public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
083    
084        /**
085         * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set.
086         */
087        public static final String DEFAULT_RELOAD_INTERVAL = "10";
088    
089        private XLog log;
090        private long interval;
091        private boolean fromClasspath;
092        private String log4jFileName;
093        private boolean logOverWS = true;
094    
095        private static final String STARTUP_MESSAGE = "{E}"
096                + " ******************************************************************************* {E}"
097                + "  STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
098                + "  STARTUP MSG:       revision [{3}]@[{4}]{E}"
099                + "*******************************************************************************";
100    
101        private String oozieLogPath;
102        private String oozieLogName;
103        private int oozieLogRotation = -1;
104    
105        public XLogService() {
106        }
107    
108        /**
109         * Initialize the log service.
110         *
111         * @param services services instance.
112         * @throws ServiceException thrown if the log service could not be initialized.
113         */
114        public void init(Services services) throws ServiceException {
115            String oozieHome = Services.getOozieHome();
116            String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs");
117            System.setProperty(OOZIE_LOG_DIR, oozieLogs);
118            try {
119                LogManager.resetConfiguration();
120                log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES);
121                if (log4jFileName.contains("/")) {
122                    throw new ServiceException(ErrorCode.E0011, log4jFileName);
123                }
124                if (!log4jFileName.endsWith(".properties")) {
125                    throw new ServiceException(ErrorCode.E0012, log4jFileName);
126                }
127                String configPath = ConfigurationService.getConfigurationDirectory();
128                File log4jFile = new File(configPath, log4jFileName);
129                if (log4jFile.exists()) {
130                    fromClasspath = false;
131                }
132                else {
133                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
134                    URL log4jUrl = cl.getResource(log4jFileName);
135                    if (log4jUrl == null) {
136                        throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
137                    }
138                    fromClasspath = true;
139                }
140    
141                if (fromClasspath) {
142                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
143                    URL log4jUrl = cl.getResource(log4jFileName);
144                    PropertyConfigurator.configure(log4jUrl);
145                }
146                else {
147                    interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
148                    PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
149                }
150    
151                log = new XLog(LogFactory.getLog(getClass()));
152    
153                log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
154                        BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
155                                .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
156                                BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
157    
158                String from = (fromClasspath) ? "CLASSPATH" : configPath;
159                String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
160                log.info("Log4j configuration file [{0}]", log4jFileName);
161                log.info("Log4j configuration file loaded from [{0}]", from);
162                log.info("Log4j reload interval [{0}]", reload);
163    
164                XLog.Info.reset();
165                XLog.Info.defineParameter(USER);
166                XLog.Info.defineParameter(GROUP);
167                XLogStreamer.Filter.reset();
168                XLogStreamer.Filter.defineParameter(USER);
169                XLogStreamer.Filter.defineParameter(GROUP);
170    
171                // Getting configuration for oozie log via WS
172                ClassLoader cl = Thread.currentThread().getContextClassLoader();
173                InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
174                extractInfoForLogWebService(is);
175            }
176            catch (IOException ex) {
177                throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
178            }
179        }
180    
181        private void extractInfoForLogWebService(InputStream is) throws IOException {
182            Properties props = new Properties();
183            props.load(is);
184    
185            Configuration conf = new XConfiguration();
186            for (Map.Entry entry : props.entrySet()) {
187                conf.set((String) entry.getKey(), (String) entry.getValue());
188            }
189            String logFile = conf.get("log4j.appender.oozie.File");
190            if (logFile == null) {
191                log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' "
192                        + "appender");
193            }
194            else {
195                logFile = logFile.trim();
196                int i = logFile.lastIndexOf("/");
197                if (i == -1) {
198                    log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
199                            logFile);
200                    logOverWS = false;
201                }
202                else {
203                    String pattern = conf.get("log4j.appender.oozie.DatePattern");
204                    if (pattern == null) {
205                        log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
206                        logOverWS = false;
207                    }
208                    else {
209                        pattern = pattern.trim();
210                        if (pattern.endsWith("HH")) {
211                            oozieLogRotation = 60 * 60;
212                        }
213                        else {
214                            if (pattern.endsWith("dd")) {
215                                oozieLogRotation = 60 * 60 * 24;
216                            }
217                            else {
218                                log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
219                                        pattern);
220                                logOverWS = false;
221                            }
222                        }
223                        if (oozieLogRotation > 0) {
224                            oozieLogPath = logFile.substring(0, i);
225                            oozieLogName = logFile.substring(i + 1);
226                        }
227                        else {
228                            logOverWS = false;
229                        }
230                    }
231                }
232            }
233        }
234    
235        /**
236         * Destroy the log service.
237         */
238        public void destroy() {
239            LogManager.shutdown();
240            XLog.Info.reset();
241            XLogStreamer.Filter.reset();
242        }
243    
244        /**
245         * Group log info constant.
246         */
247        public static final String USER = "USER";
248    
249        /**
250         * Group log info constant.
251         */
252        public static final String GROUP = "GROUP";
253    
254        /**
255         * Return the public interface for log service.
256         *
257         * @return {@link XLogService}.
258         */
259        public Class<? extends Service> getInterface() {
260            return XLogService.class;
261        }
262    
263        /**
264         * Instruments the log service.
265         * <p/>
266         * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath.
267         *
268         * @param instr instrumentation to use.
269         */
270        public void instrument(Instrumentation instr) {
271            instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
272                public String getValue() {
273                    return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
274                }
275            });
276            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
277                public String getValue() {
278                    return log4jFileName;
279                }
280            });
281            instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
282                public Long getValue() {
283                    return interval;
284                }
285            });
286            instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
287                public Boolean getValue() {
288                    return fromClasspath;
289                }
290            });
291            instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
292                public Boolean getValue() {
293                    return logOverWS;
294                }
295            });
296        }
297    
298        /**
299         * Stream the log of a job.
300         *
301         * @param filter log streamer filter.
302         * @param startTime start time for log events to filter.
303         * @param endTime end time for log events to filter.
304         * @param writer writer to stream the log to.
305         * @throws IOException thrown if the log cannot be streamed.
306         */
307        public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
308            if (logOverWS) {
309                new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
310                        .streamLog(startTime, endTime);
311            }
312            else {
313                writer.write("Log streaming disabled!!");
314            }
315    
316        }
317    
318        String getLog4jProperties() {
319            return log4jFileName;
320        }
321    
322        boolean getFromClasspath() {
323            return fromClasspath;
324        }
325    
326    }