001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import org.apache.commons.logging.LogFactory;
021 import org.apache.log4j.LogManager;
022 import org.apache.log4j.PropertyConfigurator;
023 import org.apache.oozie.util.Instrumentable;
024 import org.apache.oozie.util.Instrumentation;
025 import org.apache.oozie.util.XLog;
026 import org.apache.oozie.util.XLogStreamer;
027 import org.apache.oozie.util.XConfiguration;
028 import org.apache.oozie.BuildInfo;
029 import org.apache.oozie.ErrorCode;
030 import org.apache.hadoop.conf.Configuration;
031
032 import java.io.File;
033 import java.io.FileInputStream;
034 import java.io.IOException;
035 import java.io.InputStream;
036 import java.io.Writer;
037 import java.net.URL;
038 import java.util.Properties;
039 import java.util.Map;
040 import java.util.Date;
041 import java.util.regex.Pattern;
042
043 /**
044 * Built-in service that initializes and manages Logging via Log4j.
045 * <p/>
046 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
047 * <p/>
048 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
049 * <p/>
050 * The Log4j configuration files must be a properties file.
051 * <p/>
052 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
053 * If the file is not found there, it is looked in the classpath.
054 * <p/>
055 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
056 * <p/>
057 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
058 * <p/>
059 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default
060 * value is 10 seconds.
061 */
062 public class XLogService implements Service, Instrumentable {
063 private static final String INSTRUMENTATION_GROUP = "logging";
064
065 /**
066 * System property that indicates the logs directory.
067 */
068 public static final String OOZIE_LOG_DIR = "oozie.log.dir";
069
070 /**
071 * System property that indicates the log4j configuration file to load.
072 */
073 public static final String LOG4J_FILE = "oozie.log4j.file";
074
075 /**
076 * System property that indicates the reload interval of the configuration file.
077 */
078 public static final String LOG4J_RELOAD = "oozie.log4j.reload";
079
080 /**
081 * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set.
082 */
083 public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
084
085 /**
086 * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set.
087 */
088 public static final String DEFAULT_RELOAD_INTERVAL = "10";
089
090 private XLog log;
091 private long interval;
092 private boolean fromClasspath;
093 private String log4jFileName;
094 private boolean logOverWS = true;
095
096 private static final String STARTUP_MESSAGE = "{E}"
097 + " ******************************************************************************* {E}"
098 + " STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
099 + " STARTUP MSG: revision [{3}]@[{4}]{E}"
100 + "*******************************************************************************";
101
102 private String oozieLogPath;
103 private String oozieLogName;
104 private int oozieLogRotation = -1;
105
106 public XLogService() {
107 }
108
109 public String getOozieLogPath() {
110 return oozieLogPath;
111 }
112
113 public String getOozieLogName() {
114 return oozieLogName;
115 }
116
117 /**
118 * Initialize the log service.
119 *
120 * @param services services instance.
121 * @throws ServiceException thrown if the log service could not be initialized.
122 */
123 public void init(Services services) throws ServiceException {
124 String oozieHome = Services.getOozieHome();
125 String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs");
126 System.setProperty(OOZIE_LOG_DIR, oozieLogs);
127 try {
128 LogManager.resetConfiguration();
129 log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES);
130 if (log4jFileName.contains("/")) {
131 throw new ServiceException(ErrorCode.E0011, log4jFileName);
132 }
133 if (!log4jFileName.endsWith(".properties")) {
134 throw new ServiceException(ErrorCode.E0012, log4jFileName);
135 }
136 String configPath = ConfigurationService.getConfigurationDirectory();
137 File log4jFile = new File(configPath, log4jFileName);
138 if (log4jFile.exists()) {
139 fromClasspath = false;
140 }
141 else {
142 ClassLoader cl = Thread.currentThread().getContextClassLoader();
143 URL log4jUrl = cl.getResource(log4jFileName);
144 if (log4jUrl == null) {
145 throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
146 }
147 fromClasspath = true;
148 }
149
150 if (fromClasspath) {
151 ClassLoader cl = Thread.currentThread().getContextClassLoader();
152 URL log4jUrl = cl.getResource(log4jFileName);
153 PropertyConfigurator.configure(log4jUrl);
154 }
155 else {
156 interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
157 PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
158 }
159
160 log = new XLog(LogFactory.getLog(getClass()));
161
162 log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
163 BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
164 .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
165 BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
166
167 String from = (fromClasspath) ? "CLASSPATH" : configPath;
168 String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
169 log.info("Log4j configuration file [{0}]", log4jFileName);
170 log.info("Log4j configuration file loaded from [{0}]", from);
171 log.info("Log4j reload interval [{0}]", reload);
172
173 XLog.Info.reset();
174 XLog.Info.defineParameter(USER);
175 XLog.Info.defineParameter(GROUP);
176 XLogStreamer.Filter.reset();
177 XLogStreamer.Filter.defineParameter(USER);
178 XLogStreamer.Filter.defineParameter(GROUP);
179
180 // Getting configuration for oozie log via WS
181 ClassLoader cl = Thread.currentThread().getContextClassLoader();
182 InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
183 extractInfoForLogWebService(is);
184 }
185 catch (IOException ex) {
186 throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
187 }
188 }
189
190 private void extractInfoForLogWebService(InputStream is) throws IOException {
191 Properties props = new Properties();
192 props.load(is);
193
194 Configuration conf = new XConfiguration();
195 for (Map.Entry entry : props.entrySet()) {
196 conf.set((String) entry.getKey(), (String) entry.getValue());
197 }
198 String logFile = conf.get("log4j.appender.oozie.File");
199 if (logFile == null) {
200 log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' "
201 + "appender");
202 logOverWS = false;
203 }
204 else {
205 logFile = logFile.trim();
206 int i = logFile.lastIndexOf("/");
207 if (i == -1) {
208 log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
209 logFile);
210 logOverWS = false;
211 }
212 else {
213 String appenderClass = conf.get("log4j.appender.oozie");
214 if (appenderClass == null) {
215 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie]");
216 logOverWS = false;
217 }
218 else if (appenderClass.equals("org.apache.log4j.DailyRollingFileAppender")) {
219 String pattern = conf.get("log4j.appender.oozie.DatePattern");
220 if (pattern == null) {
221 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
222 logOverWS = false;
223 }
224 else {
225 pattern = pattern.trim();
226 if (pattern.endsWith("HH")) {
227 oozieLogRotation = 60 * 60;
228 }
229 else if (pattern.endsWith("dd")) {
230 oozieLogRotation = 60 * 60 * 24;
231 }
232 else {
233 log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
234 pattern);
235 logOverWS = false;
236 }
237 if (oozieLogRotation > 0) {
238 oozieLogPath = logFile.substring(0, i);
239 oozieLogName = logFile.substring(i + 1);
240 }
241 }
242 }
243 else if (appenderClass.equals("org.apache.log4j.rolling.RollingFileAppender")) {
244 String pattern = conf.get("log4j.appender.oozie.RollingPolicy.FileNamePattern");
245 if (pattern == null) {
246 log.warn("Oozie WS log will be disabled, missing property "
247 + "[log4j.appender.oozie.RollingPolicy.FileNamePattern]");
248 logOverWS = false;
249 }
250 else {
251 pattern = pattern.trim();
252 if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd-HH\\}(\\.gz)?")) {
253 oozieLogRotation = 60 * 60;
254 }
255 else {
256 log.warn("Oozie WS log will be disabled, RollingPolicy.FileNamePattern [{0}] should end with "
257 + "'-%d{yyyy-MM-dd-HH}' or '-%d{yyyy-MM-dd-HH}.gz' and also start with the value of "
258 + "log4j.appender.oozie.File [{1}]", pattern, logFile);
259 logOverWS = false;
260 }
261 if (oozieLogRotation > 0) {
262 oozieLogPath = logFile.substring(0, i);
263 oozieLogName = logFile.substring(i + 1);
264 }
265 }
266 }
267 else {
268 log.warn("Oozie WS log will be disabled, log4j.appender.oozie [" + appenderClass + "] should be "
269 + "either org.apache.log4j.DailyRollingFileAppender or org.apache.log4j.rolling.RollingFileAppender "
270 + "to enable it");
271 logOverWS = false;
272 }
273 }
274 }
275 }
276
277 /**
278 * Destroy the log service.
279 */
280 public void destroy() {
281 LogManager.shutdown();
282 XLog.Info.reset();
283 XLogStreamer.Filter.reset();
284 }
285
286 /**
287 * Group log info constant.
288 */
289 public static final String USER = "USER";
290
291 /**
292 * Group log info constant.
293 */
294 public static final String GROUP = "GROUP";
295
296 /**
297 * Return the public interface for log service.
298 *
299 * @return {@link XLogService}.
300 */
301 public Class<? extends Service> getInterface() {
302 return XLogService.class;
303 }
304
305 /**
306 * Instruments the log service.
307 * <p/>
308 * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath.
309 *
310 * @param instr instrumentation to use.
311 */
312 public void instrument(Instrumentation instr) {
313 instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
314 public String getValue() {
315 return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
316 }
317 });
318 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
319 public String getValue() {
320 return log4jFileName;
321 }
322 });
323 instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
324 public Long getValue() {
325 return interval;
326 }
327 });
328 instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
329 public Boolean getValue() {
330 return fromClasspath;
331 }
332 });
333 instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
334 public Boolean getValue() {
335 return logOverWS;
336 }
337 });
338 }
339
340 /**
341 * Stream the log of a job.
342 *
343 * @param filter log streamer filter.
344 * @param startTime start time for log events to filter.
345 * @param endTime end time for log events to filter.
346 * @param writer writer to stream the log to.
347 * @throws IOException thrown if the log cannot be streamed.
348 */
349 public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
350 if (logOverWS) {
351 new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
352 .streamLog(startTime, endTime);
353 }
354 else {
355 writer.write("Log streaming disabled!!");
356 }
357
358 }
359
360 String getLog4jProperties() {
361 return log4jFileName;
362 }
363
364 boolean getFromClasspath() {
365 return fromClasspath;
366 }
367
368 }