001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.oozie.service; 019 020 import org.apache.commons.logging.LogFactory; 021 import org.apache.log4j.LogManager; 022 import org.apache.log4j.PropertyConfigurator; 023 import org.apache.oozie.util.Instrumentable; 024 import org.apache.oozie.util.Instrumentation; 025 import org.apache.oozie.util.XLog; 026 import org.apache.oozie.util.XLogStreamer; 027 import org.apache.oozie.util.XConfiguration; 028 import org.apache.oozie.BuildInfo; 029 import org.apache.oozie.ErrorCode; 030 import org.apache.hadoop.conf.Configuration; 031 032 import java.io.File; 033 import java.io.FileInputStream; 034 import java.io.IOException; 035 import java.io.InputStream; 036 import java.io.Writer; 037 import java.net.URL; 038 import java.util.Properties; 039 import java.util.Map; 040 import java.util.Date; 041 042 /** 043 * Built-in service that initializes and manages Logging via Log4j. 044 * <p/> 045 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>. 046 * <p/> 047 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>. 048 * <p/> 049 * The Log4j configuration files must be a properties file. 050 * <p/> 051 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}. 052 * If the file is not found there, it is looked in the classpath. 053 * <p/> 054 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled. 055 * <p/> 056 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled. 057 * <p/> 058 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default 059 * value is 10 seconds. 060 */ 061 public class XLogService implements Service, Instrumentable { 062 private static final String INSTRUMENTATION_GROUP = "logging"; 063 064 /** 065 * System property that indicates the logs directory. 066 */ 067 public static final String OOZIE_LOG_DIR = "oozie.log.dir"; 068 069 /** 070 * System property that indicates the log4j configuration file to load. 071 */ 072 public static final String LOG4J_FILE = "oozie.log4j.file"; 073 074 /** 075 * System property that indicates the reload interval of the configuration file. 076 */ 077 public static final String LOG4J_RELOAD = "oozie.log4j.reload"; 078 079 /** 080 * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set. 081 */ 082 public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties"; 083 084 /** 085 * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set. 086 */ 087 public static final String DEFAULT_RELOAD_INTERVAL = "10"; 088 089 private XLog log; 090 private long interval; 091 private boolean fromClasspath; 092 private String log4jFileName; 093 private boolean logOverWS = true; 094 095 private static final String STARTUP_MESSAGE = "{E}" 096 + " ******************************************************************************* {E}" 097 + " STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}" 098 + " STARTUP MSG: revision [{3}]@[{4}]{E}" 099 + "*******************************************************************************"; 100 101 private String oozieLogPath; 102 private String oozieLogName; 103 private int oozieLogRotation = -1; 104 105 public XLogService() { 106 } 107 108 /** 109 * Initialize the log service. 110 * 111 * @param services services instance. 112 * @throws ServiceException thrown if the log service could not be initialized. 113 */ 114 public void init(Services services) throws ServiceException { 115 String oozieHome = Services.getOozieHome(); 116 String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs"); 117 System.setProperty(OOZIE_LOG_DIR, oozieLogs); 118 try { 119 LogManager.resetConfiguration(); 120 log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES); 121 if (log4jFileName.contains("/")) { 122 throw new ServiceException(ErrorCode.E0011, log4jFileName); 123 } 124 if (!log4jFileName.endsWith(".properties")) { 125 throw new ServiceException(ErrorCode.E0012, log4jFileName); 126 } 127 String configPath = ConfigurationService.getConfigurationDirectory(); 128 File log4jFile = new File(configPath, log4jFileName); 129 if (log4jFile.exists()) { 130 fromClasspath = false; 131 } 132 else { 133 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 134 URL log4jUrl = cl.getResource(log4jFileName); 135 if (log4jUrl == null) { 136 throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath); 137 } 138 fromClasspath = true; 139 } 140 141 if (fromClasspath) { 142 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 143 URL log4jUrl = cl.getResource(log4jFileName); 144 PropertyConfigurator.configure(log4jUrl); 145 } 146 else { 147 interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL)); 148 PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000); 149 } 150 151 log = new XLog(LogFactory.getLog(getClass())); 152 153 log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION), 154 BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo() 155 .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty( 156 BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL)); 157 158 String from = (fromClasspath) ? "CLASSPATH" : configPath; 159 String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec"; 160 log.info("Log4j configuration file [{0}]", log4jFileName); 161 log.info("Log4j configuration file loaded from [{0}]", from); 162 log.info("Log4j reload interval [{0}]", reload); 163 164 XLog.Info.reset(); 165 XLog.Info.defineParameter(USER); 166 XLog.Info.defineParameter(GROUP); 167 XLogStreamer.Filter.reset(); 168 XLogStreamer.Filter.defineParameter(USER); 169 XLogStreamer.Filter.defineParameter(GROUP); 170 171 // Getting configuration for oozie log via WS 172 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 173 InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile); 174 extractInfoForLogWebService(is); 175 } 176 catch (IOException ex) { 177 throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex); 178 } 179 } 180 181 private void extractInfoForLogWebService(InputStream is) throws IOException { 182 Properties props = new Properties(); 183 props.load(is); 184 185 Configuration conf = new XConfiguration(); 186 for (Map.Entry entry : props.entrySet()) { 187 conf.set((String) entry.getKey(), (String) entry.getValue()); 188 } 189 String logFile = conf.get("log4j.appender.oozie.File"); 190 if (logFile == null) { 191 log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' " 192 + "appender"); 193 } 194 else { 195 logFile = logFile.trim(); 196 int i = logFile.lastIndexOf("/"); 197 if (i == -1) { 198 log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender", 199 logFile); 200 logOverWS = false; 201 } 202 else { 203 String pattern = conf.get("log4j.appender.oozie.DatePattern"); 204 if (pattern == null) { 205 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]"); 206 logOverWS = false; 207 } 208 else { 209 pattern = pattern.trim(); 210 if (pattern.endsWith("HH")) { 211 oozieLogRotation = 60 * 60; 212 } 213 else { 214 if (pattern.endsWith("dd")) { 215 oozieLogRotation = 60 * 60 * 24; 216 } 217 else { 218 log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'", 219 pattern); 220 logOverWS = false; 221 } 222 } 223 if (oozieLogRotation > 0) { 224 oozieLogPath = logFile.substring(0, i); 225 oozieLogName = logFile.substring(i + 1); 226 } 227 else { 228 logOverWS = false; 229 } 230 } 231 } 232 } 233 } 234 235 /** 236 * Destroy the log service. 237 */ 238 public void destroy() { 239 LogManager.shutdown(); 240 XLog.Info.reset(); 241 XLogStreamer.Filter.reset(); 242 } 243 244 /** 245 * Group log info constant. 246 */ 247 public static final String USER = "USER"; 248 249 /** 250 * Group log info constant. 251 */ 252 public static final String GROUP = "GROUP"; 253 254 /** 255 * Return the public interface for log service. 256 * 257 * @return {@link XLogService}. 258 */ 259 public Class<? extends Service> getInterface() { 260 return XLogService.class; 261 } 262 263 /** 264 * Instruments the log service. 265 * <p/> 266 * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath. 267 * 268 * @param instr instrumentation to use. 269 */ 270 public void instrument(Instrumentation instr) { 271 instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() { 272 public String getValue() { 273 return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION); 274 } 275 }); 276 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() { 277 public String getValue() { 278 return log4jFileName; 279 } 280 }); 281 instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() { 282 public Long getValue() { 283 return interval; 284 } 285 }); 286 instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() { 287 public Boolean getValue() { 288 return fromClasspath; 289 } 290 }); 291 instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() { 292 public Boolean getValue() { 293 return logOverWS; 294 } 295 }); 296 } 297 298 /** 299 * Stream the log of a job. 300 * 301 * @param filter log streamer filter. 302 * @param startTime start time for log events to filter. 303 * @param endTime end time for log events to filter. 304 * @param writer writer to stream the log to. 305 * @throws IOException thrown if the log cannot be streamed. 306 */ 307 public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException { 308 if (logOverWS) { 309 new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation) 310 .streamLog(startTime, endTime); 311 } 312 else { 313 writer.write("Log streaming disabled!!"); 314 } 315 316 } 317 318 String getLog4jProperties() { 319 return log4jFileName; 320 } 321 322 boolean getFromClasspath() { 323 return fromClasspath; 324 } 325 326 }