001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.util; 020 021import java.io.File; 022import java.io.IOException; 023import java.io.Writer; 024import java.util.ArrayList; 025import java.util.Calendar; 026import java.util.Collections; 027import java.util.Date; 028import java.util.regex.Matcher; 029import java.util.regex.Pattern; 030import java.io.BufferedReader; 031 032/** 033 * XLogStreamer streams the given log file to writer after applying the given filter. 034 */ 035public class XLogStreamer { 036 private static XLog LOG = XLog.getLog(XLogStreamer.class); 037 private String logFile; 038 private String logPath; 039 private XLogFilter logFilter; 040 private long logRotation; 041 042 public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) { 043 this.logFilter = logFilter; 044 if (logFile == null) { 045 logFile = "oozie-app.log"; 046 } 047 this.logFile = logFile; 048 this.logPath = logPath; 049 this.logRotation = logRotationSecs * 1000l; 050 } 051 052 /** 053 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after 054 * applying the filters. 055 * 056 * @param writer 057 * @param startTime 058 * @param endTime 059 * @throws IOException 060 */ 061 public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException { 062 // Get a Reader for the log file(s) 063 BufferedReader reader = new BufferedReader(getReader(startTime, endTime)); 064 try { 065 if(logFilter.isDebugMode()){ 066 writer.write(logFilter.getDebugMessage()); 067 } 068 // Process the entire logs from the reader using the logFilter 069 new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen); 070 } 071 finally { 072 reader.close(); 073 } 074 } 075 076 /** 077 * Returns a BufferedReader configured to read the log files based on the given startTime and endTime. 078 * 079 * @param startTime 080 * @param endTime 081 * @return A BufferedReader for the log files 082 * @throws IOException 083 */ 084 085 private MultiFileReader getReader(Date startTime, Date endTime) throws IOException { 086 logFilter.calculateAndValidateDateRange(startTime, endTime); 087 return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate())); 088 } 089 090 public BufferedReader makeReader(Date startTime, Date endTime) throws IOException { 091 return new BufferedReader(getReader(startTime,endTime)); 092 } 093 094 /** 095 * Gets the log file list for specific date range. 096 * 097 * @param startTime the start time 098 * @param endTime the end time 099 * @return log file list 100 * @throws IOException Signals that an I/O exception has occurred. 101 */ 102 private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException { 103 long startTimeMillis = 0; 104 long endTimeMillis; 105 if (startTime != null) { 106 startTimeMillis = startTime.getTime(); 107 } 108 if (endTime == null) { 109 endTimeMillis = System.currentTimeMillis(); 110 } 111 else { 112 endTimeMillis = endTime.getTime(); 113 } 114 File dir = new File(logPath); 115 return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile); 116 } 117 118 /** 119 * File along with the modified time which will be used to sort later. 120 */ 121 public class FileInfo implements Comparable<FileInfo> { 122 File file; 123 long modTime; 124 125 public FileInfo(File file, long modTime) { 126 this.file = file; 127 this.modTime = modTime; 128 } 129 130 public File getFile() { 131 return file; 132 } 133 134 public long getModTime() { 135 return modTime; 136 } 137 138 public int compareTo(FileInfo fileInfo) { 139 long diff = this.modTime - fileInfo.modTime; 140 if (diff > 0) { 141 return 1; 142 } 143 else if (diff < 0) { 144 return -1; 145 } 146 else { 147 return 0; 148 } 149 } 150 } 151 152 /** 153 * Gets the file list that will have the logs between startTime and endTime. 154 * 155 * @param dir 156 * @param startTime 157 * @param endTime 158 * @param logRotationTime 159 * @param logFile 160 * @return List of files to be streamed 161 */ 162 private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) { 163 String[] children = dir.list(); 164 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 165 if (children == null) { 166 return new ArrayList<File>(); 167 } 168 else { 169 for (int i = 0; i < children.length; i++) { 170 String fileName = children[i]; 171 if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) { 172 continue; 173 } 174 File file = new File(dir.getAbsolutePath(), fileName); 175 if (fileName.endsWith(".gz")) { 176 long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime); 177 if (gzFileCreationTime != -1) { 178 fileList.add(new FileInfo(file, gzFileCreationTime)); 179 } 180 continue; 181 } 182 long modTime = file.lastModified(); 183 if (modTime < startTime) { 184 continue; 185 } 186 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) { 187 continue; 188 } 189 fileList.add(new FileInfo(file, modTime)); 190 } 191 } 192 Collections.sort(fileList); 193 ArrayList<File> files = new ArrayList<File>(fileList.size()); 194 for (FileInfo info : fileList) { 195 files.add(info.getFile()); 196 } 197 return files; 198 } 199 200 /** 201 * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part 202 * of the date 203 */ 204 public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz"); 205 206 /** 207 * Returns the creation time of the .gz archive if it is relevant to the job 208 * 209 * @param fileName 210 * @param startTime 211 * @param endTime 212 * @return Modification time of .gz file after checking if it is relevant to the job 213 */ 214 private long getGZFileCreationTime(String fileName, long startTime, long endTime) { 215 // Default return value of -1 to exclude the file 216 long returnVal = -1; 217 218 // Include oozie.log as oozie.log.gz if it is accidentally GZipped 219 if (fileName.equals("oozie.log.gz")) { 220 LOG.warn("oozie.log has been GZipped, which is unexpected"); 221 // Return a value other than -1 to include the file in list 222 returnVal = 0; 223 } else { 224 Matcher m = gzTimePattern.matcher(fileName); 225 if (m.matches() && m.groupCount() == 4) { 226 int year = Integer.parseInt(m.group(1)); 227 int month = Integer.parseInt(m.group(2)); 228 int day = Integer.parseInt(m.group(3)); 229 int hour = Integer.parseInt(m.group(4)); 230 int minute = 0; 231 Calendar calendarEntry = Calendar.getInstance(); 232 calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August) 233 long logFileStartTime = calendarEntry.getTimeInMillis(); 234 long milliSecondsPerHour = 3600000; 235 long logFileEndTime = logFileStartTime + milliSecondsPerHour; 236 /* To check whether the log content is there in the initial or later part of the log file or 237 the log content is contained entirely within this log file or 238 the entire log file contains the event log where the event spans across hours 239 */ 240 if ((startTime >= logFileStartTime && startTime <= logFileEndTime) 241 || (endTime >= logFileStartTime && endTime <= logFileEndTime) 242 || (startTime <= logFileStartTime && endTime >= logFileEndTime)) { 243 returnVal = logFileStartTime; 244 } 245 } else { 246 LOG.debug("Filename " + fileName + " does not match the expected format"); 247 returnVal = -1; 248 } 249 } 250 return returnVal; 251 } 252}