001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.oozie.util; 019 020import java.io.File; 021import java.io.IOException; 022import java.io.Writer; 023import java.util.ArrayList; 024import java.util.Calendar; 025import java.util.Collections; 026import java.util.Date; 027import java.util.regex.Matcher; 028import java.util.regex.Pattern; 029import java.io.BufferedReader; 030 031/** 032 * XLogStreamer streams the given log file to writer after applying the given filter. 033 */ 034public class XLogStreamer { 035 private static XLog LOG = XLog.getLog(XLogStreamer.class); 036 private String logFile; 037 private String logPath; 038 private XLogFilter logFilter; 039 private long logRotation; 040 041 public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) { 042 this.logFilter = logFilter; 043 if (logFile == null) { 044 logFile = "oozie-app.log"; 045 } 046 this.logFile = logFile; 047 this.logPath = logPath; 048 this.logRotation = logRotationSecs * 1000l; 049 } 050 051 /** 052 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after 053 * applying the filters. 054 * 055 * @param writer 056 * @param startTime 057 * @param endTime 058 * @throws IOException 059 */ 060 public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException { 061 // Get a Reader for the log file(s) 062 BufferedReader reader = new BufferedReader(getReader(startTime, endTime)); 063 try { 064 if(logFilter.isDebugMode()){ 065 writer.write(logFilter.getDebugMessage()); 066 } 067 // Process the entire logs from the reader using the logFilter 068 new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen); 069 } 070 finally { 071 reader.close(); 072 } 073 } 074 075 /** 076 * Returns a BufferedReader configured to read the log files based on the given startTime and endTime. 077 * 078 * @param startTime 079 * @param endTime 080 * @return A BufferedReader for the log files 081 * @throws IOException 082 */ 083 084 private MultiFileReader getReader(Date startTime, Date endTime) throws IOException { 085 logFilter.calculateAndValidateDateRange(startTime, endTime); 086 return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate())); 087 } 088 089 public BufferedReader makeReader(Date startTime, Date endTime) throws IOException { 090 return new BufferedReader(getReader(startTime,endTime)); 091 } 092 093 /** 094 * Gets the log file list for specific date range. 095 * 096 * @param startTime the start time 097 * @param endTime the end time 098 * @return log file list 099 * @throws IOException Signals that an I/O exception has occurred. 100 */ 101 private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException { 102 long startTimeMillis = 0; 103 long endTimeMillis; 104 if (startTime != null) { 105 startTimeMillis = startTime.getTime(); 106 } 107 if (endTime == null) { 108 endTimeMillis = System.currentTimeMillis(); 109 } 110 else { 111 endTimeMillis = endTime.getTime(); 112 } 113 File dir = new File(logPath); 114 return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile); 115 } 116 117 /** 118 * File along with the modified time which will be used to sort later. 119 */ 120 public class FileInfo implements Comparable<FileInfo> { 121 File file; 122 long modTime; 123 124 public FileInfo(File file, long modTime) { 125 this.file = file; 126 this.modTime = modTime; 127 } 128 129 public File getFile() { 130 return file; 131 } 132 133 public long getModTime() { 134 return modTime; 135 } 136 137 public int compareTo(FileInfo fileInfo) { 138 long diff = this.modTime - fileInfo.modTime; 139 if (diff > 0) { 140 return 1; 141 } 142 else if (diff < 0) { 143 return -1; 144 } 145 else { 146 return 0; 147 } 148 } 149 } 150 151 /** 152 * Gets the file list that will have the logs between startTime and endTime. 153 * 154 * @param dir 155 * @param startTime 156 * @param endTime 157 * @param logRotationTime 158 * @param logFile 159 * @return List of files to be streamed 160 */ 161 private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) { 162 String[] children = dir.list(); 163 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 164 if (children == null) { 165 return new ArrayList<File>(); 166 } 167 else { 168 for (int i = 0; i < children.length; i++) { 169 String fileName = children[i]; 170 if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) { 171 continue; 172 } 173 File file = new File(dir.getAbsolutePath(), fileName); 174 if (fileName.endsWith(".gz")) { 175 long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime); 176 if (gzFileCreationTime != -1) { 177 fileList.add(new FileInfo(file, gzFileCreationTime)); 178 } 179 continue; 180 } 181 long modTime = file.lastModified(); 182 if (modTime < startTime) { 183 continue; 184 } 185 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) { 186 continue; 187 } 188 fileList.add(new FileInfo(file, modTime)); 189 } 190 } 191 Collections.sort(fileList); 192 ArrayList<File> files = new ArrayList<File>(fileList.size()); 193 for (FileInfo info : fileList) { 194 files.add(info.getFile()); 195 } 196 return files; 197 } 198 199 /** 200 * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part 201 * of the date 202 */ 203 public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz"); 204 205 /** 206 * Returns the creation time of the .gz archive if it is relevant to the job 207 * 208 * @param fileName 209 * @param startTime 210 * @param endTime 211 * @return Modification time of .gz file after checking if it is relevant to the job 212 */ 213 private long getGZFileCreationTime(String fileName, long startTime, long endTime) { 214 // Default return value of -1 to exclude the file 215 long returnVal = -1; 216 217 // Include oozie.log as oozie.log.gz if it is accidentally GZipped 218 if (fileName.equals("oozie.log.gz")) { 219 LOG.warn("oozie.log has been GZipped, which is unexpected"); 220 // Return a value other than -1 to include the file in list 221 returnVal = 0; 222 } else { 223 Matcher m = gzTimePattern.matcher(fileName); 224 if (m.matches() && m.groupCount() == 4) { 225 int year = Integer.parseInt(m.group(1)); 226 int month = Integer.parseInt(m.group(2)); 227 int day = Integer.parseInt(m.group(3)); 228 int hour = Integer.parseInt(m.group(4)); 229 int minute = 0; 230 Calendar calendarEntry = Calendar.getInstance(); 231 calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August) 232 long logFileStartTime = calendarEntry.getTimeInMillis(); 233 long milliSecondsPerHour = 3600000; 234 long logFileEndTime = logFileStartTime + milliSecondsPerHour; 235 /* To check whether the log content is there in the initial or later part of the log file or 236 the log content is contained entirely within this log file or 237 the entire log file contains the event log where the event spans across hours 238 */ 239 if ((startTime >= logFileStartTime && startTime <= logFileEndTime) 240 || (endTime >= logFileStartTime && endTime <= logFileEndTime) 241 || (startTime <= logFileStartTime && endTime >= logFileEndTime)) { 242 returnVal = logFileStartTime; 243 } 244 } else { 245 LOG.debug("Filename " + fileName + " does not match the expected format"); 246 returnVal = -1; 247 } 248 } 249 return returnVal; 250 } 251}