001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import java.io.File;
022import java.io.IOException;
023import java.io.Writer;
024import java.util.ArrayList;
025import java.util.Calendar;
026import java.util.Collections;
027import java.util.Date;
028import java.util.regex.Matcher;
029import java.util.regex.Pattern;
030import java.io.BufferedReader;
031
032/**
033 * XLogStreamer streams the given log file to writer after applying the given filter.
034 */
035public class XLogStreamer {
036    private static XLog LOG = XLog.getLog(XLogStreamer.class);
037    private String logFile;
038    private String logPath;
039    private XLogFilter logFilter;
040    private long logRotation;
041
042    public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) {
043        this.logFilter = logFilter;
044        if (logFile == null) {
045            logFile = "oozie-app.log";
046        }
047        this.logFile = logFile;
048        this.logPath = logPath;
049        this.logRotation = logRotationSecs * 1000l;
050    }
051
052    /**
053     * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
054     * applying the filters.
055     *
056     * @param writer
057     * @param startTime
058     * @param endTime
059     * @throws IOException
060     */
061    public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException {
062        // Get a Reader for the log file(s)
063        BufferedReader reader = new BufferedReader(getReader(startTime, endTime));
064        try {
065            if(logFilter.isDebugMode()){
066                writer.write(logFilter.getDebugMessage());
067            }
068            // Process the entire logs from the reader using the logFilter
069            new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen);
070        }
071        finally {
072            reader.close();
073        }
074    }
075
076    /**
077     * Returns a BufferedReader configured to read the log files based on the given startTime and endTime.
078     *
079     * @param startTime
080     * @param endTime
081     * @return A BufferedReader for the log files
082     * @throws IOException
083     */
084
085    private MultiFileReader getReader(Date startTime, Date endTime) throws IOException {
086        logFilter.calculateAndValidateDateRange(startTime, endTime);
087        return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate()));
088    }
089
090    public BufferedReader makeReader(Date startTime, Date endTime) throws IOException {
091        return new BufferedReader(getReader(startTime,endTime));
092    }
093
094    /**
095     * Gets the log file list for specific date range.
096     *
097     * @param startTime the start time
098     * @param endTime the end time
099     * @return log file list
100     * @throws IOException Signals that an I/O exception has occurred.
101     */
102    private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException {
103        long startTimeMillis = 0;
104        long endTimeMillis;
105        if (startTime != null) {
106            startTimeMillis = startTime.getTime();
107        }
108        if (endTime == null) {
109            endTimeMillis = System.currentTimeMillis();
110        }
111        else {
112            endTimeMillis = endTime.getTime();
113        }
114        File dir = new File(logPath);
115        return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
116    }
117
118    /**
119     * File along with the modified time which will be used to sort later.
120     */
121    public class FileInfo implements Comparable<FileInfo> {
122        File file;
123        long modTime;
124
125        public FileInfo(File file, long modTime) {
126            this.file = file;
127            this.modTime = modTime;
128        }
129
130        public File getFile() {
131            return file;
132        }
133
134        public long getModTime() {
135            return modTime;
136        }
137
138        public int compareTo(FileInfo fileInfo) {
139            long diff = this.modTime - fileInfo.modTime;
140            if (diff > 0) {
141                return 1;
142            }
143            else if (diff < 0) {
144                return -1;
145            }
146            else {
147                return 0;
148            }
149        }
150    }
151
152    /**
153     * Gets the file list that will have the logs between startTime and endTime.
154     *
155     * @param dir
156     * @param startTime
157     * @param endTime
158     * @param logRotationTime
159     * @param logFile
160     * @return List of files to be streamed
161     */
162    private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
163        String[] children = dir.list();
164        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
165        if (children == null) {
166            return new ArrayList<File>();
167        }
168        else {
169            for (int i = 0; i < children.length; i++) {
170                String fileName = children[i];
171                if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
172                    continue;
173                }
174                File file = new File(dir.getAbsolutePath(), fileName);
175                if (fileName.endsWith(".gz")) {
176                    long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime);
177                    if (gzFileCreationTime != -1) {
178                        fileList.add(new FileInfo(file, gzFileCreationTime));
179                    }
180                    continue;
181                }
182                long modTime = file.lastModified();
183                if (modTime < startTime) {
184                    continue;
185                }
186                if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
187                    continue;
188                }
189                fileList.add(new FileInfo(file, modTime));
190            }
191        }
192        Collections.sort(fileList);
193        ArrayList<File> files = new ArrayList<File>(fileList.size());
194        for (FileInfo info : fileList) {
195            files.add(info.getFile());
196        }
197        return files;
198    }
199
200    /**
201     * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part
202     * of the date
203     */
204    public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz");
205
206    /**
207     * Returns the creation time of the .gz archive if it is relevant to the job
208     *
209     * @param fileName
210     * @param startTime
211     * @param endTime
212     * @return Modification time of .gz file after checking if it is relevant to the job
213     */
214    private long getGZFileCreationTime(String fileName, long startTime, long endTime) {
215        // Default return value of -1 to exclude the file
216        long returnVal = -1;
217
218        // Include oozie.log as oozie.log.gz if it is accidentally GZipped
219        if (fileName.equals("oozie.log.gz")) {
220            LOG.warn("oozie.log has been GZipped, which is unexpected");
221            // Return a value other than -1 to include the file in list
222            returnVal = 0;
223        } else {
224            Matcher m = gzTimePattern.matcher(fileName);
225            if (m.matches() && m.groupCount() == 4) {
226                int year = Integer.parseInt(m.group(1));
227                int month = Integer.parseInt(m.group(2));
228                int day = Integer.parseInt(m.group(3));
229                int hour = Integer.parseInt(m.group(4));
230                int minute = 0;
231                Calendar calendarEntry = Calendar.getInstance();
232                calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
233                long logFileStartTime = calendarEntry.getTimeInMillis();
234                long milliSecondsPerHour = 3600000;
235                long logFileEndTime = logFileStartTime + milliSecondsPerHour;
236                /*  To check whether the log content is there in the initial or later part of the log file or
237                    the log content is contained entirely within this log file or
238                    the entire log file contains the event log where the event spans across hours
239                */
240                if ((startTime >= logFileStartTime && startTime <= logFileEndTime)
241                        || (endTime >= logFileStartTime && endTime <= logFileEndTime)
242                        || (startTime <= logFileStartTime && endTime >= logFileEndTime)) {
243                    returnVal = logFileStartTime;
244                }
245            } else {
246                LOG.debug("Filename " + fileName + " does not match the expected format");
247                returnVal = -1;
248            }
249        }
250        return returnVal;
251    }
252}