001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.util;
019
020import java.io.File;
021import java.io.IOException;
022import java.io.Writer;
023import java.util.ArrayList;
024import java.util.Calendar;
025import java.util.Collections;
026import java.util.Date;
027import java.util.regex.Matcher;
028import java.util.regex.Pattern;
029import java.io.BufferedReader;
030
031/**
032 * XLogStreamer streams the given log file to writer after applying the given filter.
033 */
034public class XLogStreamer {
035    private static XLog LOG = XLog.getLog(XLogStreamer.class);
036    private String logFile;
037    private String logPath;
038    private XLogFilter logFilter;
039    private long logRotation;
040
041    public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) {
042        this.logFilter = logFilter;
043        if (logFile == null) {
044            logFile = "oozie-app.log";
045        }
046        this.logFile = logFile;
047        this.logPath = logPath;
048        this.logRotation = logRotationSecs * 1000l;
049    }
050
051    /**
052     * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
053     * applying the filters.
054     *
055     * @param writer
056     * @param startTime
057     * @param endTime
058     * @throws IOException
059     */
060    public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException {
061        // Get a Reader for the log file(s)
062        BufferedReader reader = new BufferedReader(getReader(startTime, endTime));
063        try {
064            if(logFilter.isDebugMode()){
065                writer.write(logFilter.getDebugMessage());
066            }
067            // Process the entire logs from the reader using the logFilter
068            new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen);
069        }
070        finally {
071            reader.close();
072        }
073    }
074
075    /**
076     * Returns a BufferedReader configured to read the log files based on the given startTime and endTime.
077     *
078     * @param startTime
079     * @param endTime
080     * @return A BufferedReader for the log files
081     * @throws IOException
082     */
083
084    private MultiFileReader getReader(Date startTime, Date endTime) throws IOException {
085        logFilter.calculateAndValidateDateRange(startTime, endTime);
086        return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate()));
087    }
088
089    public BufferedReader makeReader(Date startTime, Date endTime) throws IOException {
090        return new BufferedReader(getReader(startTime,endTime));
091    }
092
093    /**
094     * Gets the log file list for specific date range.
095     *
096     * @param startTime the start time
097     * @param endTime the end time
098     * @return log file list
099     * @throws IOException Signals that an I/O exception has occurred.
100     */
101    private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException {
102        long startTimeMillis = 0;
103        long endTimeMillis;
104        if (startTime != null) {
105            startTimeMillis = startTime.getTime();
106        }
107        if (endTime == null) {
108            endTimeMillis = System.currentTimeMillis();
109        }
110        else {
111            endTimeMillis = endTime.getTime();
112        }
113        File dir = new File(logPath);
114        return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
115    }
116
117    /**
118     * File along with the modified time which will be used to sort later.
119     */
120    public class FileInfo implements Comparable<FileInfo> {
121        File file;
122        long modTime;
123
124        public FileInfo(File file, long modTime) {
125            this.file = file;
126            this.modTime = modTime;
127        }
128
129        public File getFile() {
130            return file;
131        }
132
133        public long getModTime() {
134            return modTime;
135        }
136
137        public int compareTo(FileInfo fileInfo) {
138            long diff = this.modTime - fileInfo.modTime;
139            if (diff > 0) {
140                return 1;
141            }
142            else if (diff < 0) {
143                return -1;
144            }
145            else {
146                return 0;
147            }
148        }
149    }
150
151    /**
152     * Gets the file list that will have the logs between startTime and endTime.
153     *
154     * @param dir
155     * @param startTime
156     * @param endTime
157     * @param logRotationTime
158     * @param logFile
159     * @return List of files to be streamed
160     */
161    private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
162        String[] children = dir.list();
163        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
164        if (children == null) {
165            return new ArrayList<File>();
166        }
167        else {
168            for (int i = 0; i < children.length; i++) {
169                String fileName = children[i];
170                if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
171                    continue;
172                }
173                File file = new File(dir.getAbsolutePath(), fileName);
174                if (fileName.endsWith(".gz")) {
175                    long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime);
176                    if (gzFileCreationTime != -1) {
177                        fileList.add(new FileInfo(file, gzFileCreationTime));
178                    }
179                    continue;
180                }
181                long modTime = file.lastModified();
182                if (modTime < startTime) {
183                    continue;
184                }
185                if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
186                    continue;
187                }
188                fileList.add(new FileInfo(file, modTime));
189            }
190        }
191        Collections.sort(fileList);
192        ArrayList<File> files = new ArrayList<File>(fileList.size());
193        for (FileInfo info : fileList) {
194            files.add(info.getFile());
195        }
196        return files;
197    }
198
199    /**
200     * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part
201     * of the date
202     */
203    public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz");
204
205    /**
206     * Returns the creation time of the .gz archive if it is relevant to the job
207     *
208     * @param fileName
209     * @param startTime
210     * @param endTime
211     * @return Modification time of .gz file after checking if it is relevant to the job
212     */
213    private long getGZFileCreationTime(String fileName, long startTime, long endTime) {
214        // Default return value of -1 to exclude the file
215        long returnVal = -1;
216
217        // Include oozie.log as oozie.log.gz if it is accidentally GZipped
218        if (fileName.equals("oozie.log.gz")) {
219            LOG.warn("oozie.log has been GZipped, which is unexpected");
220            // Return a value other than -1 to include the file in list
221            returnVal = 0;
222        } else {
223            Matcher m = gzTimePattern.matcher(fileName);
224            if (m.matches() && m.groupCount() == 4) {
225                int year = Integer.parseInt(m.group(1));
226                int month = Integer.parseInt(m.group(2));
227                int day = Integer.parseInt(m.group(3));
228                int hour = Integer.parseInt(m.group(4));
229                int minute = 0;
230                Calendar calendarEntry = Calendar.getInstance();
231                calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
232                long logFileStartTime = calendarEntry.getTimeInMillis();
233                long milliSecondsPerHour = 3600000;
234                long logFileEndTime = logFileStartTime + milliSecondsPerHour;
235                /*  To check whether the log content is there in the initial or later part of the log file or
236                    the log content is contained entirely within this log file or
237                    the entire log file contains the event log where the event spans across hours
238                */
239                if ((startTime >= logFileStartTime && startTime <= logFileEndTime)
240                        || (endTime >= logFileStartTime && endTime <= logFileEndTime)
241                        || (startTime <= logFileStartTime && endTime >= logFileEndTime)) {
242                    returnVal = logFileStartTime;
243                }
244            } else {
245                LOG.debug("Filename " + fileName + " does not match the expected format");
246                returnVal = -1;
247            }
248        }
249        return returnVal;
250    }
251}