Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019
020package org.apache.oozie.util;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.regex.Matcher;
029import java.util.regex.Pattern;
030
031import org.apache.commons.lang.StringUtils;
032import org.apache.oozie.service.ConfigurationService;
033import com.google.common.annotations.VisibleForTesting;
034
035/**
036 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if
037 * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|")
038 * jobId appName actionId token
039 */
040public class XLogFilter {
041
042    private static final int LOG_TIME_BUFFER = 2; // in min
043    public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration";
044    public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration";
045    private Map<String, Integer> logLevels;
046    private final Map<String, String> filterParams;
047    private static List<String> parameters = new ArrayList<String>();
048    private boolean noFilter;
049    private Pattern filterPattern;
050    private XLogUserFilterParam userLogFilter;
051    private Date endDate;
052    private Date startDate;
053    private boolean isActionList = false;
054    private String formattedEndDate;
055    private String formattedStartDate;
056
057    // TODO Patterns to be read from config file
058    private static final String DEFAULT_REGEX = "[^\\]]*";
059
060    public static final String ALLOW_ALL_REGEX = "(.*)";
061    private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
062    private static final String WHITE_SPACE_REGEX = "\\s+";
063    private static final String LOG_LEVEL_REGEX = "(\\w+)";
064    private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
065            + WHITE_SPACE_REGEX;
066    private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
067
068    public XLogFilter() {
069        this(new XLogUserFilterParam());
070    }
071
072    public XLogFilter(XLogUserFilterParam userLogFilter) {
073        filterParams = new HashMap<String, String>();
074        for (int i = 0; i < parameters.size(); i++) {
075            filterParams.put(parameters.get(i), DEFAULT_REGEX);
076        }
077        logLevels = null;
078        noFilter = true;
079        filterPattern = null;
080        setUserLogFilter(userLogFilter);
081    }
082
083    public void setLogLevel(String logLevel) {
084        if (logLevel != null && logLevel.trim().length() > 0) {
085            this.logLevels = new HashMap<String, Integer>();
086            String[] levels = logLevel.split("\\|");
087            for (int i = 0; i < levels.length; i++) {
088                String s = levels[i].trim().toUpperCase();
089                try {
090                    XLog.Level.valueOf(s);
091                }
092                catch (Exception ex) {
093                    continue;
094                }
095                this.logLevels.put(levels[i].toUpperCase(), 1);
096            }
097        }
098    }
099
100    public void setParameter(String filterParam, String value) {
101        if (filterParams.containsKey(filterParam)) {
102            noFilter = false;
103            filterParams.put(filterParam, value);
104        }
105    }
106
107    public static void defineParameter(String filterParam) {
108        parameters.add(filterParam);
109    }
110
111    public boolean isFilterPresent() {
112        if (noFilter && logLevels == null) {
113            return false;
114        }
115        return true;
116    }
117
118    /**
119     * Checks if the logLevel and logMessage goes through the logFilter.
120     *
121     * @param logParts
122     * @return
123     */
124    public boolean matches(ArrayList<String> logParts) {
125        if (getStartDate() != null) {
126            if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
127                return false;
128            }
129        }
130        String logLevel = logParts.get(1);
131        String logMessage = logParts.get(2);
132        if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
133            Matcher logMatcher = filterPattern.matcher(logMessage);
134            return logMatcher.matches();
135        }
136        else {
137            return false;
138        }
139    }
140
141    /**
142     * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing timestamp,
143     * logLevel, and logMessage if the pattern matches i.e A new log statement, else returns null.
144     *
145     * @param logLine
146     * @return Array containing log level and log message
147     */
148    public ArrayList<String> splitLogMessage(String logLine) {
149        Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
150        if (splitter.matches()) {
151            ArrayList<String> logParts = new ArrayList<String>();
152            logParts.add(splitter.group(1));// timestamp
153            logParts.add(splitter.group(2));// log level
154            logParts.add(splitter.group(3));// Log Message
155            return logParts;
156        }
157        else {
158            return null;
159        }
160    }
161
162    /**
163     * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be assigned
164     * if no filters are set.
165     */
166    public void constructPattern() {
167        if (noFilter && logLevels == null) {
168            filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
169            return;
170        }
171        StringBuilder sb = new StringBuilder();
172        if (noFilter) {
173            sb.append("(.*)");
174        }
175        else {
176            sb.append("(.* ");
177            for (int i = 0; i < parameters.size(); i++) {
178                sb.append(parameters.get(i) + "\\[");
179                sb.append(filterParams.get(parameters.get(i)) + "\\] ");
180            }
181            sb.append(".*)");
182        }
183        if (!StringUtils.isEmpty(userLogFilter.getSearchText())) {
184            sb.append(userLogFilter.getSearchText() + ".*");
185        }
186        filterPattern = Pattern.compile(sb.toString());
187    }
188
189    public static void reset() {
190        parameters.clear();
191    }
192
193    public final Map<String, String> getFilterParams() {
194        return filterParams;
195    }
196
197    public XLogUserFilterParam getUserLogFilter() {
198        return userLogFilter;
199    }
200
201    public void setUserLogFilter(XLogUserFilterParam userLogFilter) {
202        this.userLogFilter = userLogFilter;
203        setLogLevel(userLogFilter.getLogLevel());
204    }
205
206    public Date getEndDate() {
207        return endDate;
208    }
209
210    public String getFormattedEndDate() {
211        return formattedEndDate;
212    }
213
214    public String getFormattedStartDate() {
215        return formattedStartDate;
216    }
217
218    public Date getStartDate() {
219        return startDate;
220    }
221
222    public boolean isDebugMode() {
223        return userLogFilter.isDebug();
224    }
225
226    public int getLogLimit() {
227        return userLogFilter.getLimit();
228    }
229
230    public String getDebugMessage() {
231        return "Log start time = " + getStartDate() + ". Log end time = " + getEndDate() + ". User Log Filter = "
232                + getUserLogFilter() + System.getProperty("line.separator");
233    }
234
235    public boolean isActionList() {
236        return isActionList;
237    }
238
239    public void setActionList(boolean isActionList) {
240        this.isActionList = isActionList;
241    }
242
243    private void calculateScanDate(Date jobStartTime, Date jobEndTime) throws IOException {
244
245        if (userLogFilter.getStartDate() != null) {
246            startDate = userLogFilter.getStartDate();
247        }
248        else if (userLogFilter.getStartOffset() != -1) {
249            startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset());
250        }
251        else {
252            startDate = jobStartTime;
253        }
254
255        if (userLogFilter.getEndDate() != null) {
256            endDate = userLogFilter.getEndDate();
257        }
258        else if (userLogFilter.getEndOffset() != -1) {
259            // If user has specified startdate as absolute then end offset will be on user start date,
260            // else end offset will be calculated on job startdate.
261            if (userLogFilter.getStartDate() != null) {
262                endDate = adjustOffset(startDate, userLogFilter.getEndOffset());
263            }
264            else {
265                endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset());
266            }
267        }
268        else {
269            endDate = jobEndTime;
270        }
271        // if recent offset is specified then start time = endtime - offset
272        if (getUserLogFilter().getRecent() != -1) {
273            startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1);
274        }
275
276        //add buffer iff dates are not asbsolute
277        if (userLogFilter.getStartDate() == null) {
278            startDate = adjustOffset(startDate, -LOG_TIME_BUFFER);
279        }
280        if (userLogFilter.getEndDate() == null) {
281            endDate = adjustOffset(endDate, LOG_TIME_BUFFER);
282        }
283
284        formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate());
285        formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate());
286    }
287
288    /**
289     * Calculate and validate date range.
290     *
291     * @param jobStartTime the job start time
292     * @param jobEndTime the job end time
293     * @throws IOException Signals that an I/O exception has occurred.
294     */
295    public void calculateAndValidateDateRange(Date jobStartTime, Date jobEndTime) throws IOException {
296        // for testcase, otherwise jobStartTime and jobEndTime will be always set
297        if (jobStartTime == null || jobEndTime == null) {
298            return;
299        }
300        calculateScanDate(jobStartTime, jobEndTime);
301
302        if (startDate.after(endDate)) {
303            throw new IOException("Start time should be less than end time. startTime = " + startDate + " endtime = "
304                    + endDate);
305        }
306        long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000);
307        if (isActionList) {
308            int actionLogDuration = ConfigurationService.getInt(MAX_ACTIONLIST_SCAN_DURATION);
309            if (actionLogDuration == -1) {
310                return;
311            }
312            if (diffHours > actionLogDuration) {
313                throw new IOException(
314                        "Request log streaming time range with action list is higher than configured. Please reduce the scan "
315                                + "time range. Input range (hours) = " + diffHours
316                                + " system allowed (hours) with action list = " + actionLogDuration);
317            }
318        }
319        else {
320            int logDuration = ConfigurationService.getInt(MAX_SCAN_DURATION);
321            if (logDuration == -1) {
322                return;
323            }
324            if (diffHours > logDuration) {
325                throw new IOException(
326                        "Request log streaming time range is higher than configured. Please reduce the scan time range. For coord"
327                                + " jobs you can provide action list to reduce log scan time range. Input range (hours) = "
328                                + diffHours + " system allowed (hours) = " + logDuration);
329            }
330        }
331    }
332
333    /**
334     * Adjust offset, offset will always be in min.
335     *
336     * @param date the date
337     * @param offset the offset
338     * @return the date
339     * @throws IOException Signals that an I/O exception has occurred.
340     */
341    public Date adjustOffset(Date date, int offset) throws IOException {
342        return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
343    }
344
345    public void setFilterPattern(Pattern filterPattern) {
346        this.filterPattern = filterPattern;
347    }
348
349}