001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.util; 020 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.Date; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.regex.Matcher; 028import java.util.regex.Pattern; 029 030import org.apache.commons.lang.StringUtils; 031import org.apache.oozie.service.Services; 032 033import com.google.common.annotations.VisibleForTesting; 034 035/** 036 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if 037 * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|") 038 * jobId appName actionId token 039 */ 040public class XLogFilter { 041 042 private static final int LOG_TIME_BUFFER = 2; // in min 043 public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration"; 044 public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration"; 045 private Map<String, Integer> logLevels; 046 private final Map<String, String> filterParams; 047 private static List<String> parameters = new ArrayList<String>(); 048 private boolean noFilter; 049 private Pattern filterPattern; 050 private XLogUserFilterParam userLogFilter; 051 private Date endDate; 052 private Date startDate; 053 private boolean isActionList = false; 054 private String formattedEndDate; 055 private String formattedStartDate; 056 057 // TODO Patterns to be read from config file 058 private static final String DEFAULT_REGEX = "[^\\]]*"; 059 060 public static final String ALLOW_ALL_REGEX = "(.*)"; 061 private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)"; 062 private static final String WHITE_SPACE_REGEX = "\\s+"; 063 private static final String LOG_LEVEL_REGEX = "(\\w+)"; 064 private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX 065 + WHITE_SPACE_REGEX; 066 private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX); 067 068 public XLogFilter() { 069 this(new XLogUserFilterParam()); 070 } 071 072 public XLogFilter(XLogUserFilterParam userLogFilter) { 073 filterParams = new HashMap<String, String>(); 074 for (int i = 0; i < parameters.size(); i++) { 075 filterParams.put(parameters.get(i), DEFAULT_REGEX); 076 } 077 logLevels = null; 078 noFilter = true; 079 filterPattern = null; 080 setUserLogFilter(userLogFilter); 081 } 082 083 public void setLogLevel(String logLevel) { 084 if (logLevel != null && logLevel.trim().length() > 0) { 085 this.logLevels = new HashMap<String, Integer>(); 086 String[] levels = logLevel.split("\\|"); 087 for (int i = 0; i < levels.length; i++) { 088 String s = levels[i].trim().toUpperCase(); 089 try { 090 XLog.Level.valueOf(s); 091 } 092 catch (Exception ex) { 093 continue; 094 } 095 this.logLevels.put(levels[i].toUpperCase(), 1); 096 } 097 } 098 } 099 100 public void setParameter(String filterParam, String value) { 101 if (filterParams.containsKey(filterParam)) { 102 noFilter = false; 103 filterParams.put(filterParam, value); 104 } 105 } 106 107 public static void defineParameter(String filterParam) { 108 parameters.add(filterParam); 109 } 110 111 public boolean isFilterPresent() { 112 if (noFilter && logLevels == null) { 113 return false; 114 } 115 return true; 116 } 117 118 /** 119 * Checks if the logLevel and logMessage goes through the logFilter. 120 * 121 * @param logParts 122 * @return 123 */ 124 public boolean matches(ArrayList<String> logParts) { 125 if (getStartDate() != null) { 126 if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) { 127 return false; 128 } 129 } 130 String logLevel = logParts.get(1); 131 String logMessage = logParts.get(2); 132 if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) { 133 Matcher logMatcher = filterPattern.matcher(logMessage); 134 return logMatcher.matches(); 135 } 136 else { 137 return false; 138 } 139 } 140 141 /** 142 * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing timestamp, 143 * logLevel, and logMessage if the pattern matches i.e A new log statement, else returns null. 144 * 145 * @param logLine 146 * @return Array containing log level and log message 147 */ 148 public ArrayList<String> splitLogMessage(String logLine) { 149 Matcher splitter = SPLITTER_PATTERN.matcher(logLine); 150 if (splitter.matches()) { 151 ArrayList<String> logParts = new ArrayList<String>(); 152 logParts.add(splitter.group(1));// timestamp 153 logParts.add(splitter.group(2));// log level 154 logParts.add(splitter.group(3));// Log Message 155 return logParts; 156 } 157 else { 158 return null; 159 } 160 } 161 162 /** 163 * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be assigned 164 * if no filters are set. 165 */ 166 public void constructPattern() { 167 if (noFilter && logLevels == null) { 168 filterPattern = Pattern.compile(ALLOW_ALL_REGEX); 169 return; 170 } 171 StringBuilder sb = new StringBuilder(); 172 if (noFilter) { 173 sb.append("(.*)"); 174 } 175 else { 176 sb.append("(.* "); 177 for (int i = 0; i < parameters.size(); i++) { 178 sb.append(parameters.get(i) + "\\["); 179 sb.append(filterParams.get(parameters.get(i)) + "\\] "); 180 } 181 sb.append(".*)"); 182 } 183 if (!StringUtils.isEmpty(userLogFilter.getSearchText())) { 184 sb.append(userLogFilter.getSearchText() + ".*"); 185 } 186 filterPattern = Pattern.compile(sb.toString()); 187 } 188 189 public static void reset() { 190 parameters.clear(); 191 } 192 193 @VisibleForTesting 194 public final Map<String, String> getFilterParams() { 195 return filterParams; 196 } 197 198 public XLogUserFilterParam getUserLogFilter() { 199 return userLogFilter; 200 } 201 202 public void setUserLogFilter(XLogUserFilterParam userLogFilter) { 203 this.userLogFilter = userLogFilter; 204 setLogLevel(userLogFilter.getLogLevel()); 205 } 206 207 public Date getEndDate() { 208 return endDate; 209 } 210 211 public String getFormattedEndDate() { 212 return formattedEndDate; 213 } 214 215 public String getFormattedStartDate() { 216 return formattedStartDate; 217 } 218 219 public Date getStartDate() { 220 return startDate; 221 } 222 223 public boolean isDebugMode() { 224 return userLogFilter.isDebug(); 225 } 226 227 public int getLogLimit() { 228 return userLogFilter.getLimit(); 229 } 230 231 public String getDebugMessage() { 232 return "Log start time = " + getStartDate() + ". Log end time = " + getEndDate() + ". User Log Filter = " 233 + getUserLogFilter() + System.getProperty("line.separator"); 234 } 235 236 public boolean isActionList() { 237 return isActionList; 238 } 239 240 public void setActionList(boolean isActionList) { 241 this.isActionList = isActionList; 242 } 243 244 private void calculateScanDate(Date jobStartTime, Date jobEndTime) throws IOException { 245 246 if (userLogFilter.getStartDate() != null) { 247 startDate = userLogFilter.getStartDate(); 248 } 249 else if (userLogFilter.getStartOffset() != -1) { 250 startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset()); 251 } 252 else { 253 startDate = jobStartTime; 254 } 255 256 if (userLogFilter.getEndDate() != null) { 257 endDate = userLogFilter.getEndDate(); 258 } 259 else if (userLogFilter.getEndOffset() != -1) { 260 // If user has specified startdate as absolute then end offset will be on user start date, 261 // else end offset will be calculated on job startdate. 262 if (userLogFilter.getStartDate() != null) { 263 endDate = adjustOffset(startDate, userLogFilter.getEndOffset()); 264 } 265 else { 266 endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset()); 267 } 268 } 269 else { 270 endDate = jobEndTime; 271 } 272 // if recent offset is specified then start time = endtime - offset 273 if (getUserLogFilter().getRecent() != -1) { 274 startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1); 275 } 276 277 //add buffer iff dates are not asbsolute 278 if (userLogFilter.getStartDate() == null) { 279 startDate = adjustOffset(startDate, -LOG_TIME_BUFFER); 280 } 281 if (userLogFilter.getEndDate() == null) { 282 endDate = adjustOffset(endDate, LOG_TIME_BUFFER); 283 } 284 285 formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate()); 286 formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate()); 287 } 288 289 /** 290 * Calculate and validate date range. 291 * 292 * @param jobStartTime the job start time 293 * @param jobEndTime the job end time 294 * @throws IOException Signals that an I/O exception has occurred. 295 */ 296 public void calculateAndValidateDateRange(Date jobStartTime, Date jobEndTime) throws IOException { 297 // for testcase, otherwise jobStartTime and jobEndTime will be always set 298 if (jobStartTime == null || jobEndTime == null) { 299 return; 300 } 301 calculateScanDate(jobStartTime, jobEndTime); 302 303 if (startDate.after(endDate)) { 304 throw new IOException("Start time should be less than end time. startTime = " + startDate + " endtime = " 305 + endDate); 306 } 307 long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000); 308 if (isActionList) { 309 int actionLogDuration = Services.get().getConf().getInt(MAX_ACTIONLIST_SCAN_DURATION, -1); 310 if (actionLogDuration == -1) { 311 return; 312 } 313 if (diffHours > actionLogDuration) { 314 throw new IOException( 315 "Request log streaming time range with action list is higher than configured. Please reduce the scan " 316 + "time range. Input range (hours) = " + diffHours 317 + " system allowed (hours) with action list = " + actionLogDuration); 318 } 319 } 320 else { 321 int logDuration = Services.get().getConf().getInt(MAX_SCAN_DURATION, -1); 322 if (logDuration == -1) { 323 return; 324 } 325 if (diffHours > logDuration) { 326 throw new IOException( 327 "Request log streaming time range is higher than configured. Please reduce the scan time range. For coord" 328 + " jobs you can provide action list to reduce log scan time range. Input range (hours) = " 329 + diffHours + " system allowed (hours) = " + logDuration); 330 } 331 } 332 } 333 334 /** 335 * Adjust offset, offset will always be in min. 336 * 337 * @param date the date 338 * @param offset the offset 339 * @return the date 340 * @throws IOException Signals that an I/O exception has occurred. 341 */ 342 public Date adjustOffset(Date date, int offset) throws IOException { 343 return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset); 344 } 345 346}