001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.util; 020 021import java.io.File; 022import java.util.ArrayList; 023import java.util.Calendar; 024import java.util.Collections; 025import java.util.concurrent.Semaphore; 026import java.util.regex.Matcher; 027import org.apache.log4j.Appender; 028import org.apache.log4j.rolling.RollingPolicyBase; 029import org.apache.log4j.rolling.RolloverDescription; 030import org.apache.log4j.rolling.TimeBasedRollingPolicy; 031import org.apache.log4j.rolling.TriggeringPolicy; 032import org.apache.log4j.spi.LoggingEvent; 033import org.apache.oozie.service.Services; 034import org.apache.oozie.service.XLogService; 035 036/** 037 * Has the same behavior as the TimeBasedRollingPolicy. Additionally, it will delete older logs (MaxHistory determines how many 038 * older logs are retained). 039 */ 040public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy { 041 042 /** 043 * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it 044 */ 045 private TimeBasedRollingPolicy tbrp; 046 047 private Semaphore deleteSem; 048 049 private Thread deleteThread; 050 051 private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default 052 053 public int getMaxHistory() { 054 return maxHistory; 055 } 056 057 public void setMaxHistory(int maxHistory) { 058 this.maxHistory = maxHistory; 059 } 060 061 public OozieRollingPolicy() { 062 deleteSem = new Semaphore(1); 063 deleteThread = new Thread(); 064 tbrp = new TimeBasedRollingPolicy(); 065 } 066 067 @Override 068 public void activateOptions() { 069 super.activateOptions(); 070 tbrp.setFileNamePattern(getFileNamePattern()); 071 tbrp.activateOptions(); 072 } 073 074 @Override 075 public RolloverDescription initialize(String file, boolean append) throws SecurityException { 076 return tbrp.initialize(file, append); 077 } 078 079 @Override 080 public RolloverDescription rollover(final String activeFile) throws SecurityException { 081 return tbrp.rollover(activeFile); 082 } 083 084 @Override 085 public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 086 final long fileLength) { 087 if (maxHistory >= 0) { // -1 = disable 088 // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete 089 // the old logs 090 if (deleteSem.tryAcquire()) { 091 if (!deleteThread.isAlive()) { 092 // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else 093 deleteThread = new Thread() { 094 @Override 095 public void run() { 096 deleteOldFiles(); 097 } 098 }; 099 deleteThread.start(); 100 } 101 deleteSem.release(); 102 } 103 } 104 return tbrp.isTriggeringEvent(appender, event, filename, fileLength); 105 } 106 107 private void deleteOldFiles() { 108 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 109 XLogService xls = getXLogService(); 110 if (xls != null) { // We need this to get the paths 111 String oozieLogPath = xls.getOozieLogPath(); 112 String logFile = xls.getOozieLogName(); 113 if (oozieLogPath != null && logFile != null) { 114 String[] children = new File(oozieLogPath).list(); 115 if (children != null) { 116 for (String child : children) { 117 if (child.startsWith(logFile) && !child.equals(logFile)) { 118 File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child); 119 if (child.endsWith(".gz")) { 120 long gzFileCreationTime = getGZFileCreationTime(child); 121 if (gzFileCreationTime != -1) { 122 fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime)); 123 } 124 } else{ 125 long modTime = childFile.lastModified(); 126 fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime)); 127 } 128 } 129 } 130 } 131 } 132 } 133 134 if (fileList.size() > maxHistory) { 135 Collections.sort(fileList); 136 137 for (int i = maxHistory; i < fileList.size(); i++) { 138 new File(fileList.get(i).getFileName()).delete(); 139 } 140 } 141 } 142 143 private long getGZFileCreationTime(String fileName) { 144 // Default return value of -1 to exclude the file 145 long returnVal = -1; 146 Matcher m = XLogStreamer.gzTimePattern.matcher(fileName); 147 if (m.matches() && m.groupCount() == 4) { 148 int year = Integer.parseInt(m.group(1)); 149 int month = Integer.parseInt(m.group(2)); 150 int day = Integer.parseInt(m.group(3)); 151 int hour = Integer.parseInt(m.group(4)); 152 int minute = 0; 153 Calendar calendarEntry = Calendar.getInstance(); 154 calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August) 155 long logFileStartTime = calendarEntry.getTimeInMillis(); 156 returnVal = logFileStartTime; 157 } 158 return returnVal; 159 } 160 161 class FileInfo implements Comparable<FileInfo> { 162 String fileName; 163 long modTime; 164 165 public FileInfo(String fileName, long modTime) { 166 this.fileName = fileName; 167 this.modTime = modTime; 168 } 169 170 public String getFileName() { 171 return fileName; 172 } 173 174 public long getModTime() { 175 return modTime; 176 } 177 178 public int compareTo(FileInfo fileInfo) { 179 // Note: the order is the reverse of XLogStreamer.FileInfo 180 long diff = fileInfo.modTime - this.modTime; 181 if (diff > 0) { 182 return 1; 183 } 184 else if (diff < 0) { 185 return -1; 186 } 187 else { 188 return 0; 189 } 190 } 191 } 192 193 // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName() 194 // by overriding getXLogService() 195 XLogService getXLogService() { 196 if (Services.get() != null) { 197 return Services.get().get(XLogService.class); 198 } 199 return null; 200 } 201}