001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.util;
019
020 import java.io.File;
021 import java.util.ArrayList;
022 import java.util.Calendar;
023 import java.util.Collections;
024 import java.util.concurrent.Semaphore;
025 import java.util.regex.Matcher;
026 import org.apache.log4j.Appender;
027 import org.apache.log4j.rolling.RollingPolicyBase;
028 import org.apache.log4j.rolling.RolloverDescription;
029 import org.apache.log4j.rolling.TimeBasedRollingPolicy;
030 import org.apache.log4j.rolling.TriggeringPolicy;
031 import org.apache.log4j.spi.LoggingEvent;
032 import org.apache.oozie.service.Services;
033 import org.apache.oozie.service.XLogService;
034
035 /**
036 * Has the same behavior as the TimeBasedRollingPolicy. Additionally, it will delete older logs (MaxHistory determines how many
037 * older logs are retained).
038 */
039 public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy {
040
041 /**
042 * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it
043 */
044 private TimeBasedRollingPolicy tbrp;
045
046 private Semaphore deleteSem;
047
048 private Thread deleteThread;
049
050 private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default
051
052 public int getMaxHistory() {
053 return maxHistory;
054 }
055
056 public void setMaxHistory(int maxHistory) {
057 this.maxHistory = maxHistory;
058 }
059
060 public OozieRollingPolicy() {
061 deleteSem = new Semaphore(1);
062 deleteThread = new Thread();
063 tbrp = new TimeBasedRollingPolicy();
064 }
065
066 @Override
067 public void activateOptions() {
068 super.activateOptions();
069 tbrp.setFileNamePattern(getFileNamePattern());
070 tbrp.activateOptions();
071 }
072
073 @Override
074 public RolloverDescription initialize(String file, boolean append) throws SecurityException {
075 return tbrp.initialize(file, append);
076 }
077
078 @Override
079 public RolloverDescription rollover(final String activeFile) throws SecurityException {
080 return tbrp.rollover(activeFile);
081 }
082
083 @Override
084 public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename,
085 final long fileLength) {
086 if (maxHistory >= 0) { // -1 = disable
087 // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete
088 // the old logs
089 if (deleteSem.tryAcquire()) {
090 if (!deleteThread.isAlive()) {
091 // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else
092 deleteThread = new Thread() {
093 @Override
094 public void run() {
095 deleteOldFiles();
096 }
097 };
098 deleteThread.start();
099 }
100 deleteSem.release();
101 }
102 }
103 return tbrp.isTriggeringEvent(appender, event, filename, fileLength);
104 }
105
106 private void deleteOldFiles() {
107 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
108 XLogService xls = getXLogService();
109 if (xls != null) { // We need this to get the paths
110 String oozieLogPath = xls.getOozieLogPath();
111 String logFile = xls.getOozieLogName();
112 if (oozieLogPath != null && logFile != null) {
113 String[] children = new File(oozieLogPath).list();
114 if (children != null) {
115 for (String child : children) {
116 if (child.startsWith(logFile) && !child.equals(logFile)) {
117 File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child);
118 if (child.endsWith(".gz")) {
119 long gzFileCreationTime = getGZFileCreationTime(child);
120 if (gzFileCreationTime != -1) {
121 fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
122 }
123 } else{
124 long modTime = childFile.lastModified();
125 fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
126 }
127 }
128 }
129 }
130 }
131 }
132
133 if (fileList.size() > maxHistory) {
134 Collections.sort(fileList);
135
136 for (int i = maxHistory; i < fileList.size(); i++) {
137 new File(fileList.get(i).getFileName()).delete();
138 }
139 }
140 }
141
142 private long getGZFileCreationTime(String fileName) {
143 // Default return value of -1 to exclude the file
144 long returnVal = -1;
145 Matcher m = XLogStreamer.gzTimePattern.matcher(fileName);
146 if (m.matches() && m.groupCount() == 4) {
147 int year = Integer.parseInt(m.group(1));
148 int month = Integer.parseInt(m.group(2));
149 int day = Integer.parseInt(m.group(3));
150 int hour = Integer.parseInt(m.group(4));
151 int minute = 0;
152 Calendar calendarEntry = Calendar.getInstance();
153 calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
154 long logFileStartTime = calendarEntry.getTimeInMillis();
155 returnVal = logFileStartTime;
156 }
157 return returnVal;
158 }
159
160 class FileInfo implements Comparable<FileInfo> {
161 String fileName;
162 long modTime;
163
164 public FileInfo(String fileName, long modTime) {
165 this.fileName = fileName;
166 this.modTime = modTime;
167 }
168
169 public String getFileName() {
170 return fileName;
171 }
172
173 public long getModTime() {
174 return modTime;
175 }
176
177 public int compareTo(FileInfo fileInfo) {
178 // Note: the order is the reverse of XLogStreamer.FileInfo
179 long diff = fileInfo.modTime - this.modTime;
180 if (diff > 0) {
181 return 1;
182 }
183 else if (diff < 0) {
184 return -1;
185 }
186 else {
187 return 0;
188 }
189 }
190 }
191
192 // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName()
193 // by overriding getXLogService()
194 XLogService getXLogService() {
195 if (Services.get() != null) {
196 return Services.get().get(XLogService.class);
197 }
198 return null;
199 }
200 }