This project has retired. For details please refer to its Attic page.
Source code
001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import java.io.File;
022import java.util.ArrayList;
023import java.util.Calendar;
024import java.util.Collections;
025import java.util.concurrent.Semaphore;
026import java.util.regex.Matcher;
027import org.apache.log4j.Appender;
028import org.apache.log4j.rolling.RollingPolicyBase;
029import org.apache.log4j.rolling.RolloverDescription;
030import org.apache.log4j.rolling.TimeBasedRollingPolicy;
031import org.apache.log4j.rolling.TriggeringPolicy;
032import org.apache.log4j.spi.LoggingEvent;
033import org.apache.oozie.service.Services;
034import org.apache.oozie.service.XLogService;
035
036/**
037 * Has the same behavior as the TimeBasedRollingPolicy.  Additionally, it will delete older logs (MaxHistory determines how many
038 * older logs are retained).
039 */
040public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy {
041
042    /**
043     * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it
044     */
045    private TimeBasedRollingPolicy tbrp;
046    
047    private Semaphore deleteSem;
048    
049    private Thread deleteThread;
050    
051    private int maxHistory = 720;       // (720 hours / 24 hours per day = 30 days) as default
052    
053    public int getMaxHistory() {
054        return maxHistory;
055    }
056
057    public void setMaxHistory(int maxHistory) {
058        this.maxHistory = maxHistory;
059    }
060    
061    public OozieRollingPolicy() {
062        deleteSem = new Semaphore(1);
063        deleteThread = new Thread();
064        tbrp = new TimeBasedRollingPolicy();
065    }
066    
067    @Override
068    public void activateOptions() {
069        super.activateOptions();
070        tbrp.setFileNamePattern(getFileNamePattern());
071        tbrp.activateOptions();
072    }
073    
074    @Override
075    public RolloverDescription initialize(String file, boolean append) throws SecurityException {
076        return tbrp.initialize(file, append);
077    }
078    
079    @Override
080    public RolloverDescription rollover(final String activeFile) throws SecurityException {
081        return tbrp.rollover(activeFile);
082    }
083    
084    @Override
085    public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 
086    final long fileLength) {
087        if (maxHistory >= 0) {  // -1 = disable
088            // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete
089            // the old logs
090            if (deleteSem.tryAcquire()) {
091                if (!deleteThread.isAlive()) {
092                    // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else
093                    deleteThread = new Thread() {
094                        @Override
095                        public void run() {
096                            deleteOldFiles();
097                        }
098                    };
099                    deleteThread.start();
100                }
101                deleteSem.release();
102            }
103        }
104        return tbrp.isTriggeringEvent(appender, event, filename, fileLength);
105    }
106    
107    private void deleteOldFiles() {
108        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
109        XLogService xls = getXLogService();
110        if (xls != null) {      // We need this to get the paths
111            String oozieLogPath = xls.getOozieLogPath();
112            String logFile = xls.getOozieLogName();
113            if (oozieLogPath != null && logFile != null) {
114                String[] children = new File(oozieLogPath).list();
115                if (children != null) {
116                    for (String child : children) {
117                        if (child.startsWith(logFile) && !child.equals(logFile)) {
118                            File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child);
119                            if (child.endsWith(".gz")) {
120                                long gzFileCreationTime = getGZFileCreationTime(child);
121                                if (gzFileCreationTime != -1) {
122                                    fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
123                                }
124                            } else{ 
125                                long modTime = childFile.lastModified();
126                                fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
127                            }
128                        }
129                    }
130                }
131            }
132        }
133        
134        if (fileList.size() > maxHistory) {
135            Collections.sort(fileList);
136            
137            for (int i = maxHistory; i < fileList.size(); i++) {
138                new File(fileList.get(i).getFileName()).delete();
139            }
140        }
141    }
142    
143    private long getGZFileCreationTime(String fileName) {
144        // Default return value of -1 to exclude the file
145        long returnVal = -1;
146        Matcher m = XLogStreamer.gzTimePattern.matcher(fileName);
147        if (m.matches() && m.groupCount() == 4) {
148            int year = Integer.parseInt(m.group(1));
149            int month = Integer.parseInt(m.group(2));
150            int day = Integer.parseInt(m.group(3));
151            int hour = Integer.parseInt(m.group(4));
152            int minute = 0;
153            Calendar calendarEntry = Calendar.getInstance();
154            calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
155            long logFileStartTime = calendarEntry.getTimeInMillis();
156            returnVal = logFileStartTime;
157        }
158        return returnVal;
159    }
160    
161    class FileInfo implements Comparable<FileInfo> {
162        String fileName;
163        long modTime;
164
165        public FileInfo(String fileName, long modTime) {
166            this.fileName = fileName;
167            this.modTime = modTime;
168        }
169
170        public String getFileName() {
171            return fileName;
172        }
173
174        public long getModTime() {
175            return modTime;
176        }
177
178        public int compareTo(FileInfo fileInfo) {
179            // Note: the order is the reverse of XLogStreamer.FileInfo
180            long diff = fileInfo.modTime - this.modTime;
181            if (diff > 0) {
182                return 1;
183            }
184            else if (diff < 0) {
185                return -1;
186            }
187            else {
188                return 0;
189            }
190        }
191    }
192    
193    // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName()
194    // by overriding getXLogService()
195    XLogService getXLogService() {
196        if (Services.get() != null) {
197            return Services.get().get(XLogService.class);
198        }
199        return null;
200    }
201}