001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.util;
019
020import java.io.File;
021import java.util.ArrayList;
022import java.util.Calendar;
023import java.util.Collections;
024import java.util.concurrent.Semaphore;
025import java.util.regex.Matcher;
026import org.apache.log4j.Appender;
027import org.apache.log4j.rolling.RollingPolicyBase;
028import org.apache.log4j.rolling.RolloverDescription;
029import org.apache.log4j.rolling.TimeBasedRollingPolicy;
030import org.apache.log4j.rolling.TriggeringPolicy;
031import org.apache.log4j.spi.LoggingEvent;
032import org.apache.oozie.service.Services;
033import org.apache.oozie.service.XLogService;
034
035/**
036 * Has the same behavior as the TimeBasedRollingPolicy.  Additionally, it will delete older logs (MaxHistory determines how many
037 * older logs are retained).
038 */
039public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy {
040
041    /**
042     * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it
043     */
044    private TimeBasedRollingPolicy tbrp;
045    
046    private Semaphore deleteSem;
047    
048    private Thread deleteThread;
049    
050    private int maxHistory = 720;       // (720 hours / 24 hours per day = 30 days) as default
051    
052    public int getMaxHistory() {
053        return maxHistory;
054    }
055
056    public void setMaxHistory(int maxHistory) {
057        this.maxHistory = maxHistory;
058    }
059    
060    public OozieRollingPolicy() {
061        deleteSem = new Semaphore(1);
062        deleteThread = new Thread();
063        tbrp = new TimeBasedRollingPolicy();
064    }
065    
066    @Override
067    public void activateOptions() {
068        super.activateOptions();
069        tbrp.setFileNamePattern(getFileNamePattern());
070        tbrp.activateOptions();
071    }
072    
073    @Override
074    public RolloverDescription initialize(String file, boolean append) throws SecurityException {
075        return tbrp.initialize(file, append);
076    }
077    
078    @Override
079    public RolloverDescription rollover(final String activeFile) throws SecurityException {
080        return tbrp.rollover(activeFile);
081    }
082    
083    @Override
084    public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 
085    final long fileLength) {
086        if (maxHistory >= 0) {  // -1 = disable
087            // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete
088            // the old logs
089            if (deleteSem.tryAcquire()) {
090                if (!deleteThread.isAlive()) {
091                    // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else
092                    deleteThread = new Thread() {
093                        @Override
094                        public void run() {
095                            deleteOldFiles();
096                        }
097                    };
098                    deleteThread.start();
099                }
100                deleteSem.release();
101            }
102        }
103        return tbrp.isTriggeringEvent(appender, event, filename, fileLength);
104    }
105    
106    private void deleteOldFiles() {
107        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
108        XLogService xls = getXLogService();
109        if (xls != null) {      // We need this to get the paths
110            String oozieLogPath = xls.getOozieLogPath();
111            String logFile = xls.getOozieLogName();
112            if (oozieLogPath != null && logFile != null) {
113                String[] children = new File(oozieLogPath).list();
114                if (children != null) {
115                    for (String child : children) {
116                        if (child.startsWith(logFile) && !child.equals(logFile)) {
117                            File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child);
118                            if (child.endsWith(".gz")) {
119                                long gzFileCreationTime = getGZFileCreationTime(child);
120                                if (gzFileCreationTime != -1) {
121                                    fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
122                                }
123                            } else{ 
124                                long modTime = childFile.lastModified();
125                                fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
126                            }
127                        }
128                    }
129                }
130            }
131        }
132        
133        if (fileList.size() > maxHistory) {
134            Collections.sort(fileList);
135            
136            for (int i = maxHistory; i < fileList.size(); i++) {
137                new File(fileList.get(i).getFileName()).delete();
138            }
139        }
140    }
141    
142    private long getGZFileCreationTime(String fileName) {
143        // Default return value of -1 to exclude the file
144        long returnVal = -1;
145        Matcher m = XLogStreamer.gzTimePattern.matcher(fileName);
146        if (m.matches() && m.groupCount() == 4) {
147            int year = Integer.parseInt(m.group(1));
148            int month = Integer.parseInt(m.group(2));
149            int day = Integer.parseInt(m.group(3));
150            int hour = Integer.parseInt(m.group(4));
151            int minute = 0;
152            Calendar calendarEntry = Calendar.getInstance();
153            calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
154            long logFileStartTime = calendarEntry.getTimeInMillis();
155            returnVal = logFileStartTime;
156        }
157        return returnVal;
158    }
159    
160    class FileInfo implements Comparable<FileInfo> {
161        String fileName;
162        long modTime;
163
164        public FileInfo(String fileName, long modTime) {
165            this.fileName = fileName;
166            this.modTime = modTime;
167        }
168
169        public String getFileName() {
170            return fileName;
171        }
172
173        public long getModTime() {
174            return modTime;
175        }
176
177        public int compareTo(FileInfo fileInfo) {
178            // Note: the order is the reverse of XLogStreamer.FileInfo
179            long diff = fileInfo.modTime - this.modTime;
180            if (diff > 0) {
181                return 1;
182            }
183            else if (diff < 0) {
184                return -1;
185            }
186            else {
187                return 0;
188            }
189        }
190    }
191    
192    // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName()
193    // by overriding getXLogService()
194    XLogService getXLogService() {
195        if (Services.get() != null) {
196            return Services.get().get(XLogService.class);
197        }
198        return null;
199    }
200}