001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import java.io.File;
022import java.text.ParseException;
023import java.text.SimpleDateFormat;
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.List;
027import java.util.concurrent.Semaphore;
028
029import org.apache.log4j.Appender;
030import org.apache.log4j.pattern.ExtrasPatternParser;
031import org.apache.log4j.rolling.RollingPolicyBase;
032import org.apache.log4j.rolling.RolloverDescription;
033import org.apache.log4j.rolling.TimeBasedRollingPolicy;
034import org.apache.log4j.rolling.TriggeringPolicy;
035import org.apache.log4j.spi.LoggingEvent;
036import org.apache.log4j.pattern.LiteralPatternConverter;
037import org.apache.oozie.service.Services;
038import org.apache.oozie.service.XLogService;
039
040/**
041 * Has the same behavior as the TimeBasedRollingPolicy.  Additionally, it will delete older logs (MaxHistory determines how many
042 * older logs are retained).
043 */
044public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy {
045
046    /**
047     * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it
048     */
049    private TimeBasedRollingPolicy tbrp;
050    
051    private Semaphore deleteSem;
052    
053    private Thread deleteThread;
054    
055    private int maxHistory = 720;       // (720 hours / 24 hours per day = 30 days) as default
056
057    String oozieLogDir;
058    String logFileName;
059
060    public int getMaxHistory() {
061        return maxHistory;
062    }
063
064    public void setMaxHistory(int maxHistory) {
065        this.maxHistory = maxHistory;
066    }
067    
068    public OozieRollingPolicy() {
069        deleteSem = new Semaphore(1);
070        deleteThread = new Thread();
071        tbrp = new TimeBasedRollingPolicy();
072    }
073
074
075    @SuppressWarnings("rawtypes")
076    public void setFileNamePattern(String fnp) {
077        super.setFileNamePattern(fnp);
078        List converters = new ArrayList();
079        StringBuffer file = new StringBuffer();
080        ExtrasPatternParser
081                .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules());
082        if (!converters.isEmpty()) {
083            ((LiteralPatternConverter) converters.get(0)).format(null, file);
084            File f = new File(file.toString());
085            oozieLogDir = f.getParent();
086            logFileName = f.getName();
087        }
088        else {
089            XLogService xls = Services.get().get(XLogService.class);
090            oozieLogDir = xls.getOozieLogPath();
091            logFileName = xls.getOozieLogName();
092        }
093
094    }
095    @Override
096    public void activateOptions() {
097        super.activateOptions();
098        tbrp.setFileNamePattern(getFileNamePattern());
099        tbrp.activateOptions();
100    }
101    
102    @Override
103    public RolloverDescription initialize(String file, boolean append) throws SecurityException {
104        return tbrp.initialize(file, append);
105    }
106    
107    @Override
108    public RolloverDescription rollover(final String activeFile) throws SecurityException {
109        return tbrp.rollover(activeFile);
110    }
111    
112    @Override
113    public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 
114    final long fileLength) {
115        if (maxHistory >= 0) {  // -1 = disable
116            // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete
117            // the old logs
118            if (deleteSem.tryAcquire()) {
119                if (!deleteThread.isAlive()) {
120                    // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else
121                    deleteThread = new Thread() {
122                        @Override
123                        public void run() {
124                            deleteOldFiles();
125                        }
126                    };
127                    deleteThread.start();
128                }
129                deleteSem.release();
130            }
131        }
132        return tbrp.isTriggeringEvent(appender, event, filename, fileLength);
133    }
134    
135    private void deleteOldFiles() {
136        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
137        if (oozieLogDir != null && logFileName != null) {
138            String[] children = new File(oozieLogDir).list();
139            if (children != null) {
140                for (String child : children) {
141                    if (child.startsWith(logFileName) && !child.equals(logFileName)) {
142                        File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child);
143                        if (child.endsWith(".gz")) {
144                            long gzFileCreationTime = getGZFileCreationTime(child);
145                            if (gzFileCreationTime != -1) {
146                                fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
147                            }
148                        }
149                        else {
150                            long modTime = childFile.lastModified();
151                            fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
152                        }
153                    }
154                }
155            }
156        }
157
158        if (fileList.size() > maxHistory) {
159            Collections.sort(fileList);
160
161            for (int i = maxHistory; i < fileList.size(); i++) {
162                new File(fileList.get(i).getFileName()).delete();
163            }
164        }
165    }
166    
167    private long getGZFileCreationTime(String fileName) {
168        SimpleDateFormat formatter;
169        String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz
170        if (date.length() == 10) {
171            formatter = new SimpleDateFormat("yyyy-MM-dd");
172        }
173        else {
174            formatter = new SimpleDateFormat("yyyy-MM-dd-HH");
175        }
176        try {
177            return formatter.parse(date).getTime();
178        }
179        catch (ParseException e) {
180            return -1;
181        }
182    }
183
184    class FileInfo implements Comparable<FileInfo> {
185        String fileName;
186        long modTime;
187
188        public FileInfo(String fileName, long modTime) {
189            this.fileName = fileName;
190            this.modTime = modTime;
191        }
192
193        public String getFileName() {
194            return fileName;
195        }
196
197        public long getModTime() {
198            return modTime;
199        }
200
201        public int compareTo(FileInfo fileInfo) {
202            // Note: the order is the reverse of XLogStreamer.FileInfo
203            long diff = fileInfo.modTime - this.modTime;
204            if (diff > 0) {
205                return 1;
206            }
207            else if (diff < 0) {
208                return -1;
209            }
210            else {
211                return 0;
212            }
213        }
214    }
215    
216}