001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.util; 020 021import java.io.File; 022import java.text.ParseException; 023import java.text.SimpleDateFormat; 024import java.util.ArrayList; 025import java.util.Collections; 026import java.util.List; 027import java.util.concurrent.Semaphore; 028 029import org.apache.log4j.Appender; 030import org.apache.log4j.pattern.ExtrasPatternParser; 031import org.apache.log4j.rolling.RollingPolicyBase; 032import org.apache.log4j.rolling.RolloverDescription; 033import org.apache.log4j.rolling.TimeBasedRollingPolicy; 034import org.apache.log4j.rolling.TriggeringPolicy; 035import org.apache.log4j.spi.LoggingEvent; 036import org.apache.log4j.pattern.LiteralPatternConverter; 037import org.apache.oozie.service.Services; 038import org.apache.oozie.service.XLogService; 039 040/** 041 * Has the same behavior as the TimeBasedRollingPolicy. Additionally, it will delete older logs (MaxHistory determines how many 042 * older logs are retained). 043 */ 044public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy { 045 046 /** 047 * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it 048 */ 049 private TimeBasedRollingPolicy tbrp; 050 051 private Semaphore deleteSem; 052 053 private Thread deleteThread; 054 055 private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default 056 057 String oozieLogDir; 058 String logFileName; 059 060 public int getMaxHistory() { 061 return maxHistory; 062 } 063 064 public void setMaxHistory(int maxHistory) { 065 this.maxHistory = maxHistory; 066 } 067 068 public OozieRollingPolicy() { 069 deleteSem = new Semaphore(1); 070 deleteThread = new Thread(); 071 tbrp = new TimeBasedRollingPolicy(); 072 } 073 074 075 @SuppressWarnings("rawtypes") 076 public void setFileNamePattern(String fnp) { 077 super.setFileNamePattern(fnp); 078 List converters = new ArrayList(); 079 StringBuffer file = new StringBuffer(); 080 ExtrasPatternParser 081 .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules()); 082 if (!converters.isEmpty()) { 083 ((LiteralPatternConverter) converters.get(0)).format(null, file); 084 File f = new File(file.toString()); 085 oozieLogDir = f.getParent(); 086 logFileName = f.getName(); 087 } 088 else { 089 XLogService xls = Services.get().get(XLogService.class); 090 oozieLogDir = xls.getOozieLogPath(); 091 logFileName = xls.getOozieLogName(); 092 } 093 094 } 095 @Override 096 public void activateOptions() { 097 super.activateOptions(); 098 tbrp.setFileNamePattern(getFileNamePattern()); 099 tbrp.activateOptions(); 100 } 101 102 @Override 103 public RolloverDescription initialize(String file, boolean append) throws SecurityException { 104 return tbrp.initialize(file, append); 105 } 106 107 @Override 108 public RolloverDescription rollover(final String activeFile) throws SecurityException { 109 return tbrp.rollover(activeFile); 110 } 111 112 @Override 113 public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 114 final long fileLength) { 115 if (maxHistory >= 0) { // -1 = disable 116 // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting up to delete 117 // the old logs 118 if (deleteSem.tryAcquire()) { 119 if (!deleteThread.isAlive()) { 120 // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else 121 deleteThread = new Thread() { 122 @Override 123 public void run() { 124 deleteOldFiles(); 125 } 126 }; 127 deleteThread.start(); 128 } 129 deleteSem.release(); 130 } 131 } 132 return tbrp.isTriggeringEvent(appender, event, filename, fileLength); 133 } 134 135 private void deleteOldFiles() { 136 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 137 if (oozieLogDir != null && logFileName != null) { 138 String[] children = new File(oozieLogDir).list(); 139 if (children != null) { 140 for (String child : children) { 141 if (child.startsWith(logFileName) && !child.equals(logFileName)) { 142 File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child); 143 if (child.endsWith(".gz")) { 144 long gzFileCreationTime = getGZFileCreationTime(child); 145 if (gzFileCreationTime != -1) { 146 fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime)); 147 } 148 } 149 else { 150 long modTime = childFile.lastModified(); 151 fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime)); 152 } 153 } 154 } 155 } 156 } 157 158 if (fileList.size() > maxHistory) { 159 Collections.sort(fileList); 160 161 for (int i = maxHistory; i < fileList.size(); i++) { 162 new File(fileList.get(i).getFileName()).delete(); 163 } 164 } 165 } 166 167 private long getGZFileCreationTime(String fileName) { 168 SimpleDateFormat formatter; 169 String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz 170 if (date.length() == 10) { 171 formatter = new SimpleDateFormat("yyyy-MM-dd"); 172 } 173 else { 174 formatter = new SimpleDateFormat("yyyy-MM-dd-HH"); 175 } 176 try { 177 return formatter.parse(date).getTime(); 178 } 179 catch (ParseException e) { 180 return -1; 181 } 182 } 183 184 class FileInfo implements Comparable<FileInfo> { 185 String fileName; 186 long modTime; 187 188 public FileInfo(String fileName, long modTime) { 189 this.fileName = fileName; 190 this.modTime = modTime; 191 } 192 193 public String getFileName() { 194 return fileName; 195 } 196 197 public long getModTime() { 198 return modTime; 199 } 200 201 public int compareTo(FileInfo fileInfo) { 202 // Note: the order is the reverse of XLogStreamer.FileInfo 203 long diff = fileInfo.modTime - this.modTime; 204 if (diff > 0) { 205 return 1; 206 } 207 else if (diff < 0) { 208 return -1; 209 } 210 else { 211 return 0; 212 } 213 } 214 } 215 216}