001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.coord;
019
020 import java.util.Calendar;
021 import java.util.Date;
022 import java.util.HashMap;
023 import java.util.Iterator;
024 import java.util.List;
025 import java.util.Map;
026
027 import org.apache.hadoop.conf.Configuration;
028 import org.apache.oozie.service.ELService;
029 import org.apache.oozie.service.Services;
030 import org.apache.oozie.util.DateUtils;
031 import org.apache.oozie.util.ELEvaluator;
032 import org.apache.oozie.util.XmlUtils;
033 import org.jdom.Element;
034
035 /**
036 * This class provide different evaluators required at different stages
037 */
038 public class CoordELEvaluator {
039 public static final Integer MINUTE = 1;
040 public static final Integer HOUR = 60 * MINUTE;
041
042 /**
043 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
044 * 1)
045 *
046 * @param conf : Configuration containing property variables
047 * @return configured ELEvaluator
048 */
049 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
050 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
051 setConfigToEval(eval, conf);
052 return eval;
053 }
054
055 /**
056 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
057 *
058 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
059 * @param appInst : Application Instance related information such as Action creation Time
060 * @param conf :Configuration to substitute any variables
061 * @return configured ELEvaluator
062 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
063 */
064 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
065 throws Exception {
066 return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
067 }
068
069 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
070 Configuration conf) throws Exception {
071 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
072 setConfigToEval(eval, conf);
073 SyncCoordDataset ds = getDSObject(event);
074 CoordELFunctions.configureEvaluator(eval, ds, appInst);
075 return eval;
076 }
077
078 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
079 HashMap<String, String> dataNameList) throws Exception {
080 ELEvaluator eval = createELEvaluatorForGroup(conf, group);
081 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
082 String key = it.next();
083 String value = dataNameList.get(key);
084 eval.setVariable("oozie.dataname." + key, value);
085 }
086 return eval;
087 }
088
089 /**
090 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
091 * should be resolved when all other data dependencies are met.
092 *
093 * @param actualTime : Action start time
094 * @param nominalTime : Action creation time
095 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
096 * @param conf :Configuration to substitute any variables
097 * @return configured ELEvaluator
098 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
099 */
100 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
101 throws Exception {
102 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
103 setConfigToEval(eval, conf);
104 SyncCoordDataset ds = getDSObject(dEvent);
105 SyncCoordAction appInst = new SyncCoordAction();
106 appInst.setNominalTime(nominalTime);
107 appInst.setActualTime(actualTime);
108 CoordELFunctions.configureEvaluator(eval, ds, appInst);
109 eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
110 return eval;
111 }
112
113 /**
114 * Create a SLA evaluator to be used during Materialization
115 *
116 * @param nominalTime
117 * @param conf
118 * @return
119 * @throws Exception
120 */
121 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
122 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
123 setConfigToEval(eval, conf);
124 SyncCoordAction appInst = new SyncCoordAction();// TODO:
125 appInst.setNominalTime(nominalTime);
126 CoordELFunctions.configureEvaluator(eval, null, appInst);
127 return eval;
128 }
129
130 /**
131 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
132 *
133 * @param eJob : XML element for the application instance
134 * @param conf :Configuration to substitute any variables
135 * @return configured ELEvaluator
136 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
137 */
138 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
139 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
140 setConfigToEval(e, conf);
141 SyncCoordAction appInst = new SyncCoordAction();
142 String strNominalTime = eJob.getAttributeValue("action-nominal-time");
143 if (strNominalTime != null) {
144 appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
145 appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
146 appInst.setFrequency(Integer.parseInt(eJob.getAttributeValue("frequency")));
147 appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
148 appInst.setActionId(actionId);
149 appInst.setName(eJob.getAttributeValue("name"));
150 }
151 String strActualTime = eJob.getAttributeValue("action-actual-time");
152 if (strActualTime != null) {
153 appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
154 }
155 CoordELFunctions.configureEvaluator(e, null, appInst);
156 Element events = eJob.getChild("input-events", eJob.getNamespace());
157 if (events != null) {
158 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
159 if (data.getChild("uris", data.getNamespace()) != null) {
160 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
161 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
162 e.setVariable(".datain." + data.getAttributeValue("name"), uris);
163 }
164 else {
165 }
166 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
167 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
168 // check
169 // null
170 }
171 }
172 }
173 events = eJob.getChild("output-events", eJob.getNamespace());
174 if (events != null) {
175 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
176 if (data.getChild("uris", data.getNamespace()) != null) {
177 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
178 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
179 e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
180 }
181 else {
182 }// TODO
183 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
184 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
185 // check
186 // null
187 }
188 }
189 }
190 return e;
191 }
192
193 /**
194 * Create a new Evaluator to resolve URI temple with time specific constant
195 *
196 * @param strDate : Date-time
197 * @return configured ELEvaluator
198 * @throws Exception If there is any date-time string in wrong format, the exception is thrown
199 */
200 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
201 ELEvaluator eval = new ELEvaluator();
202 Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
203 // always???
204 date.setTime(DateUtils.parseDateOozieTZ(strDate));
205 eval.setVariable("YEAR", date.get(Calendar.YEAR));
206 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
207 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
208 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
209 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
210 return eval;
211 }
212
213 /**
214 * Create Dataset object using the Dataset XML information
215 *
216 * @param eData
217 * @return
218 * @throws Exception
219 */
220 private static SyncCoordDataset getDSObject(Element eData) throws Exception {
221 SyncCoordDataset ds = new SyncCoordDataset();
222 Element eDataset = eData.getChild("dataset", eData.getNamespace());
223 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
224 Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
225 ds.setInitInstance(initInstance);
226 if (eDataset.getAttributeValue("frequency") != null) {
227 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
228 ds.setFrequency(frequency);
229 ds.setType("SYNC");
230 if (eDataset.getAttributeValue("freq_timeunit") == null) {
231 throw new RuntimeException("No freq_timeunit defined in data set definition\n"
232 + XmlUtils.prettyPrint(eDataset));
233 }
234 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
235 if (eDataset.getAttributeValue("timezone") == null) {
236 throw new RuntimeException("No timezone defined in data set definition\n"
237 + XmlUtils.prettyPrint(eDataset));
238 }
239 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
240 if (eDataset.getAttributeValue("end_of_duration") == null) {
241 throw new RuntimeException("No end_of_duration defined in data set definition\n"
242 + XmlUtils.prettyPrint(eDataset));
243 }
244 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
245
246 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
247 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
248 ds.setDoneFlag(doneFlag);
249 }
250 else {
251 ds.setType("ASYNC");
252 }
253 String name = eDataset.getAttributeValue("name");
254 ds.setName(name);
255 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
256 // eData.getNamespace()));
257 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
258 ds.setUriTemplate(uriTemplate);
259 // ds.setTimeUnit(TimeUnit.MINUTES);
260 return ds;
261 }
262
263 /**
264 * Set all job configurations properties into evaluator.
265 *
266 * @param eval : Evaluator to set variables
267 * @param conf : configurations to set Evaluator
268 */
269 private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
270 for (Map.Entry<String, String> entry : conf) {
271 eval.setVariable(entry.getKey(), entry.getValue().trim());
272 }
273 }
274
275 /**
276 * make any one digit number to two digit string pre-appending a"0"
277 *
278 * @param num : number to make sting
279 * @return :String of length at least two digit.
280 */
281 private static String make2Digits(int num) {
282 String ret = "" + num;
283 if (num <= 9) {
284 ret = "0" + ret;
285 }
286 return ret;
287 }
288 }