001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.coord;
019
020 import java.util.Calendar;
021 import java.util.Date;
022 import java.util.HashMap;
023 import java.util.Iterator;
024 import java.util.List;
025 import java.util.Map;
026
027 import org.apache.hadoop.conf.Configuration;
028 import org.apache.oozie.command.coord.CoordCommandUtils;
029 import org.apache.oozie.service.ELService;
030 import org.apache.oozie.service.Services;
031 import org.apache.oozie.util.DateUtils;
032 import org.apache.oozie.util.ELEvaluator;
033 import org.apache.oozie.util.XmlUtils;
034 import org.jdom.Element;
035
036 /**
037 * This class provide different evaluators required at different stages
038 */
039 public class CoordELEvaluator {
040 public static final Integer MINUTE = 1;
041 public static final Integer HOUR = 60 * MINUTE;
042
043 /**
044 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
045 * 1)
046 *
047 * @param conf : Configuration containing property variables
048 * @return configured ELEvaluator
049 */
050 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
051 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
052 setConfigToEval(eval, conf);
053 return eval;
054 }
055
056 /**
057 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
058 *
059 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
060 * @param appInst : Application Instance related information such as Action creation Time
061 * @param conf :Configuration to substitute any variables
062 * @return configured ELEvaluator
063 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
064 */
065 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
066 throws Exception {
067 return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
068 }
069
070 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
071 Configuration conf) throws Exception {
072 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
073 setConfigToEval(eval, conf);
074 SyncCoordDataset ds = getDSObject(event);
075 CoordELFunctions.configureEvaluator(eval, ds, appInst);
076 return eval;
077 }
078
079 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
080 HashMap<String, String> dataNameList) throws Exception {
081 ELEvaluator eval = createELEvaluatorForGroup(conf, group);
082 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
083 String key = it.next();
084 String value = dataNameList.get(key);
085 eval.setVariable("oozie.dataname." + key, value);
086 }
087 return eval;
088 }
089
090 /**
091 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
092 * should be resolved when all other data dependencies are met.
093 *
094 * @param actualTime : Action start time
095 * @param nominalTime : Action creation time
096 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
097 * @param conf :Configuration to substitute any variables
098 * @return configured ELEvaluator
099 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
100 */
101 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
102 throws Exception {
103 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
104 setConfigToEval(eval, conf);
105 SyncCoordDataset ds = getDSObject(dEvent);
106 SyncCoordAction appInst = new SyncCoordAction();
107 appInst.setNominalTime(nominalTime);
108 appInst.setActualTime(actualTime);
109 CoordELFunctions.configureEvaluator(eval, ds, appInst);
110 eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
111 return eval;
112 }
113
114 /**
115 * Create a SLA evaluator to be used during Materialization
116 *
117 * @param nominalTime
118 * @param conf
119 * @return
120 * @throws Exception
121 */
122 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
123 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
124 setConfigToEval(eval, conf);
125 SyncCoordAction appInst = new SyncCoordAction();// TODO:
126 appInst.setNominalTime(nominalTime);
127 CoordELFunctions.configureEvaluator(eval, null, appInst);
128 return eval;
129 }
130
131 /**
132 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
133 *
134 * @param eJob : XML element for the application instance
135 * @param conf :Configuration to substitute any variables
136 * @return configured ELEvaluator
137 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
138 */
139 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
140 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
141 setConfigToEval(e, conf);
142 SyncCoordAction appInst = new SyncCoordAction();
143 String strNominalTime = eJob.getAttributeValue("action-nominal-time");
144 if (strNominalTime != null) {
145 appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
146 appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
147 appInst.setFrequency(Integer.parseInt(eJob.getAttributeValue("frequency")));
148 appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
149 appInst.setActionId(actionId);
150 appInst.setName(eJob.getAttributeValue("name"));
151 }
152 String strActualTime = eJob.getAttributeValue("action-actual-time");
153 if (strActualTime != null) {
154 appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
155 }
156 CoordELFunctions.configureEvaluator(e, null, appInst);
157 Element events = eJob.getChild("input-events", eJob.getNamespace());
158 if (events != null) {
159 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
160 if (data.getChild("uris", data.getNamespace()) != null) {
161 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
162 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
163 e.setVariable(".datain." + data.getAttributeValue("name"), uris);
164 }
165 else {
166 }
167 if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
168 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
169 // check
170 // null
171 }
172 }
173 }
174 events = eJob.getChild("output-events", eJob.getNamespace());
175 if (events != null) {
176 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
177 if (data.getChild("uris", data.getNamespace()) != null) {
178 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
179 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
180 e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
181 }
182 else {
183 }// TODO
184 if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
185 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
186 // check
187 // null
188 }
189 }
190 }
191 return e;
192 }
193
194 /**
195 * Create a new Evaluator to resolve URI temple with time specific constant
196 *
197 * @param strDate : Date-time
198 * @return configured ELEvaluator
199 * @throws Exception If there is any date-time string in wrong format, the exception is thrown
200 */
201 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
202 ELEvaluator eval = new ELEvaluator();
203 Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
204 // always???
205 date.setTime(DateUtils.parseDateOozieTZ(strDate));
206 eval.setVariable("YEAR", date.get(Calendar.YEAR));
207 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
208 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
209 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
210 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
211 return eval;
212 }
213
214 /**
215 * Create Dataset object using the Dataset XML information
216 *
217 * @param eData
218 * @return
219 * @throws Exception
220 */
221 private static SyncCoordDataset getDSObject(Element eData) throws Exception {
222 SyncCoordDataset ds = new SyncCoordDataset();
223 Element eDataset = eData.getChild("dataset", eData.getNamespace());
224 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
225 Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
226 ds.setInitInstance(initInstance);
227 if (eDataset.getAttributeValue("frequency") != null) {
228 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
229 ds.setFrequency(frequency);
230 ds.setType("SYNC");
231 if (eDataset.getAttributeValue("freq_timeunit") == null) {
232 throw new RuntimeException("No freq_timeunit defined in data set definition\n"
233 + XmlUtils.prettyPrint(eDataset));
234 }
235 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
236 if (eDataset.getAttributeValue("timezone") == null) {
237 throw new RuntimeException("No timezone defined in data set definition\n"
238 + XmlUtils.prettyPrint(eDataset));
239 }
240 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
241 if (eDataset.getAttributeValue("end_of_duration") == null) {
242 throw new RuntimeException("No end_of_duration defined in data set definition\n"
243 + XmlUtils.prettyPrint(eDataset));
244 }
245 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
246
247 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
248 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
249 ds.setDoneFlag(doneFlag);
250 }
251 else {
252 ds.setType("ASYNC");
253 }
254 String name = eDataset.getAttributeValue("name");
255 ds.setName(name);
256 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
257 // eData.getNamespace()));
258 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
259 ds.setUriTemplate(uriTemplate);
260 // ds.setTimeUnit(TimeUnit.MINUTES);
261 return ds;
262 }
263
264 /**
265 * Set all job configurations properties into evaluator.
266 *
267 * @param eval : Evaluator to set variables
268 * @param conf : configurations to set Evaluator
269 */
270 private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
271 for (Map.Entry<String, String> entry : conf) {
272 eval.setVariable(entry.getKey(), entry.getValue().trim());
273 }
274 }
275
276 /**
277 * make any one digit number to two digit string pre-appending a"0"
278 *
279 * @param num : number to make sting
280 * @return :String of length at least two digit.
281 */
282 private static String make2Digits(int num) {
283 String ret = "" + num;
284 if (num <= 9) {
285 ret = "0" + ret;
286 }
287 return ret;
288 }
289 }