This project has retired. For details please refer to its
Attic page.
001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.coord;
019
020 import java.util.Calendar;
021 import java.util.Date;
022 import java.util.HashMap;
023 import java.util.Iterator;
024 import java.util.List;
025 import java.util.Map;
026 import java.util.TimeZone;
027
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.oozie.service.ELService;
030 import org.apache.oozie.service.Services;
031 import org.apache.oozie.util.DateUtils;
032 import org.apache.oozie.util.ELEvaluator;
033 import org.apache.oozie.util.XmlUtils;
034 import org.jdom.Element;
035
036 /**
037 * This class provide different evaluators required at different stages
038 */
039 public class CoordELEvaluator {
040 public static final Integer MINUTE = 1;
041 public static final Integer HOUR = 60 * MINUTE;
042
043 /**
044 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
045 * 1)
046 *
047 * @param conf : Configuration containing property variables
048 * @return configured ELEvaluator
049 */
050 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
051 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
052 setConfigToEval(eval, conf);
053 return eval;
054 }
055
056 /**
057 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
058 *
059 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
060 * @param appInst : Application Instance related information such as Action creation Time
061 * @param conf :Configuration to substitute any variables
062 * @return configured ELEvaluator
063 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
064 */
065 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
066 throws Exception {
067 return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
068 }
069
070 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
071 Configuration conf) throws Exception {
072 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
073 setConfigToEval(eval, conf);
074 SyncCoordDataset ds = getDSObject(event);
075 CoordELFunctions.configureEvaluator(eval, ds, appInst);
076 return eval;
077 }
078
079 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
080 HashMap<String, String> dataNameList) throws Exception {
081 ELEvaluator eval = createELEvaluatorForGroup(conf, group);
082 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
083 String key = it.next();
084 String value = dataNameList.get(key);
085 eval.setVariable("oozie.dataname." + key, value);
086 }
087 return eval;
088 }
089
090 /**
091 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
092 * should be resolved when all other data dependencies are met.
093 *
094 * @param actualTime : Action start time
095 * @param nominalTime : Action creation time
096 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
097 * @param conf :Configuration to substitute any variables
098 * @return configured ELEvaluator
099 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
100 */
101 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
102 throws Exception {
103 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
104 setConfigToEval(eval, conf);
105 SyncCoordDataset ds = getDSObject(dEvent);
106 SyncCoordAction appInst = new SyncCoordAction();
107 appInst.setNominalTime(nominalTime);
108 appInst.setActualTime(actualTime);
109 CoordELFunctions.configureEvaluator(eval, ds, appInst);
110 eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
111 return eval;
112 }
113
114 /**
115 * Create a SLA evaluator to be used during Materialization
116 *
117 * @param nominalTime
118 * @param conf
119 * @return
120 * @throws Exception
121 */
122 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
123 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
124 setConfigToEval(eval, conf);
125 SyncCoordAction appInst = new SyncCoordAction();// TODO:
126 appInst.setNominalTime(nominalTime);
127 CoordELFunctions.configureEvaluator(eval, null, appInst);
128 return eval;
129 }
130
131 /**
132 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
133 *
134 * @param eJob : XML element for the application instance
135 * @param conf :Configuration to substitute any variables
136 * @return configured ELEvaluator
137 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
138 */
139 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
140 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
141 setConfigToEval(e, conf);
142 SyncCoordAction appInst = new SyncCoordAction();
143 String strNominalTime = eJob.getAttributeValue("action-nominal-time");
144 if (strNominalTime != null) {
145 appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime));
146 appInst.setActionId(actionId);
147 appInst.setName(eJob.getAttributeValue("name"));
148 }
149 String strActualTime = eJob.getAttributeValue("action-actual-time");
150 if (strActualTime != null) {
151 appInst.setActualTime(DateUtils.parseDateUTC(strActualTime));
152 }
153 CoordELFunctions.configureEvaluator(e, null, appInst);
154 Element events = eJob.getChild("input-events", eJob.getNamespace());
155 if (events != null) {
156 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
157 if (data.getChild("uris", data.getNamespace()) != null) {
158 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
159 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
160 e.setVariable(".datain." + data.getAttributeValue("name"), uris);
161 }
162 else {
163 }
164 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
165 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
166 // check
167 // null
168 }
169 }
170 }
171 events = eJob.getChild("output-events", eJob.getNamespace());
172 if (events != null) {
173 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
174 if (data.getChild("uris", data.getNamespace()) != null) {
175 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
176 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
177 e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
178 }
179 else {
180 }// TODO
181 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
182 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
183 // check
184 // null
185 }
186 }
187 }
188 return e;
189 }
190
191 /**
192 * Create a new Evaluator to resolve URI temple with time specific constant
193 *
194 * @param strDate : Date-time
195 * @return configured ELEvaluator
196 * @throws Exception If there is any date-time string in wrong format, the exception is thrown
197 */
198 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
199 ELEvaluator eval = new ELEvaluator();
200 Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC
201 // always???
202 date.setTime(DateUtils.parseDateUTC(strDate));
203 eval.setVariable("YEAR", date.get(Calendar.YEAR));
204 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
205 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
206 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
207 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
208 return eval;
209 }
210
211 /**
212 * Create Dataset object using the Dataset XML information
213 *
214 * @param eData
215 * @return
216 * @throws Exception
217 */
218 private static SyncCoordDataset getDSObject(Element eData) throws Exception {
219 SyncCoordDataset ds = new SyncCoordDataset();
220 Element eDataset = eData.getChild("dataset", eData.getNamespace());
221 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
222 Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance"));
223 ds.setInitInstance(initInstance);
224 if (eDataset.getAttributeValue("frequency") != null) {
225 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
226 ds.setFrequency(frequency);
227 ds.setType("SYNC");
228 if (eDataset.getAttributeValue("freq_timeunit") == null) {
229 throw new RuntimeException("No freq_timeunit defined in data set definition\n"
230 + XmlUtils.prettyPrint(eDataset));
231 }
232 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
233 if (eDataset.getAttributeValue("timezone") == null) {
234 throw new RuntimeException("No timezone defined in data set definition\n"
235 + XmlUtils.prettyPrint(eDataset));
236 }
237 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
238 if (eDataset.getAttributeValue("end_of_duration") == null) {
239 throw new RuntimeException("No end_of_duration defined in data set definition\n"
240 + XmlUtils.prettyPrint(eDataset));
241 }
242 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
243
244 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
245 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
246 ds.setDoneFlag(doneFlag);
247 }
248 else {
249 ds.setType("ASYNC");
250 }
251 String name = eDataset.getAttributeValue("name");
252 ds.setName(name);
253 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
254 // eData.getNamespace()));
255 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
256 ds.setUriTemplate(uriTemplate);
257 // ds.setTimeUnit(TimeUnit.MINUTES);
258 return ds;
259 }
260
261 /**
262 * Set all job configurations properties into evaluator.
263 *
264 * @param eval : Evaluator to set variables
265 * @param conf : configurations to set Evaluator
266 */
267 private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
268 for (Map.Entry<String, String> entry : conf) {
269 eval.setVariable(entry.getKey(), entry.getValue().trim());
270 }
271 }
272
273 /**
274 * make any one digit number to two digit string pre-appending a"0"
275 *
276 * @param num : number to make sting
277 * @return :String of length at least two digit.
278 */
279 private static String make2Digits(int num) {
280 String ret = "" + num;
281 if (num <= 9) {
282 ret = "0" + ret;
283 }
284 return ret;
285 }
286 }