Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.coord;
019
020import java.util.Calendar;
021import java.util.Date;
022import java.util.HashMap;
023import java.util.Iterator;
024import java.util.List;
025import java.util.Map;
026
027import org.apache.hadoop.conf.Configuration;
028import org.apache.oozie.CoordinatorActionBean;
029import org.apache.oozie.command.coord.CoordCommandUtils;
030import org.apache.oozie.service.ELService;
031import org.apache.oozie.service.Services;
032import org.apache.oozie.util.DateUtils;
033import org.apache.oozie.util.ELEvaluator;
034import org.apache.oozie.util.XmlUtils;
035import org.jdom.Element;
036
037/**
038 * This class provide different evaluators required at different stages
039 */
040public class CoordELEvaluator {
041    public static final Integer MINUTE = 1;
042    public static final Integer HOUR = 60 * MINUTE;
043
044    /**
045     * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
046     * 1)
047     *
048     * @param conf : Configuration containing property variables
049     * @return configured ELEvaluator
050     */
051    public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
052        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
053        setConfigToEval(eval, conf);
054        return eval;
055    }
056
057    /**
058     * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
059     *
060     * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
061     * @param appInst : Application Instance related information such as Action creation Time
062     * @param conf :Configuration to substitute any variables
063     * @return configured ELEvaluator
064     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
065     */
066    public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
067            throws Exception {
068        return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
069    }
070
071    public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
072                                                         Configuration conf) throws Exception {
073        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
074        setConfigToEval(eval, conf);
075        SyncCoordDataset ds = getDSObject(event);
076        CoordELFunctions.configureEvaluator(eval, ds, appInst);
077        return eval;
078    }
079
080    public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
081                                                           HashMap<String, String> dataNameList) throws Exception {
082        ELEvaluator eval = createELEvaluatorForGroup(conf, group);
083        for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
084            String key = it.next();
085            String value = dataNameList.get(key);
086            eval.setVariable("oozie.dataname." + key, value);
087        }
088        return eval;
089    }
090
091    /**
092     * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
093     * should be resolved when all other data dependencies are met.
094     *
095     * @param actualTime : Action start time
096     * @param nominalTime : Action creation time
097     * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
098     * @param conf :Configuration to substitute any variables
099     * @return configured ELEvaluator
100     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
101     */
102    public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
103            throws Exception {
104        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
105        setConfigToEval(eval, conf);
106        SyncCoordDataset ds = getDSObject(dEvent);
107        SyncCoordAction appInst = new SyncCoordAction();
108        appInst.setNominalTime(nominalTime);
109        appInst.setActualTime(actualTime);
110        CoordELFunctions.configureEvaluator(eval, ds, appInst);
111        eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
112        return eval;
113    }
114
115    /**
116     * Create a SLA evaluator to be used during Materialization
117     * @param eAction
118     * @param coordAction
119     * @param conf
120     * @return
121     * @throws Exception
122     */
123    public static ELEvaluator createSLAEvaluator(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
124            throws Exception {
125        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
126        setConfigToEval(eval, conf);
127        SyncCoordAction appInst = new SyncCoordAction();// TODO:
128        appInst.setNominalTime(coordAction.getNominalTime());
129        appInst.setActualTime(coordAction.getCreatedTime());
130        appInst.setActionId(coordAction.getId());
131        appInst.setName(eAction.getAttributeValue("name"));
132        CoordELFunctions.configureEvaluator(eval, null, appInst);
133
134        Element events = eAction.getChild("output-events", eAction.getNamespace());
135        if (events != null) {
136            for (Object obj : events.getChildren("data-out", eAction.getNamespace())) {
137                Element data = (Element) obj;
138                if (data.getChild("uris", data.getNamespace()) != null) {
139                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
140                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
141                    eval.setVariable(".dataout." + data.getAttributeValue("name"), uris);
142                }
143                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
144                    eval.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true");
145                }
146            }
147        }
148        return eval;
149    }
150
151    /**
152     * Create an Evaluator using conf and input/output-data (used for sla)
153     * @param conf
154     * @param group
155     * @param dataNameList
156     * @return
157     * @throws Exception
158     */
159    public static ELEvaluator createELEvaluatorForDataAndConf(Configuration conf, String group,
160            HashMap<String, String> dataNameList) throws Exception {
161        ELEvaluator eval = createELEvaluatorForDataEcho(conf, group, dataNameList);
162        setConfigToEval(eval, conf);
163        return eval;
164    }
165
166    /**
167     * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
168     *
169     * @param eJob : XML element for the application instance
170     * @param conf :Configuration to substitute any variables
171     * @return configured ELEvaluator
172     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
173     */
174    public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
175        ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
176        setConfigToEval(e, conf);
177        SyncCoordAction appInst = new SyncCoordAction();
178        String strNominalTime = eJob.getAttributeValue("action-nominal-time");
179        if (strNominalTime != null) {
180            appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
181            appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
182            appInst.setFrequency(eJob.getAttributeValue("frequency"));
183            appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
184            appInst.setActionId(actionId);
185            appInst.setName(eJob.getAttributeValue("name"));
186        }
187        String strActualTime = eJob.getAttributeValue("action-actual-time");
188        if (strActualTime != null) {
189            appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
190        }
191        CoordELFunctions.configureEvaluator(e, null, appInst);
192        Element events = eJob.getChild("input-events", eJob.getNamespace());
193        if (events != null) {
194            for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
195                if (data.getChild("uris", data.getNamespace()) != null) {
196                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
197                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
198                    e.setVariable(".datain." + data.getAttributeValue("name"), uris);
199                }
200                else {
201                }
202                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
203                    e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
204                    // check
205                    // null
206                }
207            }
208        }
209        events = eJob.getChild("output-events", eJob.getNamespace());
210        if (events != null) {
211            for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
212                if (data.getChild("uris", data.getNamespace()) != null) {
213                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
214                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
215                    e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
216                }
217                else {
218                }// TODO
219                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
220                    e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
221                    // check
222                    // null
223                }
224            }
225        }
226        return e;
227    }
228
229    /**
230     * Create a new Evaluator to resolve URI temple with time specific constant
231     *
232     * @param strDate : Date-time
233     * @return configured ELEvaluator
234     * @throws Exception If there is any date-time string in wrong format, the exception is thrown
235     */
236    public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
237        ELEvaluator eval = new ELEvaluator();
238        Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
239        // always???
240        date.setTime(DateUtils.parseDateOozieTZ(strDate));
241        eval.setVariable("YEAR", date.get(Calendar.YEAR));
242        eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
243        eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
244        eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
245        eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
246        return eval;
247    }
248
249    /**
250     * Create Dataset object using the Dataset XML information
251     *
252     * @param eData
253     * @return
254     * @throws Exception
255     */
256    private static SyncCoordDataset getDSObject(Element eData) throws Exception {
257        SyncCoordDataset ds = new SyncCoordDataset();
258        Element eDataset = eData.getChild("dataset", eData.getNamespace());
259        // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
260        Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
261        ds.setInitInstance(initInstance);
262        if (eDataset.getAttributeValue("frequency") != null) {
263            int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
264            ds.setFrequency(frequency);
265            ds.setType("SYNC");
266            if (eDataset.getAttributeValue("freq_timeunit") == null) {
267                throw new RuntimeException("No freq_timeunit defined in data set definition\n"
268                        + XmlUtils.prettyPrint(eDataset));
269            }
270            ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
271            if (eDataset.getAttributeValue("timezone") == null) {
272                throw new RuntimeException("No timezone defined in data set definition\n"
273                        + XmlUtils.prettyPrint(eDataset));
274            }
275            ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
276            if (eDataset.getAttributeValue("end_of_duration") == null) {
277                throw new RuntimeException("No end_of_duration defined in data set definition\n"
278                        + XmlUtils.prettyPrint(eDataset));
279            }
280            ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
281
282            Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
283            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
284            ds.setDoneFlag(doneFlag);
285        }
286        else {
287            ds.setType("ASYNC");
288        }
289        String name = eDataset.getAttributeValue("name");
290        ds.setName(name);
291        // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
292        // eData.getNamespace()));
293        String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
294        ds.setUriTemplate(uriTemplate);
295        // ds.setTimeUnit(TimeUnit.MINUTES);
296        return ds;
297    }
298
299    /**
300     * Set all job configurations properties into evaluator.
301     *
302     * @param eval : Evaluator to set variables
303     * @param conf : configurations to set Evaluator
304     */
305    private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
306        for (Map.Entry<String, String> entry : conf) {
307            eval.setVariable(entry.getKey(), entry.getValue().trim());
308        }
309    }
310
311    /**
312     * make any one digit number to two digit string pre-appending a"0"
313     *
314     * @param num : number to make sting
315     * @return :String of length at least two digit.
316     */
317    private static String make2Digits(int num) {
318        String ret = "" + num;
319        if (num <= 9) {
320            ret = "0" + ret;
321        }
322        return ret;
323    }
324}