Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.coord;
020
021import java.util.Calendar;
022import java.util.Date;
023import java.util.HashMap;
024import java.util.Iterator;
025import java.util.List;
026import java.util.Map;
027
028import org.apache.hadoop.conf.Configuration;
029import org.apache.oozie.CoordinatorActionBean;
030import org.apache.oozie.command.coord.CoordCommandUtils;
031import org.apache.oozie.coord.input.dependency.CoordInputDependency;
032import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluator;
033import org.apache.oozie.service.ELService;
034import org.apache.oozie.service.Services;
035import org.apache.oozie.util.DateUtils;
036import org.apache.oozie.util.ELEvaluator;
037import org.apache.oozie.util.XmlUtils;
038import org.jdom.Element;
039
040/**
041 * This class provide different evaluators required at different stages
042 */
043public class CoordELEvaluator {
044    public static final Integer MINUTE = 1;
045    public static final Integer HOUR = 60 * MINUTE;
046
047    /**
048     * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
049     * 1)
050     *
051     * @param conf : Configuration containing property variables
052     * @param group  Name of the group of required EL Evaluator.
053     * @return configured ELEvaluator
054     */
055    public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
056        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
057        setConfigToEval(eval, conf);
058        return eval;
059    }
060
061    /**
062     * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
063     *
064     * @param event : Xml element for data-in element usually enclosed by &lt;data-in(out)&gt; tag
065     * @param appInst : Application Instance related information such as Action creation Time
066     * @param conf :Configuration to substitute any variables
067     * @return configured ELEvaluator
068     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
069     */
070    public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
071            throws Exception {
072        return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
073    }
074
075    public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
076                                                         Configuration conf) throws Exception {
077        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
078        setConfigToEval(eval, conf);
079        SyncCoordDataset ds = getDSObject(event);
080        CoordELFunctions.configureEvaluator(eval, ds, appInst);
081        return eval;
082    }
083
084    public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
085                                                           HashMap<String, String> dataNameList) throws Exception {
086        ELEvaluator eval = createELEvaluatorForGroup(conf, group);
087        for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
088            String key = it.next();
089            String value = dataNameList.get(key);
090            eval.setVariable("oozie.dataname." + key, value);
091        }
092        return eval;
093    }
094
095    /**
096     * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
097     * should be resolved when all other data dependencies are met.
098     *
099     * @param actualTime : Action start time
100     * @param nominalTime : Action creation time
101     * @param dEvent :XML element for data-in element usually enclosed by &lt;data-in(out)&gt; tag
102     * @param conf :Configuration to substitute any variables
103     * @return configured ELEvaluator
104     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
105     */
106    public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
107            throws Exception {
108        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
109        setConfigToEval(eval, conf);
110        SyncCoordDataset ds = getDSObject(dEvent);
111        SyncCoordAction appInst = new SyncCoordAction();
112        appInst.setNominalTime(nominalTime);
113        appInst.setActualTime(actualTime);
114        CoordELFunctions.configureEvaluator(eval, ds, appInst);
115        eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
116        return eval;
117    }
118
119    /**
120     * Create a SLA evaluator to be used during Materialization
121     * @param eAction the action
122     * @param coordAction the coordinator action
123     * @param conf the configuration
124     * @return eval returns SLA evaluator to be used during Materialization
125     * @throws Exception in case of error
126     */
127    public static ELEvaluator createSLAEvaluator(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
128            throws Exception {
129        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
130        setConfigToEval(eval, conf);
131        SyncCoordAction appInst = new SyncCoordAction();// TODO:
132        appInst.setNominalTime(coordAction.getNominalTime());
133        appInst.setActualTime(coordAction.getCreatedTime());
134        appInst.setActionId(coordAction.getId());
135        appInst.setName(eAction.getAttributeValue("name"));
136        CoordELFunctions.configureEvaluator(eval, null, appInst);
137
138        Element events = eAction.getChild("output-events", eAction.getNamespace());
139        if (events != null) {
140            for (Object obj : events.getChildren("data-out", eAction.getNamespace())) {
141                Element data = (Element) obj;
142                if (data.getChild("uris", data.getNamespace()) != null) {
143                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
144                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
145                    eval.setVariable(".dataout." + data.getAttributeValue("name"), uris);
146                }
147                if (data.getChild(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, data.getNamespace()) != null) {
148                    eval.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true");
149                }
150            }
151        }
152        return eval;
153    }
154
155    /**
156     * Create an Evaluator using conf and input/output-data (used for sla)
157     * @param conf the configuration
158     * @param group the group for the EL expression
159     * @param dataNameList the name list for the data
160     * @return eval returns an Evaluator using conf and input/output-data (used for sla)
161     * @throws Exception in case of error
162     */
163    public static ELEvaluator createELEvaluatorForDataAndConf(Configuration conf, String group,
164            HashMap<String, String> dataNameList) throws Exception {
165        ELEvaluator eval = createELEvaluatorForDataEcho(conf, group, dataNameList);
166        setConfigToEval(eval, conf);
167        return eval;
168    }
169
170    /**
171     * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
172     *
173     * @param eJob : XML element for the application instance
174     * @param conf :Configuration to substitute any variables
175     * @param actionId the action Id
176     * @return configured ELEvaluator
177     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
178     */
179
180    public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
181        return createDataEvaluator(eJob, conf, actionId, null, null);
182    }
183
184    public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId,
185            CoordInputDependency pullDependencies, CoordInputDependency pushDependencies) throws Exception {
186        ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
187        setConfigToEval(e, conf);
188        SyncCoordAction appInst = new SyncCoordAction();
189        String strNominalTime = eJob.getAttributeValue("action-nominal-time");
190        if (strNominalTime != null) {
191            appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
192            appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
193            appInst.setFrequency(eJob.getAttributeValue("frequency"));
194            appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
195            appInst.setActionId(actionId);
196            appInst.setName(eJob.getAttributeValue("name"));
197            appInst.setPullDependencies(pullDependencies);
198            appInst.setPushDependencies(pushDependencies);
199            if (CoordUtils.isInputLogicSpecified(eJob)) {
200                e.setVariable(".actionInputLogic",
201                        XmlUtils.prettyPrint(eJob.getChild(CoordInputLogicEvaluator.INPUT_LOGIC, eJob.getNamespace())).toString());
202            }
203        }
204        String strActualTime = eJob.getAttributeValue("action-actual-time");
205        if (strActualTime != null) {
206            appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
207        }
208        CoordELFunctions.configureEvaluator(e, null, appInst);
209        Element events = eJob.getChild("input-events", eJob.getNamespace());
210        if (events != null) {
211            for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
212                if (data.getChild("uris", data.getNamespace()) != null) {
213                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
214                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
215                    e.setVariable(".datain." + data.getAttributeValue("name"), uris);
216                }
217                else {
218                }
219                if (data.getChild(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, data.getNamespace()) != null) {
220                    e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
221                    // check
222                    // null
223                }
224                Element doneFlagElement = data.getChild("dataset", data.getNamespace()).getChild("done-flag",
225                        data.getNamespace());
226                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
227                e.setVariable(".datain." + data.getAttributeValue("name") + ".doneFlag", doneFlag);
228            }
229        }
230        events = eJob.getChild("output-events", eJob.getNamespace());
231        if (events != null) {
232            for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
233                if (data.getChild("uris", data.getNamespace()) != null) {
234                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
235                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
236                    e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
237                }
238                else {
239                }// TODO
240                if (data.getChild(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, data.getNamespace()) != null) {
241                    e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
242                    // check
243                    // null
244                }
245            }
246        }
247        return e;
248    }
249
250    /**
251     * Create a new Evaluator to resolve URI temple with time specific constant
252     *
253     * @param strDate : Date-time
254     * @return configured ELEvaluator
255     * @throws Exception If there is any date-time string in wrong format, the exception is thrown
256     */
257    public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
258        ELEvaluator eval = new ELEvaluator();
259        Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
260        // always???
261        date.setTime(DateUtils.parseDateOozieTZ(strDate));
262        eval.setVariable("YEAR", date.get(Calendar.YEAR));
263        eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
264        eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
265        eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
266        eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
267        return eval;
268    }
269
270    /**
271     * Create Dataset object using the Dataset XML information
272     *
273     * @param eData the xml
274     * @return ds returns Dataset object using the Dataset XML information
275     * @throws Exception if the Dataset object can't be created
276     */
277    private static SyncCoordDataset getDSObject(Element eData) throws Exception {
278        SyncCoordDataset ds = new SyncCoordDataset();
279        Element eDataset = eData.getChild("dataset", eData.getNamespace());
280        // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
281        Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
282        ds.setInitInstance(initInstance);
283        if (eDataset.getAttributeValue("frequency") != null) {
284            int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
285            ds.setFrequency(frequency);
286            ds.setType("SYNC");
287            if (eDataset.getAttributeValue("freq_timeunit") == null) {
288                throw new RuntimeException("No freq_timeunit defined in data set definition\n"
289                        + XmlUtils.prettyPrint(eDataset));
290            }
291            ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
292            if (eDataset.getAttributeValue("timezone") == null) {
293                throw new RuntimeException("No timezone defined in data set definition\n"
294                        + XmlUtils.prettyPrint(eDataset));
295            }
296            ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
297            if (eDataset.getAttributeValue("end_of_duration") == null) {
298                throw new RuntimeException("No end_of_duration defined in data set definition\n"
299                        + XmlUtils.prettyPrint(eDataset));
300            }
301            ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
302
303            Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
304            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
305            ds.setDoneFlag(doneFlag);
306        }
307        else {
308            ds.setType("ASYNC");
309        }
310        String name = eDataset.getAttributeValue("name");
311        ds.setName(name);
312        // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
313        // eData.getNamespace()));
314        String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
315        ds.setUriTemplate(uriTemplate);
316        // ds.setTimeUnit(TimeUnit.MINUTES);
317        return ds;
318    }
319
320    /**
321     * Set all job configurations properties into evaluator.
322     *
323     * @param eval : Evaluator to set variables
324     * @param conf : configurations to set Evaluator
325     */
326    private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
327        for (Map.Entry<String, String> entry : conf) {
328            eval.setVariable(entry.getKey(), entry.getValue().trim());
329        }
330    }
331
332    /**
333     * make any one digit number to two digit string pre-appending a"0"
334     *
335     * @param num : number to make sting
336     * @return :String of length at least two digit.
337     */
338    private static String make2Digits(int num) {
339        String ret = "" + num;
340        if (num <= 9) {
341            ret = "0" + ret;
342        }
343        return ret;
344    }
345}