001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.coord;
019    
020    import java.util.Calendar;
021    import java.util.Date;
022    import java.util.HashMap;
023    import java.util.Iterator;
024    import java.util.List;
025    import java.util.Map;
026    
027    import org.apache.hadoop.conf.Configuration;
028    import org.apache.oozie.service.ELService;
029    import org.apache.oozie.service.Services;
030    import org.apache.oozie.util.DateUtils;
031    import org.apache.oozie.util.ELEvaluator;
032    import org.apache.oozie.util.XmlUtils;
033    import org.jdom.Element;
034    
035    /**
036     * This class provide different evaluators required at different stages
037     */
038    public class CoordELEvaluator {
039        public static final Integer MINUTE = 1;
040        public static final Integer HOUR = 60 * MINUTE;
041    
042        /**
043         * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
044         * 1)
045         *
046         * @param conf : Configuration containing property variables
047         * @return configured ELEvaluator
048         */
049        public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
050            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
051            setConfigToEval(eval, conf);
052            return eval;
053        }
054    
055        /**
056         * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
057         *
058         * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
059         * @param appInst : Application Instance related information such as Action creation Time
060         * @param conf :Configuration to substitute any variables
061         * @return configured ELEvaluator
062         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
063         */
064        public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
065                throws Exception {
066            return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
067        }
068    
069        public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
070                                                             Configuration conf) throws Exception {
071            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
072            setConfigToEval(eval, conf);
073            SyncCoordDataset ds = getDSObject(event);
074            CoordELFunctions.configureEvaluator(eval, ds, appInst);
075            return eval;
076        }
077    
078        public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
079                                                               HashMap<String, String> dataNameList) throws Exception {
080            ELEvaluator eval = createELEvaluatorForGroup(conf, group);
081            for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
082                String key = it.next();
083                String value = dataNameList.get(key);
084                eval.setVariable("oozie.dataname." + key, value);
085            }
086            return eval;
087        }
088    
089        /**
090         * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
091         * should be resolved when all other data dependencies are met.
092         *
093         * @param actualTime : Action start time
094         * @param nominalTime : Action creation time
095         * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
096         * @param conf :Configuration to substitute any variables
097         * @return configured ELEvaluator
098         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
099         */
100        public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
101                throws Exception {
102            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
103            setConfigToEval(eval, conf);
104            SyncCoordDataset ds = getDSObject(dEvent);
105            SyncCoordAction appInst = new SyncCoordAction();
106            appInst.setNominalTime(nominalTime);
107            appInst.setActualTime(actualTime);
108            CoordELFunctions.configureEvaluator(eval, ds, appInst);
109            eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
110            return eval;
111        }
112    
113        /**
114         * Create a SLA evaluator to be used during Materialization
115         *
116         * @param nominalTime
117         * @param conf
118         * @return
119         * @throws Exception
120         */
121        public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
122            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
123            setConfigToEval(eval, conf);
124            SyncCoordAction appInst = new SyncCoordAction();// TODO:
125            appInst.setNominalTime(nominalTime);
126            CoordELFunctions.configureEvaluator(eval, null, appInst);
127            return eval;
128        }
129    
130        /**
131         * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
132         *
133         * @param eJob : XML element for the application instance
134         * @param conf :Configuration to substitute any variables
135         * @return configured ELEvaluator
136         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
137         */
138        public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
139            ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
140            setConfigToEval(e, conf);
141            SyncCoordAction appInst = new SyncCoordAction();
142            String strNominalTime = eJob.getAttributeValue("action-nominal-time");
143            if (strNominalTime != null) {
144                appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
145                appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
146                appInst.setFrequency(Integer.parseInt(eJob.getAttributeValue("frequency")));
147                appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
148                appInst.setActionId(actionId);
149                appInst.setName(eJob.getAttributeValue("name"));
150            }
151            String strActualTime = eJob.getAttributeValue("action-actual-time");
152            if (strActualTime != null) {
153                appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
154            }
155            CoordELFunctions.configureEvaluator(e, null, appInst);
156            Element events = eJob.getChild("input-events", eJob.getNamespace());
157            if (events != null) {
158                for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
159                    if (data.getChild("uris", data.getNamespace()) != null) {
160                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
161                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
162                        e.setVariable(".datain." + data.getAttributeValue("name"), uris);
163                    }
164                    else {
165                    }
166                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
167                        e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
168                        // check
169                        // null
170                    }
171                }
172            }
173            events = eJob.getChild("output-events", eJob.getNamespace());
174            if (events != null) {
175                for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
176                    if (data.getChild("uris", data.getNamespace()) != null) {
177                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
178                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
179                        e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
180                    }
181                    else {
182                    }// TODO
183                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
184                        e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
185                        // check
186                        // null
187                    }
188                }
189            }
190            return e;
191        }
192    
193        /**
194         * Create a new Evaluator to resolve URI temple with time specific constant
195         *
196         * @param strDate : Date-time
197         * @return configured ELEvaluator
198         * @throws Exception If there is any date-time string in wrong format, the exception is thrown
199         */
200        public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
201            ELEvaluator eval = new ELEvaluator();
202            Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
203            // always???
204            date.setTime(DateUtils.parseDateOozieTZ(strDate));
205            eval.setVariable("YEAR", date.get(Calendar.YEAR));
206            eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
207            eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
208            eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
209            eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
210            return eval;
211        }
212    
213        /**
214         * Create Dataset object using the Dataset XML information
215         *
216         * @param eData
217         * @return
218         * @throws Exception
219         */
220        private static SyncCoordDataset getDSObject(Element eData) throws Exception {
221            SyncCoordDataset ds = new SyncCoordDataset();
222            Element eDataset = eData.getChild("dataset", eData.getNamespace());
223            // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
224            Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
225            ds.setInitInstance(initInstance);
226            if (eDataset.getAttributeValue("frequency") != null) {
227                int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
228                ds.setFrequency(frequency);
229                ds.setType("SYNC");
230                if (eDataset.getAttributeValue("freq_timeunit") == null) {
231                    throw new RuntimeException("No freq_timeunit defined in data set definition\n"
232                            + XmlUtils.prettyPrint(eDataset));
233                }
234                ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
235                if (eDataset.getAttributeValue("timezone") == null) {
236                    throw new RuntimeException("No timezone defined in data set definition\n"
237                            + XmlUtils.prettyPrint(eDataset));
238                }
239                ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
240                if (eDataset.getAttributeValue("end_of_duration") == null) {
241                    throw new RuntimeException("No end_of_duration defined in data set definition\n"
242                            + XmlUtils.prettyPrint(eDataset));
243                }
244                ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
245    
246                Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
247                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
248                ds.setDoneFlag(doneFlag);
249            }
250            else {
251                ds.setType("ASYNC");
252            }
253            String name = eDataset.getAttributeValue("name");
254            ds.setName(name);
255            // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
256            // eData.getNamespace()));
257            String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
258            ds.setUriTemplate(uriTemplate);
259            // ds.setTimeUnit(TimeUnit.MINUTES);
260            return ds;
261        }
262    
263        /**
264         * Set all job configurations properties into evaluator.
265         *
266         * @param eval : Evaluator to set variables
267         * @param conf : configurations to set Evaluator
268         */
269        private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
270            for (Map.Entry<String, String> entry : conf) {
271                eval.setVariable(entry.getKey(), entry.getValue().trim());
272            }
273        }
274    
275        /**
276         * make any one digit number to two digit string pre-appending a"0"
277         *
278         * @param num : number to make sting
279         * @return :String of length at least two digit.
280         */
281        private static String make2Digits(int num) {
282            String ret = "" + num;
283            if (num <= 9) {
284                ret = "0" + ret;
285            }
286            return ret;
287        }
288    }