001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.coord;
019    
020    import java.util.Calendar;
021    import java.util.Date;
022    import java.util.HashMap;
023    import java.util.Iterator;
024    import java.util.List;
025    import java.util.Map;
026    import java.util.TimeZone;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.oozie.service.ELService;
030    import org.apache.oozie.service.Services;
031    import org.apache.oozie.util.DateUtils;
032    import org.apache.oozie.util.ELEvaluator;
033    import org.apache.oozie.util.XmlUtils;
034    import org.jdom.Element;
035    
036    /**
037     * This class provide different evaluators required at different stages
038     */
039    public class CoordELEvaluator {
040        public static final Integer MINUTE = 1;
041        public static final Integer HOUR = 60 * MINUTE;
042    
043        /**
044         * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
045         * 1)
046         *
047         * @param conf : Configuration containing property variables
048         * @return configured ELEvaluator
049         */
050        public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
051            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
052            setConfigToEval(eval, conf);
053            return eval;
054        }
055    
056        /**
057         * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
058         *
059         * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
060         * @param appInst : Application Instance related information such as Action creation Time
061         * @param conf :Configuration to substitute any variables
062         * @return configured ELEvaluator
063         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
064         */
065        public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
066                throws Exception {
067            return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
068        }
069    
070        public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
071                                                             Configuration conf) throws Exception {
072            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
073            setConfigToEval(eval, conf);
074            SyncCoordDataset ds = getDSObject(event);
075            CoordELFunctions.configureEvaluator(eval, ds, appInst);
076            return eval;
077        }
078    
079        public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
080                                                               HashMap<String, String> dataNameList) throws Exception {
081            ELEvaluator eval = createELEvaluatorForGroup(conf, group);
082            for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
083                String key = it.next();
084                String value = dataNameList.get(key);
085                eval.setVariable("oozie.dataname." + key, value);
086            }
087            return eval;
088        }
089    
090        /**
091         * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
092         * should be resolved when all other data dependencies are met.
093         *
094         * @param actualTime : Action start time
095         * @param nominalTime : Action creation time
096         * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
097         * @param conf :Configuration to substitute any variables
098         * @return configured ELEvaluator
099         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
100         */
101        public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
102                throws Exception {
103            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
104            setConfigToEval(eval, conf);
105            SyncCoordDataset ds = getDSObject(dEvent);
106            SyncCoordAction appInst = new SyncCoordAction();
107            appInst.setNominalTime(nominalTime);
108            appInst.setActualTime(actualTime);
109            CoordELFunctions.configureEvaluator(eval, ds, appInst);
110            eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
111            return eval;
112        }
113    
114        /**
115         * Create a SLA evaluator to be used during Materialization
116         *
117         * @param nominalTime
118         * @param conf
119         * @return
120         * @throws Exception
121         */
122        public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
123            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
124            setConfigToEval(eval, conf);
125            SyncCoordAction appInst = new SyncCoordAction();// TODO:
126            appInst.setNominalTime(nominalTime);
127            CoordELFunctions.configureEvaluator(eval, null, appInst);
128            return eval;
129        }
130    
131        /**
132         * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
133         *
134         * @param eJob : XML element for the application instance
135         * @param conf :Configuration to substitute any variables
136         * @return configured ELEvaluator
137         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
138         */
139        public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
140            ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
141            setConfigToEval(e, conf);
142            SyncCoordAction appInst = new SyncCoordAction();
143            String strNominalTime = eJob.getAttributeValue("action-nominal-time");
144            if (strNominalTime != null) {
145                appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime));
146                appInst.setActionId(actionId);
147                appInst.setName(eJob.getAttributeValue("name"));
148            }
149            String strActualTime = eJob.getAttributeValue("action-actual-time");
150            if (strActualTime != null) {
151                appInst.setActualTime(DateUtils.parseDateUTC(strActualTime));
152            }
153            CoordELFunctions.configureEvaluator(e, null, appInst);
154            Element events = eJob.getChild("input-events", eJob.getNamespace());
155            if (events != null) {
156                for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
157                    if (data.getChild("uris", data.getNamespace()) != null) {
158                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
159                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
160                        e.setVariable(".datain." + data.getAttributeValue("name"), uris);
161                    }
162                    else {
163                    }
164                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
165                        e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
166                        // check
167                        // null
168                    }
169                }
170            }
171            events = eJob.getChild("output-events", eJob.getNamespace());
172            if (events != null) {
173                for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
174                    if (data.getChild("uris", data.getNamespace()) != null) {
175                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
176                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
177                        e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
178                    }
179                    else {
180                    }// TODO
181                    if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
182                        e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
183                        // check
184                        // null
185                    }
186                }
187            }
188            return e;
189        }
190    
191        /**
192         * Create a new Evaluator to resolve URI temple with time specific constant
193         *
194         * @param strDate : Date-time
195         * @return configured ELEvaluator
196         * @throws Exception If there is any date-time string in wrong format, the exception is thrown
197         */
198        public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
199            ELEvaluator eval = new ELEvaluator();
200            Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC
201            // always???
202            date.setTime(DateUtils.parseDateUTC(strDate));
203            eval.setVariable("YEAR", date.get(Calendar.YEAR));
204            eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
205            eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
206            eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
207            eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
208            return eval;
209        }
210    
211        /**
212         * Create Dataset object using the Dataset XML information
213         *
214         * @param eData
215         * @return
216         * @throws Exception
217         */
218        private static SyncCoordDataset getDSObject(Element eData) throws Exception {
219            SyncCoordDataset ds = new SyncCoordDataset();
220            Element eDataset = eData.getChild("dataset", eData.getNamespace());
221            // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
222            Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance"));
223            ds.setInitInstance(initInstance);
224            if (eDataset.getAttributeValue("frequency") != null) {
225                int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
226                ds.setFrequency(frequency);
227                ds.setType("SYNC");
228                if (eDataset.getAttributeValue("freq_timeunit") == null) {
229                    throw new RuntimeException("No freq_timeunit defined in data set definition\n"
230                            + XmlUtils.prettyPrint(eDataset));
231                }
232                ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
233                if (eDataset.getAttributeValue("timezone") == null) {
234                    throw new RuntimeException("No timezone defined in data set definition\n"
235                            + XmlUtils.prettyPrint(eDataset));
236                }
237                ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
238                if (eDataset.getAttributeValue("end_of_duration") == null) {
239                    throw new RuntimeException("No end_of_duration defined in data set definition\n"
240                            + XmlUtils.prettyPrint(eDataset));
241                }
242                ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
243    
244                Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
245                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
246                ds.setDoneFlag(doneFlag);
247            }
248            else {
249                ds.setType("ASYNC");
250            }
251            String name = eDataset.getAttributeValue("name");
252            ds.setName(name);
253            // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
254            // eData.getNamespace()));
255            String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
256            ds.setUriTemplate(uriTemplate);
257            // ds.setTimeUnit(TimeUnit.MINUTES);
258            return ds;
259        }
260    
261        /**
262         * Set all job configurations properties into evaluator.
263         *
264         * @param eval : Evaluator to set variables
265         * @param conf : configurations to set Evaluator
266         */
267        private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
268            for (Map.Entry<String, String> entry : conf) {
269                eval.setVariable(entry.getKey(), entry.getValue().trim());
270            }
271        }
272    
273        /**
274         * make any one digit number to two digit string pre-appending a"0"
275         *
276         * @param num : number to make sting
277         * @return :String of length at least two digit.
278         */
279        private static String make2Digits(int num) {
280            String ret = "" + num;
281            if (num <= 9) {
282                ret = "0" + ret;
283            }
284            return ret;
285        }
286    }