Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.oozie.service.ConfigurationService;
023import org.apache.oozie.service.Services;
024import org.w3c.dom.DOMException;
025import org.w3c.dom.Document;
026import org.w3c.dom.Element;
027import org.w3c.dom.Node;
028import org.w3c.dom.NodeList;
029import org.w3c.dom.Text;
030import org.xml.sax.SAXException;
031import org.xml.sax.InputSource;
032
033import javax.xml.parsers.DocumentBuilder;
034import javax.xml.parsers.DocumentBuilderFactory;
035import javax.xml.parsers.ParserConfigurationException;
036import java.io.IOException;
037import java.io.InputStream;
038import java.io.Reader;
039import java.io.ByteArrayOutputStream;
040import java.util.Map;
041import java.util.Properties;
042import java.util.regex.Matcher;
043import java.util.regex.Pattern;
044
045/**
046 * Extends Hadoop Configuration providing a new constructor which reads an XML configuration from an InputStream. <p/>
047 * OConfiguration(InputStream is).
048 */
049public class XConfiguration extends Configuration {
050
051    public static final String CONFIGURATION_SUBSTITUTE_DEPTH = "oozie.configuration.substitute.depth";
052
053    /**
054     * Create an empty configuration. <p/> Default values are not loaded.
055     */
056    public XConfiguration() {
057        super(false);
058        initSubstituteDepth();
059    }
060
061    /**
062     * Create a configuration from an InputStream. <p/> Code canibalized from <code>Configuration.loadResource()</code>.
063     *
064     * @param is inputstream to read the configuration from.
065     * @throws IOException thrown if the configuration could not be read.
066     */
067    public XConfiguration(InputStream is) throws IOException {
068        this();
069        parse(is);
070    }
071
072    /**
073     * Create a configuration from an Reader. <p/> Code canibalized from <code>Configuration.loadResource()</code>.
074     *
075     * @param reader reader to read the configuration from.
076     * @throws IOException thrown if the configuration could not be read.
077     */
078    public XConfiguration(Reader reader) throws IOException {
079        this();
080        parse(reader);
081    }
082
083    /**
084     * Create an configuration from a Properties instance.
085     *
086     * @param props Properties instance to get all properties from.
087     */
088    public XConfiguration(Properties props) {
089        this();
090        for (Map.Entry entry : props.entrySet()) {
091            set((String) entry.getKey(), (String) entry.getValue());
092        }
093
094    }
095
096    /**
097     * Return a Properties instance with the configuration properties.
098     *
099     * @return a Properties instance with the configuration properties.
100     */
101    public Properties toProperties() {
102        Properties props = new Properties();
103        for (Map.Entry<String, String> entry : this) {
104            props.setProperty(entry.getKey(), entry.getValue());
105        }
106        return props;
107    }
108
109    // overriding get() & substitueVars from Configuration to honor defined variables
110    // over system properties
111    //wee need this because substituteVars() is a private method and does not behave like virtual
112    //in Configuration
113    /**
114     * Get the value of the <code>name</code> property, <code>null</code> if
115     * no such property exists.
116     *
117     * Values are processed for <a href="#VariableExpansion">variable expansion</a>
118     * before being returned.
119     *
120     * @param name the property name.
121     * @return the value of the <code>name</code> property,
122     *         or null if no such property exists.
123     */
124    @Override
125    public String get(String name) {
126      return substituteVars(getRaw(name));
127    }
128
129    /**
130     * Get the value of the <code>name</code> property. If no such property
131     * exists, then <code>defaultValue</code> is returned.
132     *
133     * @param name property name.
134     * @param defaultValue default value.
135     * @return property value, or <code>defaultValue</code> if the property
136     *         doesn't exist.
137     */
138    @Override
139    public String get(String name, String defaultValue) {
140        String value = getRaw(name);
141        if (value == null) {
142            value = defaultValue;
143        }
144        else {
145            value = substituteVars(value);
146        }
147        return value;
148    }
149
150    private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
151    private static int MAX_SUBST = 20;
152    protected static volatile boolean initalized = false;
153    private static void initSubstituteDepth() {
154        if (!initalized && Services.get() != null && Services.get().get(ConfigurationService.class) != null) {
155            MAX_SUBST = ConfigurationService.getInt(CONFIGURATION_SUBSTITUTE_DEPTH);
156            initalized = true;
157        }
158    }
159
160    private String substituteVars(String expr) {
161        if (expr == null) {
162            return null;
163        }
164        Matcher match = varPat.matcher("");
165        String eval = expr;
166        int s = 0;
167        while (MAX_SUBST == -1 || s < MAX_SUBST ) {
168            match.reset(eval);
169            if (!match.find()) {
170                return eval;
171            }
172            String var = match.group();
173            var = var.substring(2, var.length() - 1); // remove ${ .. }
174
175            String val = getRaw(var);
176            if (val == null) {
177                val = System.getProperty(var);
178            }
179
180            if (val == null) {
181                return eval; // return literal ${var}: var is unbound
182            }
183            // substitute
184            eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
185            s++;
186        }
187        throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
188    }
189
190    /**
191     * This is a stop gap fix for <link href="https://issues.apache.org/jira/browse/HADOOP-4416">HADOOP-4416</link>.
192     */
193    public Class<?> getClassByName(String name) throws ClassNotFoundException {
194        return super.getClassByName(name.trim());
195    }
196
197    /**
198     * Copy configuration key/value pairs from one configuration to another if a property exists in the target, it gets
199     * replaced.
200     *
201     * @param source source configuration.
202     * @param target target configuration.
203     */
204    public static void copy(Configuration source, Configuration target) {
205        for (Map.Entry<String, String> entry : source) {
206            target.set(entry.getKey(), entry.getValue());
207        }
208    }
209
210    /**
211     * Injects configuration key/value pairs from one configuration to another if the key does not exist in the target
212     * configuration.
213     *
214     * @param source source configuration.
215     * @param target target configuration.
216     */
217    public static void injectDefaults(Configuration source, Configuration target) {
218        if (source != null) {
219            for (Map.Entry<String, String> entry : source) {
220                if (target.get(entry.getKey()) == null) {
221                    target.set(entry.getKey(), entry.getValue());
222                }
223            }
224        }
225    }
226
227    /**
228     * Returns a new XConfiguration with all values trimmed.
229     *
230     * @return a new XConfiguration with all values trimmed.
231     */
232    public XConfiguration trim() {
233        XConfiguration trimmed = new XConfiguration();
234        for (Map.Entry<String, String> entry : this) {
235            trimmed.set(entry.getKey(), entry.getValue().trim());
236        }
237        return trimmed;
238    }
239
240    /**
241     * Returns a new XConfiguration instance with all inline values resolved.
242     *
243     * @return a new XConfiguration instance with all inline values resolved.
244     */
245    public XConfiguration resolve() {
246        XConfiguration resolved = new XConfiguration();
247        for (Map.Entry<String, String> entry : this) {
248            resolved.set(entry.getKey(), get(entry.getKey()));
249        }
250        return resolved;
251    }
252
253    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
254    private void parse(InputStream is) throws IOException {
255        try {
256            DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
257            // support for includes in the xml file
258            docBuilderFactory.setNamespaceAware(true);
259            docBuilderFactory.setXIncludeAware(true);
260            // ignore all comments inside the xml file
261            docBuilderFactory.setIgnoringComments(true);
262            DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
263            Document doc = builder.parse(is);
264            parseDocument(doc);
265
266        }
267        catch (SAXException e) {
268            throw new IOException(e);
269        }
270        catch (ParserConfigurationException e) {
271            throw new IOException(e);
272        }
273    }
274
275    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
276    private void parse(Reader reader) throws IOException {
277        try {
278            DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
279            // support for includes in the xml file
280            docBuilderFactory.setNamespaceAware(true);
281            docBuilderFactory.setXIncludeAware(true);
282            // ignore all comments inside the xml file
283            docBuilderFactory.setIgnoringComments(true);
284            DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
285            Document doc = builder.parse(new InputSource(reader));
286            parseDocument(doc);
287        }
288        catch (SAXException e) {
289            throw new IOException(e);
290        }
291        catch (ParserConfigurationException e) {
292            throw new IOException(e);
293        }
294    }
295
296    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
297    private void parseDocument(Document doc) throws IOException {
298        Element root = doc.getDocumentElement();
299        if (!"configuration".equals(root.getTagName())) {
300            throw new IOException("bad conf file: top-level element not <configuration>");
301        }
302        processNodes(root);
303    }
304
305    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
306    private void processNodes(Element root) throws IOException {
307        try {
308            NodeList props = root.getChildNodes();
309            for (int i = 0; i < props.getLength(); i++) {
310                Node propNode = props.item(i);
311                if (!(propNode instanceof Element)) {
312                    continue;
313                }
314                Element prop = (Element) propNode;
315                if (prop.getTagName().equals("configuration")) {
316                    processNodes(prop);
317                    continue;
318                }
319                if (!"property".equals(prop.getTagName())) {
320                    throw new IOException("bad conf file: element not <property>");
321                }
322                NodeList fields = prop.getChildNodes();
323                String attr = null;
324                String value = null;
325                for (int j = 0; j < fields.getLength(); j++) {
326                    Node fieldNode = fields.item(j);
327                    if (!(fieldNode instanceof Element)) {
328                        continue;
329                    }
330                    Element field = (Element) fieldNode;
331                    if ("name".equals(field.getTagName()) && field.hasChildNodes()) {
332                        attr = ((Text) field.getFirstChild()).getData().trim();
333                    }
334                    if ("value".equals(field.getTagName()) && field.hasChildNodes()) {
335                        value = ((Text) field.getFirstChild()).getData();
336                    }
337                }
338                if (attr != null && value != null) {
339                    set(attr, value);
340                }
341            }
342
343        }
344        catch (DOMException e) {
345            throw new IOException(e);
346        }
347    }
348
349    /**
350     * Return a string with the configuration in XML format.
351     *
352     * @return a string with the configuration in XML format.
353     */
354    public String toXmlString() {
355        return toXmlString(true);
356    }
357
358    public String toXmlString(boolean prolog) {
359        String xml;
360        try {
361            ByteArrayOutputStream baos = new ByteArrayOutputStream();
362            this.writeXml(baos);
363            baos.close();
364            xml = new String(baos.toByteArray());
365        }
366        catch (IOException ex) {
367            throw new RuntimeException("It should not happen, " + ex.getMessage(), ex);
368        }
369        if (!prolog) {
370            xml = xml.substring(xml.indexOf("<configuration>"));
371        }
372        return xml;
373    }
374
375    /**
376     * Get the comma delimited values of the name property as an array of trimmed Strings. If no such property is specified then
377     * null is returned.
378     *
379     * @param name property name.
380     * @return property value as an array of trimmed Strings, or null.
381     */
382    public String[] getTrimmedStrings(String name) {
383        String[] values = getStrings(name);
384        if (values != null) {
385            for (int i = 0; i < values.length; i++) {
386                values[i] = values[i].trim();
387            }
388        }
389        return values;
390    }
391}