Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.oozie.service.ConfigurationService;
023import org.apache.oozie.service.Services;
024import org.w3c.dom.DOMException;
025import org.w3c.dom.Document;
026import org.w3c.dom.Element;
027import org.w3c.dom.Node;
028import org.w3c.dom.NodeList;
029import org.w3c.dom.Text;
030import org.xml.sax.SAXException;
031import org.xml.sax.InputSource;
032
033import javax.xml.parsers.DocumentBuilder;
034import javax.xml.parsers.DocumentBuilderFactory;
035import javax.xml.parsers.ParserConfigurationException;
036import java.io.IOException;
037import java.io.InputStream;
038import java.io.Reader;
039import java.io.ByteArrayOutputStream;
040import java.util.Map;
041import java.util.Properties;
042import java.util.regex.Matcher;
043import java.util.regex.Pattern;
044
045/**
046 * Extends Hadoop Configuration providing a new constructor which reads an XML configuration from an InputStream. <p>
047 * OConfiguration(InputStream is).
048 */
049public class XConfiguration extends Configuration {
050
051    public static final String CONFIGURATION_SUBSTITUTE_DEPTH = "oozie.configuration.substitute.depth";
052
053    private boolean restrictSystemProperties = true;
054    private boolean restrictParser = true;
055    /**
056     * Create an empty configuration. <p> Default values are not loaded.
057     */
058    public XConfiguration() {
059        super(false);
060        initSubstituteDepth();
061    }
062
063    /**
064     * Create a configuration from an InputStream. <p> Code canibalized from <code>Configuration.loadResource()</code>.
065     *
066     * @param is inputstream to read the configuration from.
067     * @param restrictParser whether to restrict the parser
068     * @throws IOException thrown if the configuration could not be read.
069     */
070    public XConfiguration(InputStream is, boolean restrictParser) throws IOException {
071        this();
072        this.restrictParser = restrictParser;
073        parse(is);
074    }
075
076    /**
077     * Create a configuration from an InputStream. <p> Code canibalized from <code>Configuration.loadResource()</code>.
078     *
079     * @param is inputstream to read the configuration from.
080     * @throws IOException thrown if the configuration could not be read.
081     */
082    public XConfiguration(InputStream is) throws IOException {
083        this(is, true);
084    }
085
086    /**
087     * Create a configuration from an Reader. <p> Code canibalized from <code>Configuration.loadResource()</code>.
088     *
089     * @param reader reader to read the configuration from.
090     * @param restrictParser whether to restrict the parser
091     * @throws IOException thrown if the configuration could not be read.
092     */
093    public XConfiguration(Reader reader, boolean restrictParser) throws IOException {
094        this();
095        this.restrictParser = restrictParser;
096        parse(reader);
097    }
098
099    /**
100     * Create a configuration from an Reader. <p> Code canibalized from <code>Configuration.loadResource()</code>.
101     *
102     * @param reader reader to read the configuration from.
103     * @throws IOException thrown if the configuration could not be read.
104     */
105    public XConfiguration(Reader reader) throws IOException {
106        this(reader, true);
107    }
108
109    /**
110     * Create an configuration from a Properties instance.
111     *
112     * @param props Properties instance to get all properties from.
113     */
114    public XConfiguration(Properties props) {
115        this();
116        for (Map.Entry entry : props.entrySet()) {
117            set((String) entry.getKey(), (String) entry.getValue());
118        }
119    }
120
121    /**
122     * Return a Properties instance with the configuration properties.
123     *
124     * @return a Properties instance with the configuration properties.
125     */
126    public Properties toProperties() {
127        Properties props = new Properties();
128        for (Map.Entry<String, String> entry : this) {
129            props.setProperty(entry.getKey(), entry.getValue());
130        }
131        return props;
132    }
133
134    // overriding get() & substitueVars from Configuration to honor defined variables
135    // over system properties
136    //wee need this because substituteVars() is a private method and does not behave like virtual
137    //in Configuration
138    /**
139     * Get the value of the <code>name</code> property, <code>null</code> if
140     * no such property exists.
141     *
142     * Values are processed for <a href="#VariableExpansion">variable expansion</a>
143     * before being returned.
144     *
145     * @param name the property name.
146     * @return the value of the <code>name</code> property,
147     *         or null if no such property exists.
148     */
149    @Override
150    public String get(String name) {
151        return substituteVars(getRaw(name));
152    }
153
154    /**
155     * Get the value of the <code>name</code> property. If no such property
156     * exists, then <code>defaultValue</code> is returned.
157     *
158     * @param name property name.
159     * @param defaultValue default value.
160     * @return property value, or <code>defaultValue</code> if the property
161     *         doesn't exist.
162     */
163    @Override
164    public String get(String name, String defaultValue) {
165        String value = getRaw(name);
166        if (value == null) {
167            value = defaultValue;
168        }
169        else {
170            value = substituteVars(value);
171        }
172        return value;
173    }
174
175    private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
176    private static int MAX_SUBST = 20;
177    protected static volatile boolean initalized = false;
178    private static void initSubstituteDepth() {
179        if (!initalized && Services.get() != null && Services.get().get(ConfigurationService.class) != null) {
180            MAX_SUBST = ConfigurationService.getInt(CONFIGURATION_SUBSTITUTE_DEPTH);
181            initalized = true;
182        }
183    }
184
185    private String substituteVars(String expr) {
186        if (expr == null) {
187            return null;
188        }
189        Matcher match = varPat.matcher("");
190        String eval = expr;
191        int s = 0;
192        while (MAX_SUBST == -1 || s < MAX_SUBST ) {
193            match.reset(eval);
194            if (!match.find()) {
195                return eval;
196            }
197            String var = match.group();
198            var = var.substring(2, var.length() - 1); // remove ${ .. }
199
200            String val = getRaw(var);
201
202            if (val == null && !this.restrictSystemProperties) {
203                val =  System.getProperty(var);
204            }
205
206            if (val == null) {
207                return eval; // return literal ${var}: var is unbound
208            }
209            // substitute
210            eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
211            s++;
212        }
213        throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
214    }
215
216    /**
217     * This is a stop gap fix for HADOOP-4416.
218     */
219    public Class<?> getClassByName(String name) throws ClassNotFoundException {
220        return super.getClassByName(name.trim());
221    }
222
223    /**
224     * Copy configuration key/value pairs from one configuration to another if a property exists in the target, it gets
225     * replaced.
226     *
227     * @param source source configuration.
228     * @param target target configuration.
229     */
230    public static void copy(Configuration source, Configuration target) {
231        for (Map.Entry<String, String> entry : source) {
232            target.set(entry.getKey(), entry.getValue());
233        }
234    }
235
236    /**
237     * Injects configuration key/value pairs from one configuration to another if the key does not exist in the target
238     * configuration.
239     *
240     * @param source source configuration.
241     * @param target target configuration.
242     */
243    public static void injectDefaults(Configuration source, Configuration target) {
244        if (source != null) {
245            for (Map.Entry<String, String> entry : source) {
246                if (target.get(entry.getKey()) == null) {
247                    target.set(entry.getKey(), entry.getValue());
248                }
249            }
250        }
251    }
252
253    /**
254     * Returns a new XConfiguration with all values trimmed.
255     *
256     * @return a new XConfiguration with all values trimmed.
257     */
258    public XConfiguration trim() {
259        XConfiguration trimmed = new XConfiguration();
260        for (Map.Entry<String, String> entry : this) {
261            trimmed.set(entry.getKey(), entry.getValue().trim());
262        }
263        return trimmed;
264    }
265
266    /**
267     * Returns a new XConfiguration instance with all inline values resolved.
268     *
269     * @return a new XConfiguration instance with all inline values resolved.
270     */
271    public XConfiguration resolve() {
272        XConfiguration resolved = new XConfiguration();
273        for (Map.Entry<String, String> entry : this) {
274            resolved.set(entry.getKey(), get(entry.getKey()));
275        }
276        return resolved;
277    }
278
279    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
280    private void parse(InputStream is) throws IOException {
281        try {
282            Document doc = getDocumentBuilder().parse(is);
283            parseDocument(doc);
284
285        }
286        catch (SAXException e) {
287            throw new IOException(e);
288        }
289        catch (ParserConfigurationException e) {
290            throw new IOException(e);
291        }
292    }
293
294    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
295    private void parse(Reader reader) throws IOException {
296        try {
297            Document doc = getDocumentBuilder().parse(new InputSource(reader));
298            parseDocument(doc);
299        }
300        catch (SAXException e) {
301            throw new IOException(e);
302        }
303        catch (ParserConfigurationException e) {
304            throw new IOException(e);
305        }
306    }
307
308    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
309    private void parseDocument(Document doc) throws IOException {
310        Element root = doc.getDocumentElement();
311        if (!"configuration".equals(root.getLocalName())) {
312            throw new IOException("bad conf file: top-level element not <configuration>");
313        }
314        processNodes(root);
315    }
316
317    // Cannibalized from Hadoop <code>Configuration.loadResource()</code>.
318    private void processNodes(Element root) throws IOException {
319        try {
320            NodeList props = root.getChildNodes();
321            for (int i = 0; i < props.getLength(); i++) {
322                Node propNode = props.item(i);
323                if (!(propNode instanceof Element)) {
324                    continue;
325                }
326                Element prop = (Element) propNode;
327                if (prop.getLocalName().equals("configuration")) {
328                    processNodes(prop);
329                    continue;
330                }
331                if (!"property".equals(prop.getLocalName())) {
332                    throw new IOException("bad conf file: element not <property>");
333                }
334                NodeList fields = prop.getChildNodes();
335                String attr = null;
336                String value = null;
337                for (int j = 0; j < fields.getLength(); j++) {
338                    Node fieldNode = fields.item(j);
339                    if (!(fieldNode instanceof Element)) {
340                        continue;
341                    }
342                    Element field = (Element) fieldNode;
343                    if ("name".equals(field.getLocalName()) && field.hasChildNodes()) {
344                        attr = ((Text) field.getFirstChild()).getData().trim();
345                    }
346                    if ("value".equals(field.getLocalName()) && field.hasChildNodes()) {
347                        value = ((Text) field.getFirstChild()).getData();
348                    }
349                }
350                if (attr != null && value != null) {
351                    set(attr, value);
352                }
353            }
354
355        }
356        catch (DOMException e) {
357            throw new IOException(e);
358        }
359    }
360
361    /**
362     * Return a string with the configuration in XML format.
363     *
364     * @return a string with the configuration in XML format.
365     */
366    public String toXmlString() {
367        return toXmlString(true);
368    }
369
370    public String toXmlString(boolean prolog) {
371        String xml;
372        try {
373            ByteArrayOutputStream baos = new ByteArrayOutputStream();
374            this.writeXml(baos);
375            baos.close();
376            xml = new String(baos.toByteArray());
377        }
378        catch (IOException ex) {
379            throw new RuntimeException("It should not happen, " + ex.getMessage(), ex);
380        }
381        if (!prolog) {
382            xml = xml.substring(xml.indexOf("<configuration>"));
383        }
384        return xml;
385    }
386
387    /**
388     * Get the comma delimited values of the name property as an array of trimmed Strings. If no such property is specified then
389     * null is returned.
390     *
391     * @param name property name.
392     * @return property value as an array of trimmed Strings, or null.
393     */
394    public String[] getTrimmedStrings(String name) {
395        String[] values = getStrings(name);
396        if (values != null) {
397            for (int i = 0; i < values.length; i++) {
398                values[i] = values[i].trim();
399            }
400        }
401        return values;
402    }
403
404    private DocumentBuilder getDocumentBuilder() throws ParserConfigurationException {
405        DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
406        docBuilderFactory.setNamespaceAware(true);
407        docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl",true);
408        docBuilderFactory.setXIncludeAware(true);
409        if(this.restrictParser) {
410            docBuilderFactory.setXIncludeAware(false);
411            //Redundant with disallow-doctype, but just in case
412            docBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
413            docBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
414            docBuilderFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
415        }
416        docBuilderFactory.setExpandEntityReferences(false);
417        // ignore all comments inside the xml file
418        docBuilderFactory.setIgnoringComments(true);
419        return docBuilderFactory.newDocumentBuilder();
420    }
421
422    /**
423     * Restrict the parser
424     * @param restrictParser
425     */
426    public void setRestrictParser(boolean restrictParser) {
427     this.restrictParser = restrictParser;
428    }
429
430    public boolean getRestrictParser() {
431        return restrictParser;
432    }
433    /**
434     * Restrict reading property from System.getProperty()
435     * @param restrictSystemProperties
436     */
437    public void setRestrictSystemProperties(boolean restrictSystemProperties) {
438        this.restrictSystemProperties = restrictSystemProperties;
439    }
440
441    public boolean getRestrictSystemProperties() {
442        return restrictSystemProperties;
443    }
444
445
446
447}