Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.oozie.service.ConfigurationService;
023import org.apache.oozie.service.Services;
024import org.w3c.dom.DOMException;
025import org.w3c.dom.Document;
026import org.w3c.dom.Element;
027import org.w3c.dom.Node;
028import org.w3c.dom.NodeList;
029import org.w3c.dom.Text;
030import org.xml.sax.SAXException;
031import org.xml.sax.InputSource;
032
033import javax.xml.parsers.DocumentBuilder;
034import javax.xml.parsers.DocumentBuilderFactory;
035import javax.xml.parsers.ParserConfigurationException;
036import java.io.IOException;
037import java.io.InputStream;
038import java.io.Reader;
039import java.io.ByteArrayOutputStream;
040import java.util.Map;
041import java.util.Properties;
042import java.util.regex.Matcher;
043import java.util.regex.Pattern;
044
045/**
046 * Extends Hadoop Configuration providing a new constructor which reads an XML configuration from an InputStream. <p>
047 * OConfiguration(InputStream is).
048 */
049public class XConfiguration extends Configuration {
050
051    public static final String CONFIGURATION_SUBSTITUTE_DEPTH = "oozie.configuration.substitute.depth";
052
053    /**
054     * Create an empty configuration. <p> Default values are not loaded.
055     */
056    public XConfiguration() {
057        super(false);
058        initSubstituteDepth();
059    }
060
061    /**
062     * Create a configuration from an InputStream. <p> Code canibalized from <code>Configuration.loadResource()</code>.
063     *
064     * @param is inputstream to read the configuration from.
065     * @throws IOException thrown if the configuration could not be read.
066     */
067    public XConfiguration(InputStream is) throws IOException {
068        this();
069        parse(is);
070    }
071
072    /**
073     * Create a configuration from an Reader. <p> Code canibalized from <code>Configuration.loadResource()</code>.
074     *
075     * @param reader reader to read the configuration from.
076     * @throws IOException thrown if the configuration could not be read.
077     */
078    public XConfiguration(Reader reader) throws IOException {
079        this();
080        parse(reader);
081    }
082
083    /**
084     * Create an configuration from a Properties instance.
085     *
086     * @param props Properties instance to get all properties from.
087     */
088    public XConfiguration(Properties props) {
089        this();
090        for (Map.Entry entry : props.entrySet()) {
091            set((String) entry.getKey(), (String) entry.getValue());
092        }
093
094    }
095
096    /**
097     * Return a Properties instance with the configuration properties.
098     *
099     * @return a Properties instance with the configuration properties.
100     */
101    public Properties toProperties() {
102        Properties props = new Properties();
103        for (Map.Entry<String, String> entry : this) {
104            props.setProperty(entry.getKey(), entry.getValue());
105        }
106        return props;
107    }
108
109    // overriding get() & substitueVars from Configuration to honor defined variables
110    // over system properties
111    //wee need this because substituteVars() is a private method and does not behave like virtual
112    //in Configuration
113    /**
114     * Get the value of the <code>name</code> property, <code>null</code> if
115     * no such property exists.
116     *
117     * Values are processed for <a href="#VariableExpansion">variable expansion</a>
118     * before being returned.
119     *
120     * @param name the property name.
121     * @return the value of the <code>name</code> property,
122     *         or null if no such property exists.
123     */
124    @Override
125    public String get(String name) {
126      return substituteVars(getRaw(name));
127    }
128
129    /**
130     * Get the value of the <code>name</code> property. If no such property
131     * exists, then <code>defaultValue</code> is returned.
132     *
133     * @param name property name.
134     * @param defaultValue default value.
135     * @return property value, or <code>defaultValue</code> if the property
136     *         doesn't exist.
137     */
138    @Override
139    public String get(String name, String defaultValue) {
140        String value = getRaw(name);
141        if (value == null) {
142            value = defaultValue;
143        }
144        else {
145            value = substituteVars(value);
146        }
147        return value;
148    }
149
150    private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
151    private static int MAX_SUBST = 20;
152    protected static volatile boolean initalized = false;
153    private static void initSubstituteDepth() {
154        if (!initalized && Services.get() != null && Services.get().get(ConfigurationService.class) != null) {
155            MAX_SUBST = ConfigurationService.getInt(CONFIGURATION_SUBSTITUTE_DEPTH);
156            initalized = true;
157        }
158    }
159
160    private String substituteVars(String expr) {
161        if (expr == null) {
162            return null;
163        }
164        Matcher match = varPat.matcher("");
165        String eval = expr;
166        int s = 0;
167        while (MAX_SUBST == -1 || s < MAX_SUBST ) {
168            match.reset(eval);
169            if (!match.find()) {
170                return eval;
171            }
172            String var = match.group();
173            var = var.substring(2, var.length() - 1); // remove ${ .. }
174
175            String val = getRaw(var);
176            if (val == null) {
177                val = System.getProperty(var);
178            }
179
180            if (val == null) {
181                return eval; // return literal ${var}: var is unbound
182            }
183            // substitute
184            eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
185            s++;
186        }
187        throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
188    }
189
190    /**
191     * This is a stop gap fix for HADOOP-4416.
192     */
193    public Class<?> getClassByName(String name) throws ClassNotFoundException {
194        return super.getClassByName(name.trim());
195    }
196
197    /**
198     * Copy configuration key/value pairs from one configuration to another if a property exists in the target, it gets
199     * replaced.
200     *
201     * @param source source configuration.
202     * @param target target configuration.
203     */
204    public static void copy(Configuration source, Configuration target) {
205        for (Map.Entry<String, String> entry : source) {
206            target.set(entry.getKey(), entry.getValue());
207        }
208    }
209
210    /**
211     * Injects configuration key/value pairs from one configuration to another if the key does not exist in the target
212     * configuration.
213     *
214     * @param source source configuration.
215     * @param target target configuration.
216     */
217    public static void injectDefaults(Configuration source, Configuration target) {
218        if (source != null) {
219            for (Map.Entry<String, String> entry : source) {
220                if (target.get(entry.getKey()) == null) {
221                    target.set(entry.getKey(), entry.getValue());
222                }
223            }
224        }
225    }
226
227    /**
228     * Returns a new XConfiguration with all values trimmed.
229     *
230     * @return a new XConfiguration with all values trimmed.
231     */
232    public XConfiguration trim() {
233        XConfiguration trimmed = new XConfiguration();
234        for (Map.Entry<String, String> entry : this) {
235            trimmed.set(entry.getKey(), entry.getValue().trim());
236        }
237        return trimmed;
238    }
239
240    /**
241     * Returns a new XConfiguration instance with all inline values resolved.
242     *
243     * @return a new XConfiguration instance with all inline values resolved.
244     */
245    public XConfiguration resolve() {
246        XConfiguration resolved = new XConfiguration();
247        for (Map.Entry<String, String> entry : this) {
248            resolved.set(entry.getKey(), get(entry.getKey()));
249        }
250        return resolved;
251    }
252
253    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
254    private void parse(InputStream is) throws IOException {
255        try {
256            DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
257            // support for includes in the xml file
258            docBuilderFactory.setNamespaceAware(true);
259            docBuilderFactory.setXIncludeAware(true);
260            // ignore all comments inside the xml file
261            docBuilderFactory.setIgnoringComments(true);
262            docBuilderFactory.setExpandEntityReferences(false);
263            docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
264            DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
265            Document doc = builder.parse(is);
266            parseDocument(doc);
267
268        }
269        catch (SAXException e) {
270            throw new IOException(e);
271        }
272        catch (ParserConfigurationException e) {
273            throw new IOException(e);
274        }
275    }
276
277    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
278    private void parse(Reader reader) throws IOException {
279        try {
280            DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
281            // support for includes in the xml file
282            docBuilderFactory.setNamespaceAware(true);
283            docBuilderFactory.setXIncludeAware(true);
284            // ignore all comments inside the xml file
285            docBuilderFactory.setIgnoringComments(true);
286            docBuilderFactory.setExpandEntityReferences(false);
287            docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
288            DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
289            Document doc = builder.parse(new InputSource(reader));
290            parseDocument(doc);
291        }
292        catch (SAXException e) {
293            throw new IOException(e);
294        }
295        catch (ParserConfigurationException e) {
296            throw new IOException(e);
297        }
298    }
299
300    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
301    private void parseDocument(Document doc) throws IOException {
302        Element root = doc.getDocumentElement();
303        if (!"configuration".equals(root.getLocalName())) {
304            throw new IOException("bad conf file: top-level element not <configuration>");
305        }
306        processNodes(root);
307    }
308
309    // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
310    private void processNodes(Element root) throws IOException {
311        try {
312            NodeList props = root.getChildNodes();
313            for (int i = 0; i < props.getLength(); i++) {
314                Node propNode = props.item(i);
315                if (!(propNode instanceof Element)) {
316                    continue;
317                }
318                Element prop = (Element) propNode;
319                if (prop.getLocalName().equals("configuration")) {
320                    processNodes(prop);
321                    continue;
322                }
323                if (!"property".equals(prop.getLocalName())) {
324                    throw new IOException("bad conf file: element not <property>");
325                }
326                NodeList fields = prop.getChildNodes();
327                String attr = null;
328                String value = null;
329                for (int j = 0; j < fields.getLength(); j++) {
330                    Node fieldNode = fields.item(j);
331                    if (!(fieldNode instanceof Element)) {
332                        continue;
333                    }
334                    Element field = (Element) fieldNode;
335                    if ("name".equals(field.getLocalName()) && field.hasChildNodes()) {
336                        attr = ((Text) field.getFirstChild()).getData().trim();
337                    }
338                    if ("value".equals(field.getLocalName()) && field.hasChildNodes()) {
339                        value = ((Text) field.getFirstChild()).getData();
340                    }
341                }
342                if (attr != null && value != null) {
343                    set(attr, value);
344                }
345            }
346
347        }
348        catch (DOMException e) {
349            throw new IOException(e);
350        }
351    }
352
353    /**
354     * Return a string with the configuration in XML format.
355     *
356     * @return a string with the configuration in XML format.
357     */
358    public String toXmlString() {
359        return toXmlString(true);
360    }
361
362    public String toXmlString(boolean prolog) {
363        String xml;
364        try {
365            ByteArrayOutputStream baos = new ByteArrayOutputStream();
366            this.writeXml(baos);
367            baos.close();
368            xml = new String(baos.toByteArray());
369        }
370        catch (IOException ex) {
371            throw new RuntimeException("It should not happen, " + ex.getMessage(), ex);
372        }
373        if (!prolog) {
374            xml = xml.substring(xml.indexOf("<configuration>"));
375        }
376        return xml;
377    }
378
379    /**
380     * Get the comma delimited values of the name property as an array of trimmed Strings. If no such property is specified then
381     * null is returned.
382     *
383     * @param name property name.
384     * @return property value as an array of trimmed Strings, or null.
385     */
386    public String[] getTrimmedStrings(String name) {
387        String[] values = getStrings(name);
388        if (values != null) {
389            for (int i = 0; i < values.length; i++) {
390                values[i] = values[i].trim();
391            }
392        }
393        return values;
394    }
395}