001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import com.google.common.annotations.VisibleForTesting;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.FileSystem;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.mapreduce.MRConfig;
026import org.apache.hadoop.mapreduce.MRJobConfig;
027import org.apache.hadoop.mapreduce.v2.util.MRApps;
028import org.apache.hadoop.util.StringUtils;
029import org.apache.hadoop.yarn.api.ApplicationConstants;
030import org.apache.hadoop.yarn.conf.YarnConfiguration;
031
032import java.io.File;
033import java.io.IOException;
034import java.net.URI;
035import java.util.Arrays;
036import java.util.HashMap;
037import java.util.List;
038import java.util.Locale;
039import java.util.Map;
040
041
042public class ClasspathUtils {
043    private static boolean usingMiniYarnCluster = false;
044    private static final List<String> CLASSPATH_ENTRIES = Arrays.asList(
045            ApplicationConstants.Environment.PWD.$(),
046            ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + "*"
047    );
048
049    @VisibleForTesting
050    public static void setUsingMiniYarnCluster(boolean useMiniYarnCluster) {
051        usingMiniYarnCluster = useMiniYarnCluster;
052    }
053
054    // Adapted from MRApps#setClasspath.  Adds Yarn, HDFS, Common, and distributed cache jars.
055    public static void setupClasspath(Map<String, String> env, Configuration conf) throws IOException {
056        // Propagate the system classpath when using the mini cluster
057        if (usingMiniYarnCluster) {
058            MRApps.addToEnvironment(
059                    env,
060                    ApplicationConstants.Environment.CLASSPATH.name(),
061                    System.getProperty("java.class.path"), conf);
062        }
063
064        for (String entry : CLASSPATH_ENTRIES) {
065            MRApps.addToEnvironment(env, ApplicationConstants.Environment.CLASSPATH.name(), entry, conf);
066        }
067
068        // a * in the classpath will only find a .jar, so we need to filter out
069        // all .jars and add everything else
070        addToClasspathIfNotJar(org.apache.hadoop.mapreduce.filecache.DistributedCache.getFileClassPaths(conf),
071                org.apache.hadoop.mapreduce.filecache.DistributedCache.getCacheFiles(conf),
072                conf,
073                env, ApplicationConstants.Environment.PWD.$());
074        addToClasspathIfNotJar(org.apache.hadoop.mapreduce.filecache.DistributedCache.getArchiveClassPaths(conf),
075                org.apache.hadoop.mapreduce.filecache.DistributedCache.getCacheArchives(conf),
076                conf,
077                env, ApplicationConstants.Environment.PWD.$());
078
079
080        boolean crossPlatform = conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
081                MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
082
083        for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
084                crossPlatform
085                        ? YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH
086                        : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
087            MRApps.addToEnvironment(env, ApplicationConstants.Environment.CLASSPATH.name(),
088                    c.trim(), conf);
089        }
090    }
091
092    // Adapted from MRApps#setClasspath
093    public static void addMapReduceToClasspath(Map<String, String> env, Configuration conf) {
094        boolean crossPlatform = conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
095                MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
096
097        for (String c : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
098                crossPlatform ?
099                        StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH)
100                        : StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) {
101            MRApps.addToEnvironment(env, ApplicationConstants.Environment.CLASSPATH.name(),
102                    c.trim(), conf);
103        }
104    }
105
106    // Borrowed from MRApps#addToClasspathIfNotJar
107    private static void addToClasspathIfNotJar(Path[] paths,
108                                               URI[] withLinks, Configuration conf,
109                                               Map<String, String> environment,
110                                               String classpathEnvVar) throws IOException {
111        if (paths != null) {
112            HashMap<Path, String> linkLookup = new HashMap<Path, String>();
113            if (withLinks != null) {
114                for (URI u: withLinks) {
115                    Path p = new Path(u);
116                    FileSystem remoteFS = p.getFileSystem(conf);
117                    p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
118                            remoteFS.getWorkingDirectory()));
119                    String name = (null == u.getFragment())
120                            ? p.getName() : u.getFragment();
121                    if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) {
122                        linkLookup.put(p, name);
123                    }
124                }
125            }
126
127            for (Path p : paths) {
128                FileSystem remoteFS = p.getFileSystem(conf);
129                p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
130                        remoteFS.getWorkingDirectory()));
131                String name = linkLookup.get(p);
132                if (name == null) {
133                    name = p.getName();
134                }
135                if(!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) {
136                    MRApps.addToEnvironment(
137                            environment,
138                            classpathEnvVar,
139                            ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + name, conf);
140                }
141            }
142        }
143    }
144
145    public static Configuration addToClasspathFromLocalShareLib(Configuration conf, Path libPath) {
146        if (conf == null) {
147            conf = new Configuration(false);
148        }
149        final String pathStr = normalizedLocalFsPath(libPath);
150
151        String appClassPath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
152
153        if (org.apache.commons.lang.StringUtils.isEmpty(appClassPath)) {
154            addPathToYarnClasspathInConfig(conf, pathStr, StringUtils.join(File.pathSeparator,
155                    YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
156        } else {
157            addPathToYarnClasspathInConfig(conf, pathStr, appClassPath);
158        }
159        return conf;
160    }
161
162    private static void addPathToYarnClasspathInConfig(Configuration conf, String pathStr, String appClassPath) {
163        conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, appClassPath + File.pathSeparator + pathStr);
164    }
165
166    private static String normalizedLocalFsPath(Path libPath) {
167        return org.apache.commons.lang.StringUtils.replace(libPath.toString(), FSUtils.FILE_SCHEME_PREFIX, "");
168    }
169}