001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.action.hadoop; 020 021import org.apache.hadoop.conf.Configuration; 022import org.apache.oozie.DagELFunctions; 023import org.apache.oozie.service.HadoopAccessorService; 024import org.apache.oozie.service.Services; 025import org.apache.oozie.util.ELEvaluationException; 026import org.apache.oozie.util.XLog; 027import org.apache.oozie.workflow.WorkflowInstance; 028import org.json.simple.JSONValue; 029 030import java.util.Map; 031 032/** 033 * Hadoop EL functions. 034 */ 035public class HadoopELFunctions { 036 037 private static final String HADOOP_COUNTERS = "oozie.el.action.hadoop.counters"; 038 039 public static final String RECORDS = "org.apache.hadoop.mapred.Task$Counter"; 040 public static final String MAP_IN = "MAP_INPUT_RECORDS"; 041 public static final String MAP_OUT = "MAP_OUTPUT_RECORDS"; 042 public static final String REDUCE_IN = "REDUCE_INPUT_RECORDS"; 043 public static final String REDUCE_OUT = "REDUCE_OUTPUT_RECORDS"; 044 public static final String GROUPS = "REDUCE_INPUT_GROUPS"; 045 046 private static final String RECORDS_023 = "org.apache.hadoop.mapreduce.TaskCounter"; 047 048 @SuppressWarnings("unchecked") 049 public static Map<String, Map<String, Long>> hadoop_counters(String nodeName) throws ELEvaluationException { 050 WorkflowInstance instance = DagELFunctions.getWorkflow().getWorkflowInstance(); 051 Object obj = instance.getTransientVar(nodeName + WorkflowInstance.NODE_VAR_SEPARATOR + HADOOP_COUNTERS); 052 Map<String, Map<String, Long>> counters = (Map<String, Map<String, Long>>) obj; 053 if (counters == null) { 054 counters = getCounters(nodeName); 055 instance.setTransientVar(nodeName + WorkflowInstance.NODE_VAR_SEPARATOR + HADOOP_COUNTERS, counters); 056 } 057 return counters; 058 } 059 060 public static String hadoop_conf(String hadoopConfHostPort, String propName) { 061 Configuration conf = Services.get().get(HadoopAccessorService.class) 062 .createConfiguration(hadoopConfHostPort); 063 String prop = conf.get(propName); 064 if (prop == null || prop.equals("")) { 065 conf = new Configuration(); 066 prop = conf.get(propName); 067 } 068 if (prop == null) 069 prop = ""; 070 return prop; 071 } 072 073 @SuppressWarnings("unchecked") 074 private static Map<String, Map<String, Long>> getCounters(String nodeName) throws ELEvaluationException { 075 String jsonCounters = DagELFunctions.getActionVar(nodeName, MapReduceActionExecutor.HADOOP_COUNTERS); 076 if (jsonCounters == null) { 077 throw new IllegalArgumentException(XLog.format("Hadoop counters not available for action [{0}]", nodeName)); 078 } 079 return (Map<String, Map<String, Long>>) JSONValue.parse(jsonCounters); 080 } 081 082}