001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.oozie.cli; 019 020import com.google.common.annotations.VisibleForTesting; 021import org.apache.commons.cli.CommandLine; 022import org.apache.commons.cli.Option; 023import org.apache.commons.cli.OptionBuilder; 024import org.apache.commons.cli.OptionGroup; 025import org.apache.commons.cli.Options; 026import org.apache.commons.cli.ParseException; 027import org.apache.oozie.BuildInfo; 028import org.apache.oozie.client.AuthOozieClient; 029import org.apache.oozie.client.BulkResponse; 030import org.apache.oozie.client.BundleJob; 031import org.apache.oozie.client.CoordinatorAction; 032import org.apache.oozie.client.CoordinatorJob; 033import org.apache.oozie.client.OozieClient; 034import org.apache.oozie.client.OozieClient.SYSTEM_MODE; 035import org.apache.oozie.client.OozieClientException; 036import org.apache.oozie.client.WorkflowAction; 037import org.apache.oozie.client.WorkflowJob; 038import org.apache.oozie.client.XOozieClient; 039import org.apache.oozie.client.rest.RestConstants; 040import org.w3c.dom.DOMException; 041import org.w3c.dom.Document; 042import org.w3c.dom.Element; 043import org.w3c.dom.Node; 044import org.w3c.dom.NodeList; 045import org.w3c.dom.Text; 046import org.xml.sax.SAXException; 047 048import javax.xml.XMLConstants; 049import javax.xml.parsers.DocumentBuilder; 050import javax.xml.parsers.DocumentBuilderFactory; 051import javax.xml.parsers.ParserConfigurationException; 052import javax.xml.transform.stream.StreamSource; 053import javax.xml.validation.Schema; 054import javax.xml.validation.SchemaFactory; 055import javax.xml.validation.Validator; 056import java.io.File; 057import java.io.FileInputStream; 058import java.io.FileReader; 059import java.io.IOException; 060import java.io.InputStream; 061import java.io.PrintStream; 062import java.text.SimpleDateFormat; 063import java.util.ArrayList; 064import java.util.Date; 065import java.util.List; 066import java.util.Locale; 067import java.util.Map; 068import java.util.Properties; 069import java.util.TimeZone; 070import java.util.concurrent.Callable; 071import java.util.regex.Matcher; 072import java.util.regex.Pattern; 073 074/** 075 * Oozie command line utility. 076 */ 077public class OozieCLI { 078 public static final String ENV_OOZIE_URL = "OOZIE_URL"; 079 public static final String ENV_OOZIE_DEBUG = "OOZIE_DEBUG"; 080 public static final String ENV_OOZIE_TIME_ZONE = "OOZIE_TIMEZONE"; 081 public static final String ENV_OOZIE_AUTH = "OOZIE_AUTH"; 082 public static final String OOZIE_RETRY_COUNT = "oozie.connection.retry.count"; 083 public static final String WS_HEADER_PREFIX = "header:"; 084 085 public static final String HELP_CMD = "help"; 086 public static final String VERSION_CMD = "version"; 087 public static final String JOB_CMD = "job"; 088 public static final String JOBS_CMD = "jobs"; 089 public static final String ADMIN_CMD = "admin"; 090 public static final String VALIDATE_CMD = "validate"; 091 public static final String SLA_CMD = "sla"; 092 public static final String PIG_CMD = "pig"; 093 public static final String HIVE_CMD = "hive"; 094 public static final String SQOOP_CMD = "sqoop"; 095 public static final String MR_CMD = "mapreduce"; 096 public static final String INFO_CMD = "info"; 097 098 public static final String OOZIE_OPTION = "oozie"; 099 public static final String CONFIG_OPTION = "config"; 100 public static final String SUBMIT_OPTION = "submit"; 101 public static final String OFFSET_OPTION = "offset"; 102 public static final String START_OPTION = "start"; 103 public static final String RUN_OPTION = "run"; 104 public static final String DRYRUN_OPTION = "dryrun"; 105 public static final String SUSPEND_OPTION = "suspend"; 106 public static final String RESUME_OPTION = "resume"; 107 public static final String KILL_OPTION = "kill"; 108 public static final String CHANGE_OPTION = "change"; 109 public static final String CHANGE_VALUE_OPTION = "value"; 110 public static final String RERUN_OPTION = "rerun"; 111 public static final String INFO_OPTION = "info"; 112 public static final String LOG_OPTION = "log"; 113 public static final String ACTION_OPTION = "action"; 114 public static final String DEFINITION_OPTION = "definition"; 115 public static final String CONFIG_CONTENT_OPTION = "configcontent"; 116 public static final String SQOOP_COMMAND_OPTION = "command"; 117 public static final String SHOWDIFF_OPTION = "diff"; 118 public static final String UPDATE_OPTION = "update"; 119 public static final String IGNORE_OPTION = "ignore"; 120 121 public static final String DO_AS_OPTION = "doas"; 122 123 public static final String LEN_OPTION = "len"; 124 public static final String FILTER_OPTION = "filter"; 125 public static final String JOBTYPE_OPTION = "jobtype"; 126 public static final String SYSTEM_MODE_OPTION = "systemmode"; 127 public static final String VERSION_OPTION = "version"; 128 public static final String STATUS_OPTION = "status"; 129 public static final String LOCAL_TIME_OPTION = "localtime"; 130 public static final String TIME_ZONE_OPTION = "timezone"; 131 public static final String QUEUE_DUMP_OPTION = "queuedump"; 132 public static final String RERUN_COORD_OPTION = "coordinator"; 133 public static final String DATE_OPTION = "date"; 134 public static final String RERUN_REFRESH_OPTION = "refresh"; 135 public static final String RERUN_NOCLEANUP_OPTION = "nocleanup"; 136 public static final String ORDER_OPTION = "order"; 137 138 public static final String UPDATE_SHARELIB_OPTION = "sharelibupdate"; 139 140 public static final String LIST_SHARELIB_LIB_OPTION = "shareliblist"; 141 142 143 144 public static final String AUTH_OPTION = "auth"; 145 146 public static final String VERBOSE_OPTION = "verbose"; 147 public static final String VERBOSE_DELIMITER = "\t"; 148 public static final String DEBUG_OPTION = "debug"; 149 150 public static final String SCRIPTFILE_OPTION = "file"; 151 152 public static final String INFO_TIME_ZONES_OPTION = "timezones"; 153 154 public static final String BULK_OPTION = "bulk"; 155 156 public static final String AVAILABLE_SERVERS_OPTION = "servers"; 157 158 public static final String ALL_WORKFLOWS_FOR_COORD_ACTION = "allruns"; 159 160 private static final String[] OOZIE_HELP = { 161 "the env variable '" + ENV_OOZIE_URL + "' is used as default value for the '-" + OOZIE_OPTION + "' option", 162 "the env variable '" + ENV_OOZIE_TIME_ZONE + "' is used as default value for the '-" + TIME_ZONE_OPTION + "' option", 163 "the env variable '" + ENV_OOZIE_AUTH + "' is used as default value for the '-" + AUTH_OPTION + "' option", 164 "custom headers for Oozie web services can be specified using '-D" + WS_HEADER_PREFIX + "NAME=VALUE'" }; 165 166 private static final String RULER; 167 private static final int LINE_WIDTH = 132; 168 169 private static final int RETRY_COUNT = 4; 170 171 private boolean used; 172 173 private static final String INSTANCE_SEPARATOR = "#"; 174 175 private static final String MAPRED_MAPPER = "mapred.mapper.class"; 176 private static final String MAPRED_MAPPER_2 = "mapreduce.map.class"; 177 private static final String MAPRED_REDUCER = "mapred.reducer.class"; 178 private static final String MAPRED_REDUCER_2 = "mapreduce.reduce.class"; 179 private static final String MAPRED_INPUT = "mapred.input.dir"; 180 private static final String MAPRED_OUTPUT = "mapred.output.dir"; 181 182 private static final Pattern GMT_OFFSET_SHORTEN_PATTERN = Pattern.compile("(.* )GMT((?:-|\\+)\\d{2}:\\d{2})"); 183 184 static { 185 StringBuilder sb = new StringBuilder(); 186 for (int i = 0; i < LINE_WIDTH; i++) { 187 sb.append("-"); 188 } 189 RULER = sb.toString(); 190 } 191 192 /** 193 * Entry point for the Oozie CLI when invoked from the command line. 194 * <p/> 195 * Upon completion this method exits the JVM with '0' (success) or '-1' (failure). 196 * 197 * @param args options and arguments for the Oozie CLI. 198 */ 199 public static void main(String[] args) { 200 if (!System.getProperties().containsKey(AuthOozieClient.USE_AUTH_TOKEN_CACHE_SYS_PROP)) { 201 System.setProperty(AuthOozieClient.USE_AUTH_TOKEN_CACHE_SYS_PROP, "true"); 202 } 203 System.exit(new OozieCLI().run(args)); 204 } 205 206 /** 207 * Create an Oozie CLI instance. 208 */ 209 public OozieCLI() { 210 used = false; 211 } 212 213 /** 214 * Return Oozie CLI top help lines. 215 * 216 * @return help lines. 217 */ 218 protected String[] getCLIHelp() { 219 return OOZIE_HELP; 220 } 221 222 /** 223 * Add authentication specific options to oozie cli 224 * 225 * @param options the collection of options to add auth options 226 */ 227 protected void addAuthOptions(Options options) { 228 Option auth = new Option(AUTH_OPTION, true, "select authentication type [SIMPLE|KERBEROS]"); 229 options.addOption(auth); 230 } 231 232 /** 233 * Create option for command line option 'admin' 234 * @return admin options 235 */ 236 protected Options createAdminOptions() { 237 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 238 Option system_mode = new Option(SYSTEM_MODE_OPTION, true, 239 "Supported in Oozie-2.0 or later versions ONLY. Change oozie system mode [NORMAL|NOWEBSERVICE|SAFEMODE]"); 240 Option status = new Option(STATUS_OPTION, false, "show the current system status"); 241 Option version = new Option(VERSION_OPTION, false, "show Oozie server build version"); 242 Option queuedump = new Option(QUEUE_DUMP_OPTION, false, "show Oozie server queue elements"); 243 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 244 Option availServers = new Option(AVAILABLE_SERVERS_OPTION, false, "list available Oozie servers" 245 + " (more than one only if HA is enabled)"); 246 Option sharelibUpdate = new Option(UPDATE_SHARELIB_OPTION, false, "Update server to use a newer version of sharelib"); 247 248 Option sharelib = new Option(LIST_SHARELIB_LIB_OPTION, false, 249 "List available sharelib that can be specified in a workflow action"); 250 sharelib.setOptionalArg(true); 251 252 Options adminOptions = new Options(); 253 adminOptions.addOption(oozie); 254 adminOptions.addOption(doAs); 255 OptionGroup group = new OptionGroup(); 256 group.addOption(system_mode); 257 group.addOption(status); 258 group.addOption(version); 259 group.addOption(queuedump); 260 group.addOption(availServers); 261 group.addOption(sharelibUpdate); 262 group.addOption(sharelib); 263 adminOptions.addOptionGroup(group); 264 addAuthOptions(adminOptions); 265 return adminOptions; 266 } 267 268 /** 269 * Create option for command line option 'job' 270 * @return job options 271 */ 272 protected Options createJobOptions() { 273 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 274 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.xml' or '.properties'"); 275 Option submit = new Option(SUBMIT_OPTION, false, "submit a job"); 276 Option run = new Option(RUN_OPTION, false, "run a job"); 277 Option debug = new Option(DEBUG_OPTION, false, "Use debug mode to see debugging statements on stdout"); 278 Option rerun = new Option(RERUN_OPTION, true, 279 "rerun a job (coordinator requires -action or -date, bundle requires -coordinator or -date)"); 280 Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2) or coordinator (since 2.0) job without" 281 + " actually executing it"); 282 Option update = new Option(UPDATE_OPTION, true, "Update coord definition and properties"); 283 Option showdiff = new Option(SHOWDIFF_OPTION, true, 284 "Show diff of the new coord definition and properties with the existing one (default true)"); 285 Option start = new Option(START_OPTION, true, "start a job"); 286 Option suspend = new Option(SUSPEND_OPTION, true, "suspend a job"); 287 Option resume = new Option(RESUME_OPTION, true, "resume a job"); 288 Option kill = new Option(KILL_OPTION, true, "kill a job (coordinator can mention -action or -date)"); 289 Option change = new Option(CHANGE_OPTION, true, "change a coordinator job"); 290 Option changeValue = new Option(CHANGE_VALUE_OPTION, true, 291 "new endtime/concurrency/pausetime value for changing a coordinator job"); 292 Option info = new Option(INFO_OPTION, true, "info of a job"); 293 Option offset = new Option(OFFSET_OPTION, true, "job info offset of actions (default '1', requires -info)"); 294 Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)"); 295 Option filter = new Option(FILTER_OPTION, true, 296 "<key><comparator><value>[;<key><comparator><value>]*\n" 297 + "(All Coordinator actions satisfying the filters will be retreived).\n" 298 + "key: status or nominaltime\n" 299 + "comparator: =, !=, <, <=, >, >=. = is used as OR and others as AND\n" 300 + "status: values are valid status like SUCCEEDED, KILLED etc. Only = and != apply for status\n" 301 + "nominaltime: time of format yyyy-MM-dd'T'HH:mm'Z'"); 302 Option order = new Option(ORDER_OPTION, true, 303 "order to show coord actions (default ascending order, 'desc' for descending order, requires -info)"); 304 Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" + 305 TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option"); 306 Option timezone = new Option(TIME_ZONE_OPTION, true, 307 "use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list"); 308 Option log = new Option(LOG_OPTION, true, "job log"); 309 Option logFilter = new Option( 310 RestConstants.LOG_FILTER_OPTION, true, 311 "job log search parameter. Can be specified as -logfilter opt1=val1;opt2=val1;opt3=val1. " 312 + "Supported options are recent, start, end, loglevel, text, limit and debug"); 313 Option definition = new Option(DEFINITION_OPTION, true, "job definition"); 314 Option config_content = new Option(CONFIG_CONTENT_OPTION, true, "job configuration"); 315 Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode"); 316 Option action = new Option(ACTION_OPTION, true, 317 "coordinator rerun on action ids (requires -rerun); coordinator log retrieval on action ids (requires -log)"); 318 Option date = new Option(DATE_OPTION, true, 319 "coordinator/bundle rerun on action dates (requires -rerun); coordinator log retrieval on action dates (requires -log)"); 320 Option rerun_coord = new Option(RERUN_COORD_OPTION, true, "bundle rerun on coordinator names (requires -rerun)"); 321 Option rerun_refresh = new Option(RERUN_REFRESH_OPTION, false, 322 "re-materialize the coordinator rerun actions (requires -rerun)"); 323 Option rerun_nocleanup = new Option(RERUN_NOCLEANUP_OPTION, false, 324 "do not clean up output-events of the coordiantor rerun actions (requires -rerun)"); 325 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( 326 "set/override value for given property").create("D"); 327 Option getAllWorkflows = new Option(ALL_WORKFLOWS_FOR_COORD_ACTION, false, 328 "Get workflow jobs corresponding to a coordinator action including all the reruns"); 329 Option ignore = new Option(IGNORE_OPTION, true, 330 "change status of a coordinator job or action to IGNORED" 331 + " (-action required to ignore coord actions)"); 332 333 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 334 335 OptionGroup actions = new OptionGroup(); 336 actions.addOption(submit); 337 actions.addOption(start); 338 actions.addOption(run); 339 actions.addOption(dryrun); 340 actions.addOption(suspend); 341 actions.addOption(resume); 342 actions.addOption(kill); 343 actions.addOption(change); 344 actions.addOption(update); 345 actions.addOption(info); 346 actions.addOption(rerun); 347 actions.addOption(log); 348 actions.addOption(definition); 349 actions.addOption(config_content); 350 actions.addOption(ignore); 351 actions.setRequired(true); 352 Options jobOptions = new Options(); 353 jobOptions.addOption(oozie); 354 jobOptions.addOption(doAs); 355 jobOptions.addOption(config); 356 jobOptions.addOption(property); 357 jobOptions.addOption(changeValue); 358 jobOptions.addOption(localtime); 359 jobOptions.addOption(timezone); 360 jobOptions.addOption(verbose); 361 jobOptions.addOption(debug); 362 jobOptions.addOption(offset); 363 jobOptions.addOption(len); 364 jobOptions.addOption(filter); 365 jobOptions.addOption(order); 366 jobOptions.addOption(action); 367 jobOptions.addOption(date); 368 jobOptions.addOption(rerun_coord); 369 jobOptions.addOption(rerun_refresh); 370 jobOptions.addOption(rerun_nocleanup); 371 jobOptions.addOption(getAllWorkflows); 372 jobOptions.addOptionGroup(actions); 373 jobOptions.addOption(logFilter); 374 addAuthOptions(jobOptions); 375 jobOptions.addOption(showdiff); 376 377 //Needed to make dryrun and update mutually exclusive options 378 OptionGroup updateOption = new OptionGroup(); 379 updateOption.addOption(dryrun); 380 jobOptions.addOptionGroup(updateOption); 381 return jobOptions; 382 } 383 384 /** 385 * Create option for command line option 'jobs' 386 * @return jobs options 387 */ 388 protected Options createJobsOptions() { 389 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 390 Option start = new Option(OFFSET_OPTION, true, "jobs offset (default '1')"); 391 Option jobtype = new Option(JOBTYPE_OPTION, true, 392 "job type ('Supported in Oozie-2.0 or later versions ONLY - 'coordinator' or 'bundle' or 'wf'(default))"); 393 Option len = new Option(LEN_OPTION, true, "number of jobs (default '100')"); 394 Option filter = new Option(FILTER_OPTION, true, "user=<U>\\;name=<N>\\;group=<G>\\;status=<S>\\;frequency=<F>\\;unit=<M> " + 395 "(Valid unit values are 'months', 'days', 'hours' or 'minutes'.)"); 396 Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" + 397 TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option"); 398 Option timezone = new Option(TIME_ZONE_OPTION, true, 399 "use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list"); 400 Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode"); 401 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 402 Option bulkMonitor = new Option(BULK_OPTION, true, "key-value pairs to filter bulk jobs response. e.g. bundle=<B>\\;" + 403 "coordinators=<C>\\;actionstatus=<S>\\;startcreatedtime=<SC>\\;endcreatedtime=<EC>\\;" + 404 "startscheduledtime=<SS>\\;endscheduledtime=<ES>\\; bundle, coordinators and actionstatus can be multiple comma separated values" + 405 "bundle and coordinators can be id(s) or appName(s) of those jobs. Specifying bundle is mandatory, other params are optional"); 406 start.setType(Integer.class); 407 len.setType(Integer.class); 408 Options jobsOptions = new Options(); 409 jobsOptions.addOption(oozie); 410 jobsOptions.addOption(doAs); 411 jobsOptions.addOption(localtime); 412 jobsOptions.addOption(timezone); 413 jobsOptions.addOption(start); 414 jobsOptions.addOption(len); 415 jobsOptions.addOption(oozie); 416 jobsOptions.addOption(filter); 417 jobsOptions.addOption(jobtype); 418 jobsOptions.addOption(verbose); 419 jobsOptions.addOption(bulkMonitor); 420 addAuthOptions(jobsOptions); 421 return jobsOptions; 422 } 423 424 /** 425 * Create option for command line option 'sla' 426 * 427 * @return sla options 428 */ 429 protected Options createSlaOptions() { 430 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 431 Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')"); 432 Option len = new Option(LEN_OPTION, true, "number of results (default '100', max '1000')"); 433 Option filter = new Option(FILTER_OPTION, true, "filter of SLA events. e.g., jobid=<J>\\;appname=<A>"); 434 start.setType(Integer.class); 435 len.setType(Integer.class); 436 Options slaOptions = new Options(); 437 slaOptions.addOption(start); 438 slaOptions.addOption(len); 439 slaOptions.addOption(filter); 440 slaOptions.addOption(oozie); 441 addAuthOptions(slaOptions); 442 return slaOptions; 443 } 444 445 /** 446 * Create option for command line option 'pig' or 'hive' 447 * @return pig or hive options 448 */ 449 @SuppressWarnings("static-access") 450 protected Options createScriptLanguageOptions(String jobType) { 451 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 452 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'"); 453 Option file = new Option(SCRIPTFILE_OPTION, true, jobType + " script"); 454 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( 455 "set/override value for given property").create("D"); 456 Option params = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( 457 "set parameters for script").create("P"); 458 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 459 Options Options = new Options(); 460 Options.addOption(oozie); 461 Options.addOption(doAs); 462 Options.addOption(config); 463 Options.addOption(property); 464 Options.addOption(params); 465 Options.addOption(file); 466 addAuthOptions(Options); 467 return Options; 468 } 469 470 /** 471 * Create option for command line option 'sqoop' 472 * @return sqoop options 473 */ 474 @SuppressWarnings("static-access") 475 protected Options createSqoopCLIOptions() { 476 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 477 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'"); 478 Option command = OptionBuilder.withArgName(SQOOP_COMMAND_OPTION).hasArgs().withValueSeparator().withDescription( 479 "sqoop command").create(SQOOP_COMMAND_OPTION); 480 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( 481 "set/override value for given property").create("D"); 482 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 483 Options Options = new Options(); 484 Options.addOption(oozie); 485 Options.addOption(doAs); 486 Options.addOption(config); 487 Options.addOption(property); 488 Options.addOption(command); 489 addAuthOptions(Options); 490 return Options; 491 } 492 493 /** 494 * Create option for command line option 'info' 495 * @return info options 496 */ 497 protected Options createInfoOptions() { 498 Option timezones = new Option(INFO_TIME_ZONES_OPTION, false, "display a list of available time zones"); 499 Options infoOptions = new Options(); 500 infoOptions.addOption(timezones); 501 return infoOptions; 502 } 503 504 /** 505 * Create option for command line option 'mapreduce' 506 * @return mapreduce options 507 */ 508 @SuppressWarnings("static-access") 509 protected Options createMROptions() { 510 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); 511 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'"); 512 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( 513 "set/override value for given property").create("D"); 514 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); 515 Options mrOptions = new Options(); 516 mrOptions.addOption(oozie); 517 mrOptions.addOption(doAs); 518 mrOptions.addOption(config); 519 mrOptions.addOption(property); 520 addAuthOptions(mrOptions); 521 return mrOptions; 522 } 523 524 /** 525 * Run a CLI programmatically. 526 * <p/> 527 * It does not exit the JVM. 528 * <p/> 529 * A CLI instance can be used only once. 530 * 531 * @param args options and arguments for the Oozie CLI. 532 * @return '0' (success), '-1' (failure). 533 */ 534 public synchronized int run(String[] args) { 535 if (used) { 536 throw new IllegalStateException("CLI instance already used"); 537 } 538 used = true; 539 final CLIParser parser = getCLIParser(); 540 try { 541 final CLIParser.Command command = parser.parse(args); 542 543 String doAsUser = command.getCommandLine().getOptionValue(DO_AS_OPTION); 544 545 if (doAsUser != null) { 546 OozieClient.doAs(doAsUser, new Callable<Void>() { 547 @Override 548 public Void call() throws Exception { 549 processCommand(parser, command); 550 return null; 551 } 552 }); 553 } 554 else { 555 processCommand(parser, command); 556 } 557 return 0; 558 } 559 catch (OozieCLIException ex) { 560 System.err.println("Error: " + ex.getMessage()); 561 return -1; 562 } 563 catch (ParseException ex) { 564 System.err.println("Invalid sub-command: " + ex.getMessage()); 565 System.err.println(); 566 System.err.println(parser.shortHelp()); 567 return -1; 568 } 569 catch (Exception ex) { 570 ex.printStackTrace(); 571 System.err.println(ex.getMessage()); 572 return -1; 573 } 574 } 575 576 @VisibleForTesting 577 public CLIParser getCLIParser(){ 578 CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp()); 579 parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false); 580 parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false); 581 parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false); 582 parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false); 583 parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false); 584 parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true); 585 parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false); 586 parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' " 587 + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true); 588 parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' " 589 + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true); 590 parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " + 591 "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true); 592 parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false); 593 parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false); 594 return parser; 595 } 596 597 public void processCommand(CLIParser parser, CLIParser.Command command) throws Exception { 598 if (command.getName().equals(HELP_CMD)) { 599 parser.showHelp(command.getCommandLine()); 600 } 601 else if (command.getName().equals(JOB_CMD)) { 602 jobCommand(command.getCommandLine()); 603 } 604 else if (command.getName().equals(JOBS_CMD)) { 605 jobsCommand(command.getCommandLine()); 606 } 607 else if (command.getName().equals(ADMIN_CMD)) { 608 adminCommand(command.getCommandLine()); 609 } 610 else if (command.getName().equals(VERSION_CMD)) { 611 versionCommand(); 612 } 613 else if (command.getName().equals(VALIDATE_CMD)) { 614 validateCommand(command.getCommandLine()); 615 } 616 else if (command.getName().equals(SLA_CMD)) { 617 slaCommand(command.getCommandLine()); 618 } 619 else if (command.getName().equals(PIG_CMD)) { 620 scriptLanguageCommand(command.getCommandLine(), PIG_CMD); 621 } 622 else if (command.getName().equals(HIVE_CMD)) { 623 scriptLanguageCommand(command.getCommandLine(), HIVE_CMD); 624 } 625 else if (command.getName().equals(SQOOP_CMD)) { 626 sqoopCommand(command.getCommandLine()); 627 } 628 else if (command.getName().equals(INFO_CMD)) { 629 infoCommand(command.getCommandLine()); 630 } 631 else if (command.getName().equals(MR_CMD)){ 632 mrCommand(command.getCommandLine()); 633 } 634 } 635 protected String getOozieUrl(CommandLine commandLine) { 636 String url = commandLine.getOptionValue(OOZIE_OPTION); 637 if (url == null) { 638 url = System.getenv(ENV_OOZIE_URL); 639 if (url == null) { 640 throw new IllegalArgumentException( 641 "Oozie URL is not available neither in command option or in the environment"); 642 } 643 } 644 return url; 645 } 646 647 private String getTimeZoneId(CommandLine commandLine) 648 { 649 if (commandLine.hasOption(LOCAL_TIME_OPTION)) { 650 return null; 651 } 652 if (commandLine.hasOption(TIME_ZONE_OPTION)) { 653 return commandLine.getOptionValue(TIME_ZONE_OPTION); 654 } 655 String timeZoneId = System.getenv(ENV_OOZIE_TIME_ZONE); 656 if (timeZoneId != null) { 657 return timeZoneId; 658 } 659 return "GMT"; 660 } 661 662 // Canibalized from Hadoop <code>Configuration.loadResource()</code>. 663 private Properties parse(InputStream is, Properties conf) throws IOException { 664 try { 665 DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); 666 // ignore all comments inside the xml file 667 docBuilderFactory.setIgnoringComments(true); 668 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); 669 Document doc = builder.parse(is); 670 return parseDocument(doc, conf); 671 } 672 catch (SAXException e) { 673 throw new IOException(e); 674 } 675 catch (ParserConfigurationException e) { 676 throw new IOException(e); 677 } 678 } 679 680 // Canibalized from Hadoop <code>Configuration.loadResource()</code>. 681 private Properties parseDocument(Document doc, Properties conf) throws IOException { 682 try { 683 Element root = doc.getDocumentElement(); 684 if (!"configuration".equals(root.getTagName())) { 685 throw new RuntimeException("bad conf file: top-level element not <configuration>"); 686 } 687 NodeList props = root.getChildNodes(); 688 for (int i = 0; i < props.getLength(); i++) { 689 Node propNode = props.item(i); 690 if (!(propNode instanceof Element)) { 691 continue; 692 } 693 Element prop = (Element) propNode; 694 if (!"property".equals(prop.getTagName())) { 695 throw new RuntimeException("bad conf file: element not <property>"); 696 } 697 NodeList fields = prop.getChildNodes(); 698 String attr = null; 699 String value = null; 700 for (int j = 0; j < fields.getLength(); j++) { 701 Node fieldNode = fields.item(j); 702 if (!(fieldNode instanceof Element)) { 703 continue; 704 } 705 Element field = (Element) fieldNode; 706 if ("name".equals(field.getTagName()) && field.hasChildNodes()) { 707 attr = ((Text) field.getFirstChild()).getData(); 708 } 709 if ("value".equals(field.getTagName()) && field.hasChildNodes()) { 710 value = ((Text) field.getFirstChild()).getData(); 711 } 712 } 713 714 if (attr != null && value != null) { 715 conf.setProperty(attr, value); 716 } 717 } 718 return conf; 719 } 720 catch (DOMException e) { 721 throw new IOException(e); 722 } 723 } 724 725 private Properties getConfiguration(OozieClient wc, CommandLine commandLine) throws IOException { 726 if (!isConfigurationSpecified(wc, commandLine)) { 727 throw new IOException("configuration is not specified"); 728 } 729 Properties conf = wc.createConfiguration(); 730 String configFile = commandLine.getOptionValue(CONFIG_OPTION); 731 if (configFile != null) { 732 File file = new File(configFile); 733 if (!file.exists()) { 734 throw new IOException("configuration file [" + configFile + "] not found"); 735 } 736 if (configFile.endsWith(".properties")) { 737 conf.load(new FileReader(file)); 738 } 739 else if (configFile.endsWith(".xml")) { 740 parse(new FileInputStream(configFile), conf); 741 } 742 else { 743 throw new IllegalArgumentException("configuration must be a '.properties' or a '.xml' file"); 744 } 745 } 746 if (commandLine.hasOption("D")) { 747 Properties commandLineProperties = commandLine.getOptionProperties("D"); 748 conf.putAll(commandLineProperties); 749 } 750 return conf; 751 } 752 753 /** 754 * Check if configuration has specified 755 * @param wc 756 * @param commandLine 757 * @return 758 * @throws IOException 759 */ 760 private boolean isConfigurationSpecified(OozieClient wc, CommandLine commandLine) throws IOException { 761 boolean isConf = false; 762 String configFile = commandLine.getOptionValue(CONFIG_OPTION); 763 if (configFile == null) { 764 isConf = false; 765 } 766 else { 767 isConf = new File(configFile).exists(); 768 } 769 if (commandLine.hasOption("D")) { 770 isConf = true; 771 } 772 return isConf; 773 } 774 775 /** 776 * @param commandLine command line string. 777 * @return change value specified by -value. 778 * @throws OozieCLIException 779 */ 780 private String getChangeValue(CommandLine commandLine) throws OozieCLIException { 781 String changeValue = commandLine.getOptionValue(CHANGE_VALUE_OPTION); 782 783 if (changeValue == null) { 784 throw new OozieCLIException("-value option needs to be specified for -change option"); 785 } 786 787 return changeValue; 788 } 789 790 protected void addHeader(OozieClient wc) { 791 for (Map.Entry entry : System.getProperties().entrySet()) { 792 String key = (String) entry.getKey(); 793 if (key.startsWith(WS_HEADER_PREFIX)) { 794 String header = key.substring(WS_HEADER_PREFIX.length()); 795 wc.setHeader(header, (String) entry.getValue()); 796 } 797 } 798 } 799 800 /** 801 * Get auth option from command line 802 * 803 * @param commandLine the command line object 804 * @return auth option 805 */ 806 protected String getAuthOption(CommandLine commandLine) { 807 String authOpt = commandLine.getOptionValue(AUTH_OPTION); 808 if (authOpt == null) { 809 authOpt = System.getenv(ENV_OOZIE_AUTH); 810 } 811 if (commandLine.hasOption(DEBUG_OPTION)) { 812 System.out.println(" Auth type : " + authOpt); 813 } 814 return authOpt; 815 } 816 817 /** 818 * Create a OozieClient. 819 * <p/> 820 * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}. 821 * 822 * @param commandLine the parsed command line options. 823 * @return a pre configured eXtended workflow client. 824 * @throws OozieCLIException thrown if the OozieClient could not be configured. 825 */ 826 protected OozieClient createOozieClient(CommandLine commandLine) throws OozieCLIException { 827 return createXOozieClient(commandLine); 828 } 829 830 /** 831 * Create a XOozieClient. 832 * <p/> 833 * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}. 834 * 835 * @param commandLine the parsed command line options. 836 * @return a pre configured eXtended workflow client. 837 * @throws OozieCLIException thrown if the XOozieClient could not be configured. 838 */ 839 protected XOozieClient createXOozieClient(CommandLine commandLine) throws OozieCLIException { 840 XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine)); 841 addHeader(wc); 842 setDebugMode(wc,commandLine.hasOption(DEBUG_OPTION)); 843 setRetryCount(wc); 844 return wc; 845 } 846 847 protected void setDebugMode(OozieClient wc, boolean debugOpt) { 848 849 String debug = System.getenv(ENV_OOZIE_DEBUG); 850 if (debug != null && !debug.isEmpty()) { 851 int debugVal = 0; 852 try { 853 debugVal = Integer.parseInt(debug.trim()); 854 } 855 catch (Exception ex) { 856 System.out.println("Unable to parse the debug settings. May be not an integer [" + debug + "]"); 857 ex.printStackTrace(); 858 } 859 wc.setDebugMode(debugVal); 860 } 861 else if(debugOpt){ // CLI argument "-debug" used 862 wc.setDebugMode(1); 863 } 864 } 865 866 protected void setRetryCount(OozieClient wc) { 867 String retryCount = System.getProperty(OOZIE_RETRY_COUNT); 868 if (retryCount != null && !retryCount.isEmpty()) { 869 try { 870 int retry = Integer.parseInt(retryCount.trim()); 871 wc.setRetryCount(retry); 872 } 873 catch (Exception ex) { 874 System.err.println("Unable to parse the retry settings. May be not an integer [" + retryCount + "]"); 875 ex.printStackTrace(); 876 } 877 } 878 } 879 880 private static String JOB_ID_PREFIX = "job: "; 881 882 private void jobCommand(CommandLine commandLine) throws IOException, OozieCLIException { 883 XOozieClient wc = createXOozieClient(commandLine); 884 885 List<String> options = new ArrayList<String>(); 886 for (Option option : commandLine.getOptions()) { 887 options.add(option.getOpt()); 888 } 889 890 try { 891 if (options.contains(SUBMIT_OPTION)) { 892 System.out.println(JOB_ID_PREFIX + wc.submit(getConfiguration(wc, commandLine))); 893 } 894 else if (options.contains(START_OPTION)) { 895 wc.start(commandLine.getOptionValue(START_OPTION)); 896 } 897 else if (options.contains(DRYRUN_OPTION) && !options.contains(UPDATE_OPTION)) { 898 String dryrunStr = wc.dryrun(getConfiguration(wc, commandLine)); 899 if (dryrunStr.equals("OK")) { // workflow 900 System.out.println("OK"); 901 } else { // coordinator 902 String[] dryrunStrs = dryrunStr.split("action for new instance"); 903 int arraysize = dryrunStrs.length; 904 System.out.println("***coordJob after parsing: ***"); 905 System.out.println(dryrunStrs[0]); 906 int aLen = dryrunStrs.length - 1; 907 if (aLen < 0) { 908 aLen = 0; 909 } 910 System.out.println("***total coord actions is " + aLen + " ***"); 911 for (int i = 1; i <= arraysize - 1; i++) { 912 System.out.println(RULER); 913 System.out.println("coordAction instance: " + i + ":"); 914 System.out.println(dryrunStrs[i]); 915 } 916 } 917 } 918 else if (options.contains(SUSPEND_OPTION)) { 919 wc.suspend(commandLine.getOptionValue(SUSPEND_OPTION)); 920 } 921 else if (options.contains(RESUME_OPTION)) { 922 wc.resume(commandLine.getOptionValue(RESUME_OPTION)); 923 } 924 else if (options.contains(IGNORE_OPTION)) { 925 String ignoreScope = null; 926 if (options.contains(ACTION_OPTION)) { 927 ignoreScope = commandLine.getOptionValue(ACTION_OPTION); 928 if (ignoreScope == null || ignoreScope.isEmpty()) { 929 throw new OozieCLIException("-" + ACTION_OPTION + " is empty"); 930 } 931 } 932 printCoordActionsStatus(wc.ignore(commandLine.getOptionValue(IGNORE_OPTION), ignoreScope)); 933 } 934 else if (options.contains(KILL_OPTION)) { 935 if (commandLine.getOptionValue(KILL_OPTION).contains("-C") 936 && (options.contains(DATE_OPTION) || options.contains(ACTION_OPTION))) { 937 String coordJobId = commandLine.getOptionValue(KILL_OPTION); 938 String scope = null; 939 String rangeType = null; 940 if (options.contains(DATE_OPTION) && options.contains(ACTION_OPTION)) { 941 throw new OozieCLIException("Invalid options provided for rerun: either" + DATE_OPTION + " or " 942 + ACTION_OPTION + " expected. Don't use both at the same time."); 943 } 944 if (options.contains(DATE_OPTION)) { 945 rangeType = RestConstants.JOB_COORD_SCOPE_DATE; 946 scope = commandLine.getOptionValue(DATE_OPTION); 947 } 948 else if (options.contains(ACTION_OPTION)) { 949 rangeType = RestConstants.JOB_COORD_SCOPE_ACTION; 950 scope = commandLine.getOptionValue(ACTION_OPTION); 951 } 952 else { 953 throw new OozieCLIException("Invalid options provided for rerun: " + DATE_OPTION + " or " 954 + ACTION_OPTION + " expected."); 955 } 956 printCoordActions(wc.kill(coordJobId, rangeType, scope)); 957 } 958 else { 959 wc.kill(commandLine.getOptionValue(KILL_OPTION)); 960 } 961 } 962 else if (options.contains(CHANGE_OPTION)) { 963 wc.change(commandLine.getOptionValue(CHANGE_OPTION), getChangeValue(commandLine)); 964 } 965 else if (options.contains(RUN_OPTION)) { 966 System.out.println(JOB_ID_PREFIX + wc.run(getConfiguration(wc, commandLine))); 967 } 968 else if (options.contains(RERUN_OPTION)) { 969 if (commandLine.getOptionValue(RERUN_OPTION).contains("-W")) { 970 if (isConfigurationSpecified(wc, commandLine)) { 971 wc.reRun(commandLine.getOptionValue(RERUN_OPTION), getConfiguration(wc, commandLine)); 972 } 973 else { 974 wc.reRun(commandLine.getOptionValue(RERUN_OPTION), new Properties()); 975 } 976 } 977 else if (commandLine.getOptionValue(RERUN_OPTION).contains("-B")) { 978 String bundleJobId = commandLine.getOptionValue(RERUN_OPTION); 979 String coordScope = null; 980 String dateScope = null; 981 boolean refresh = false; 982 boolean noCleanup = false; 983 if (options.contains(ACTION_OPTION)) { 984 throw new OozieCLIException("Invalid options provided for bundle rerun. " + ACTION_OPTION 985 + " is not valid for bundle rerun"); 986 } 987 if (options.contains(DATE_OPTION)) { 988 dateScope = commandLine.getOptionValue(DATE_OPTION); 989 } 990 991 if (options.contains(RERUN_COORD_OPTION)) { 992 coordScope = commandLine.getOptionValue(RERUN_COORD_OPTION); 993 } 994 995 if (options.contains(RERUN_REFRESH_OPTION)) { 996 refresh = true; 997 } 998 if (options.contains(RERUN_NOCLEANUP_OPTION)) { 999 noCleanup = true; 1000 } 1001 wc.reRunBundle(bundleJobId, coordScope, dateScope, refresh, noCleanup); 1002 if (coordScope != null && !coordScope.isEmpty()) { 1003 System.out.println("Coordinators [" + coordScope + "] of bundle " + bundleJobId 1004 + " are scheduled to rerun on date ranges [" + dateScope + "]."); 1005 } 1006 else { 1007 System.out.println("All coordinators of bundle " + bundleJobId 1008 + " are scheduled to rerun on the date ranges [" + dateScope + "]."); 1009 } 1010 } 1011 else { 1012 String coordJobId = commandLine.getOptionValue(RERUN_OPTION); 1013 String scope = null; 1014 String rerunType = null; 1015 boolean refresh = false; 1016 boolean noCleanup = false; 1017 if (options.contains(DATE_OPTION) && options.contains(ACTION_OPTION)) { 1018 throw new OozieCLIException("Invalid options provided for rerun: either" + DATE_OPTION + " or " 1019 + ACTION_OPTION + " expected. Don't use both at the same time."); 1020 } 1021 if (options.contains(DATE_OPTION)) { 1022 rerunType = RestConstants.JOB_COORD_SCOPE_DATE; 1023 scope = commandLine.getOptionValue(DATE_OPTION); 1024 } 1025 else if (options.contains(ACTION_OPTION)) { 1026 rerunType = RestConstants.JOB_COORD_SCOPE_ACTION; 1027 scope = commandLine.getOptionValue(ACTION_OPTION); 1028 } 1029 else { 1030 throw new OozieCLIException("Invalid options provided for rerun: " + DATE_OPTION + " or " 1031 + ACTION_OPTION + " expected."); 1032 } 1033 if (options.contains(RERUN_REFRESH_OPTION)) { 1034 refresh = true; 1035 } 1036 if (options.contains(RERUN_NOCLEANUP_OPTION)) { 1037 noCleanup = true; 1038 } 1039 printCoordActions(wc.reRunCoord(coordJobId, rerunType, scope, refresh, noCleanup)); 1040 } 1041 } 1042 else if (options.contains(INFO_OPTION)) { 1043 String timeZoneId = getTimeZoneId(commandLine); 1044 final String optionValue = commandLine.getOptionValue(INFO_OPTION); 1045 if (optionValue.endsWith("-B")) { 1046 String filter = commandLine.getOptionValue(FILTER_OPTION); 1047 if (filter != null) { 1048 throw new OozieCLIException("Filter option is currently not supported for a Bundle job"); 1049 } 1050 printBundleJob(wc.getBundleJobInfo(optionValue), timeZoneId, 1051 options.contains(VERBOSE_OPTION)); 1052 } 1053 else if (optionValue.endsWith("-C")) { 1054 String s = commandLine.getOptionValue(OFFSET_OPTION); 1055 int start = Integer.parseInt((s != null) ? s : "-1"); 1056 s = commandLine.getOptionValue(LEN_OPTION); 1057 int len = Integer.parseInt((s != null) ? s : "-1"); 1058 String filter = commandLine.getOptionValue(FILTER_OPTION); 1059 String order = commandLine.getOptionValue(ORDER_OPTION); 1060 printCoordJob(wc.getCoordJobInfo(optionValue, filter, start, len, order), timeZoneId, 1061 options.contains(VERBOSE_OPTION)); 1062 } 1063 else if (optionValue.contains("-C@")) { 1064 if (options.contains(ALL_WORKFLOWS_FOR_COORD_ACTION)) { 1065 printWfsForCoordAction(wc.getWfsForCoordAction(optionValue), timeZoneId); 1066 } 1067 else { 1068 String filter = commandLine.getOptionValue(FILTER_OPTION); 1069 if (filter != null) { 1070 throw new OozieCLIException("Filter option is not supported for a Coordinator action"); 1071 } 1072 printCoordAction(wc.getCoordActionInfo(optionValue), timeZoneId); 1073 } 1074 } 1075 else if (optionValue.contains("-W@")) { 1076 String filter = commandLine.getOptionValue(FILTER_OPTION); 1077 if (filter != null) { 1078 throw new OozieCLIException("Filter option is not supported for a Workflow action"); 1079 } 1080 printWorkflowAction(wc.getWorkflowActionInfo(optionValue), timeZoneId, 1081 options.contains(VERBOSE_OPTION)); 1082 } 1083 else { 1084 String filter = commandLine.getOptionValue(FILTER_OPTION); 1085 if (filter != null) { 1086 throw new OozieCLIException("Filter option is currently not supported for a Workflow job"); 1087 } 1088 String s = commandLine.getOptionValue(OFFSET_OPTION); 1089 int start = Integer.parseInt((s != null) ? s : "0"); 1090 s = commandLine.getOptionValue(LEN_OPTION); 1091 String jobtype = commandLine.getOptionValue(JOBTYPE_OPTION); 1092 jobtype = (jobtype != null) ? jobtype : "wf"; 1093 int len = Integer.parseInt((s != null) ? s : "0"); 1094 printJob(wc.getJobInfo(optionValue, start, len), timeZoneId, 1095 options.contains(VERBOSE_OPTION)); 1096 } 1097 } 1098 else if (options.contains(LOG_OPTION)) { 1099 PrintStream ps = System.out; 1100 String logFilter = null; 1101 if (options.contains(RestConstants.LOG_FILTER_OPTION)) { 1102 logFilter = commandLine.getOptionValue(RestConstants.LOG_FILTER_OPTION); 1103 } 1104 if (commandLine.getOptionValue(LOG_OPTION).contains("-C")) { 1105 String logRetrievalScope = null; 1106 String logRetrievalType = null; 1107 if (options.contains(ACTION_OPTION)) { 1108 logRetrievalType = RestConstants.JOB_LOG_ACTION; 1109 logRetrievalScope = commandLine.getOptionValue(ACTION_OPTION); 1110 } 1111 if (options.contains(DATE_OPTION)) { 1112 logRetrievalType = RestConstants.JOB_LOG_DATE; 1113 logRetrievalScope = commandLine.getOptionValue(DATE_OPTION); 1114 } 1115 try { 1116 wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), logRetrievalType, logRetrievalScope, 1117 logFilter, ps); 1118 } 1119 finally { 1120 ps.close(); 1121 } 1122 } 1123 else { 1124 if (!options.contains(ACTION_OPTION) && !options.contains(DATE_OPTION)) { 1125 wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), null, null, logFilter, ps); 1126 } 1127 else { 1128 throw new OozieCLIException("Invalid options provided for log retrieval. " + ACTION_OPTION 1129 + " and " + DATE_OPTION + " are valid only for coordinator job log retrieval"); 1130 } 1131 } 1132 } 1133 else if (options.contains(DEFINITION_OPTION)) { 1134 System.out.println(wc.getJobDefinition(commandLine.getOptionValue(DEFINITION_OPTION))); 1135 } 1136 else if (options.contains(CONFIG_CONTENT_OPTION)) { 1137 if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-C")) { 1138 System.out.println(wc.getCoordJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf()); 1139 } 1140 else if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-W")) { 1141 System.out.println(wc.getJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf()); 1142 } 1143 else if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-B")) { 1144 System.out 1145 .println(wc.getBundleJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf()); 1146 } 1147 else { 1148 System.out.println("ERROR: job id [" + commandLine.getOptionValue(CONFIG_CONTENT_OPTION) 1149 + "] doesn't end with either C or W or B"); 1150 } 1151 } 1152 else if (options.contains(UPDATE_OPTION)) { 1153 String coordJobId = commandLine.getOptionValue(UPDATE_OPTION); 1154 Properties conf = null; 1155 1156 String dryrun = ""; 1157 String showdiff = ""; 1158 1159 if (commandLine.getOptionValue(CONFIG_OPTION) != null) { 1160 conf = getConfiguration(wc, commandLine); 1161 } 1162 if (options.contains(DRYRUN_OPTION)) { 1163 dryrun = "true"; 1164 } 1165 if (commandLine.getOptionValue(SHOWDIFF_OPTION) != null) { 1166 showdiff = commandLine.getOptionValue(SHOWDIFF_OPTION); 1167 } 1168 if (conf == null) { 1169 System.out.println(wc.updateCoord(coordJobId, dryrun, showdiff)); 1170 } 1171 else { 1172 System.out.println(wc.updateCoord(coordJobId, conf, dryrun, showdiff)); 1173 } 1174 } 1175 } 1176 catch (OozieClientException ex) { 1177 throw new OozieCLIException(ex.toString(), ex); 1178 } 1179 } 1180 1181 @VisibleForTesting 1182 void printCoordJob(CoordinatorJob coordJob, String timeZoneId, boolean verbose) { 1183 System.out.println("Job ID : " + coordJob.getId()); 1184 1185 System.out.println(RULER); 1186 1187 List<CoordinatorAction> actions = coordJob.getActions(); 1188 System.out.println("Job Name : " + maskIfNull(coordJob.getAppName())); 1189 System.out.println("App Path : " + maskIfNull(coordJob.getAppPath())); 1190 System.out.println("Status : " + coordJob.getStatus()); 1191 System.out.println("Start Time : " + maskDate(coordJob.getStartTime(), timeZoneId, false)); 1192 System.out.println("End Time : " + maskDate(coordJob.getEndTime(), timeZoneId, false)); 1193 System.out.println("Pause Time : " + maskDate(coordJob.getPauseTime(), timeZoneId, false)); 1194 System.out.println("Concurrency : " + coordJob.getConcurrency()); 1195 System.out.println(RULER); 1196 1197 if (verbose) { 1198 System.out.println("ID" + VERBOSE_DELIMITER + "Action Number" + VERBOSE_DELIMITER + "Console URL" 1199 + VERBOSE_DELIMITER + "Error Code" + VERBOSE_DELIMITER + "Error Message" + VERBOSE_DELIMITER 1200 + "External ID" + VERBOSE_DELIMITER + "External Status" + VERBOSE_DELIMITER + "Job ID" 1201 + VERBOSE_DELIMITER + "Tracker URI" + VERBOSE_DELIMITER + "Created" + VERBOSE_DELIMITER 1202 + "Nominal Time" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified" 1203 + VERBOSE_DELIMITER + "Missing Dependencies"); 1204 System.out.println(RULER); 1205 1206 for (CoordinatorAction action : actions) { 1207 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER + action.getActionNumber() 1208 + VERBOSE_DELIMITER + maskIfNull(action.getConsoleUrl()) + VERBOSE_DELIMITER 1209 + maskIfNull(action.getErrorCode()) + VERBOSE_DELIMITER + maskIfNull(action.getErrorMessage()) 1210 + VERBOSE_DELIMITER + maskIfNull(action.getExternalId()) + VERBOSE_DELIMITER 1211 + maskIfNull(action.getExternalStatus()) + VERBOSE_DELIMITER + maskIfNull(action.getJobId()) 1212 + VERBOSE_DELIMITER + maskIfNull(action.getTrackerUri()) + VERBOSE_DELIMITER 1213 + maskDate(action.getCreatedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1214 + maskDate(action.getNominalTime(), timeZoneId, verbose) + action.getStatus() + VERBOSE_DELIMITER 1215 + maskDate(action.getLastModifiedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1216 + maskIfNull(getFirstMissingDependencies(action))); 1217 1218 System.out.println(RULER); 1219 } 1220 } 1221 else { 1222 System.out.println(String.format(COORD_ACTION_FORMATTER, "ID", "Status", "Ext ID", "Err Code", "Created", 1223 "Nominal Time", "Last Mod")); 1224 1225 for (CoordinatorAction action : actions) { 1226 System.out.println(String.format(COORD_ACTION_FORMATTER, maskIfNull(action.getId()), 1227 action.getStatus(), maskIfNull(action.getExternalId()), maskIfNull(action.getErrorCode()), 1228 maskDate(action.getCreatedTime(), timeZoneId, verbose), maskDate(action.getNominalTime(), timeZoneId, verbose), 1229 maskDate(action.getLastModifiedTime(), timeZoneId, verbose))); 1230 1231 System.out.println(RULER); 1232 } 1233 } 1234 } 1235 1236 @VisibleForTesting 1237 void printBundleJob(BundleJob bundleJob, String timeZoneId, boolean verbose) { 1238 System.out.println("Job ID : " + bundleJob.getId()); 1239 1240 System.out.println(RULER); 1241 1242 List<CoordinatorJob> coordinators = bundleJob.getCoordinators(); 1243 System.out.println("Job Name : " + maskIfNull(bundleJob.getAppName())); 1244 System.out.println("App Path : " + maskIfNull(bundleJob.getAppPath())); 1245 System.out.println("Status : " + bundleJob.getStatus()); 1246 System.out.println("Kickoff time : " + bundleJob.getKickoffTime()); 1247 System.out.println(RULER); 1248 1249 System.out.println(String.format(BUNDLE_COORD_JOBS_FORMATTER, "Job ID", "Status", "Freq", "Unit", "Started", 1250 "Next Materialized")); 1251 System.out.println(RULER); 1252 1253 for (CoordinatorJob job : coordinators) { 1254 System.out.println(String.format(BUNDLE_COORD_JOBS_FORMATTER, maskIfNull(job.getId()), job.getStatus(), 1255 job.getFrequency(), job.getTimeUnit(), maskDate(job.getStartTime(), timeZoneId, verbose), 1256 maskDate(job.getNextMaterializedTime(), timeZoneId, verbose))); 1257 1258 System.out.println(RULER); 1259 } 1260 } 1261 1262 @VisibleForTesting 1263 void printCoordAction(CoordinatorAction coordAction, String timeZoneId) { 1264 System.out.println("ID : " + maskIfNull(coordAction.getId())); 1265 1266 System.out.println(RULER); 1267 1268 System.out.println("Action Number : " + coordAction.getActionNumber()); 1269 System.out.println("Console URL : " + maskIfNull(coordAction.getConsoleUrl())); 1270 System.out.println("Error Code : " + maskIfNull(coordAction.getErrorCode())); 1271 System.out.println("Error Message : " + maskIfNull(coordAction.getErrorMessage())); 1272 System.out.println("External ID : " + maskIfNull(coordAction.getExternalId())); 1273 System.out.println("External Status : " + maskIfNull(coordAction.getExternalStatus())); 1274 System.out.println("Job ID : " + maskIfNull(coordAction.getJobId())); 1275 System.out.println("Tracker URI : " + maskIfNull(coordAction.getTrackerUri())); 1276 System.out.println("Created : " + maskDate(coordAction.getCreatedTime(), timeZoneId, false)); 1277 System.out.println("Nominal Time : " + maskDate(coordAction.getNominalTime(), timeZoneId, false)); 1278 System.out.println("Status : " + coordAction.getStatus()); 1279 System.out.println("Last Modified : " + maskDate(coordAction.getLastModifiedTime(), timeZoneId, false)); 1280 System.out.println("First Missing Dependency : " + maskIfNull(getFirstMissingDependencies(coordAction))); 1281 1282 System.out.println(RULER); 1283 } 1284 1285 private void printCoordActions(List<CoordinatorAction> actions) { 1286 if (actions != null && actions.size() > 0) { 1287 System.out.println("Action ID" + VERBOSE_DELIMITER + "Nominal Time"); 1288 System.out.println(RULER); 1289 for (CoordinatorAction action : actions) { 1290 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER 1291 + maskDate(action.getNominalTime(), null,false)); 1292 } 1293 } 1294 else { 1295 System.out.println("No Actions match your criteria!"); 1296 } 1297 } 1298 1299 private void printCoordActionsStatus(List<CoordinatorAction> actions) { 1300 if (actions != null && actions.size() > 0) { 1301 System.out.println("Action ID" + VERBOSE_DELIMITER + "Nominal Time" + VERBOSE_DELIMITER + "Status"); 1302 System.out.println(RULER); 1303 for (CoordinatorAction action : actions) { 1304 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER 1305 + maskDate(action.getNominalTime(), null, false) + VERBOSE_DELIMITER 1306 + maskIfNull(action.getStatus().name())); 1307 } 1308 } 1309 } 1310 1311 @VisibleForTesting 1312 void printWorkflowAction(WorkflowAction action, String timeZoneId, boolean verbose) { 1313 1314 System.out.println("ID : " + maskIfNull(action.getId())); 1315 1316 System.out.println(RULER); 1317 1318 System.out.println("Console URL : " + maskIfNull(action.getConsoleUrl())); 1319 System.out.println("Error Code : " + maskIfNull(action.getErrorCode())); 1320 System.out.println("Error Message : " + maskIfNull(action.getErrorMessage())); 1321 System.out.println("External ID : " + maskIfNull(action.getExternalId())); 1322 System.out.println("External Status : " + maskIfNull(action.getExternalStatus())); 1323 System.out.println("Name : " + maskIfNull(action.getName())); 1324 System.out.println("Retries : " + action.getRetries()); 1325 System.out.println("Tracker URI : " + maskIfNull(action.getTrackerUri())); 1326 System.out.println("Type : " + maskIfNull(action.getType())); 1327 System.out.println("Started : " + maskDate(action.getStartTime(), timeZoneId, verbose)); 1328 System.out.println("Status : " + action.getStatus()); 1329 System.out.println("Ended : " + maskDate(action.getEndTime(), timeZoneId, verbose)); 1330 1331 if (verbose) { 1332 System.out.println("External Stats : " + action.getStats()); 1333 System.out.println("External ChildIDs : " + action.getExternalChildIDs()); 1334 } 1335 1336 System.out.println(RULER); 1337 } 1338 1339 private static final String WORKFLOW_JOBS_FORMATTER = "%-41s%-13s%-10s%-10s%-10s%-24s%-24s"; 1340 private static final String COORD_JOBS_FORMATTER = "%-41s%-15s%-10s%-5s%-13s%-24s%-24s"; 1341 private static final String BUNDLE_JOBS_FORMATTER = "%-41s%-15s%-10s%-20s%-20s%-13s%-13s"; 1342 private static final String BUNDLE_COORD_JOBS_FORMATTER = "%-41s%-15s%-5s%-13s%-24s%-24s"; 1343 1344 private static final String WORKFLOW_ACTION_FORMATTER = "%-78s%-10s%-23s%-11s%-10s"; 1345 private static final String COORD_ACTION_FORMATTER = "%-43s%-10s%-37s%-10s%-21s%-21s"; 1346 private static final String BULK_RESPONSE_FORMATTER = "%-13s%-38s%-13s%-41s%-10s%-38s%-21s%-38s"; 1347 1348 @VisibleForTesting 1349 void printJob(WorkflowJob job, String timeZoneId, boolean verbose) throws IOException { 1350 System.out.println("Job ID : " + maskIfNull(job.getId())); 1351 1352 System.out.println(RULER); 1353 1354 System.out.println("Workflow Name : " + maskIfNull(job.getAppName())); 1355 System.out.println("App Path : " + maskIfNull(job.getAppPath())); 1356 System.out.println("Status : " + job.getStatus()); 1357 System.out.println("Run : " + job.getRun()); 1358 System.out.println("User : " + maskIfNull(job.getUser())); 1359 System.out.println("Group : " + maskIfNull(job.getGroup())); 1360 System.out.println("Created : " + maskDate(job.getCreatedTime(), timeZoneId, verbose)); 1361 System.out.println("Started : " + maskDate(job.getStartTime(), timeZoneId, verbose)); 1362 System.out.println("Last Modified : " + maskDate(job.getLastModifiedTime(), timeZoneId, verbose)); 1363 System.out.println("Ended : " + maskDate(job.getEndTime(), timeZoneId, verbose)); 1364 System.out.println("CoordAction ID: " + maskIfNull(job.getParentId())); 1365 1366 List<WorkflowAction> actions = job.getActions(); 1367 1368 if (actions != null && actions.size() > 0) { 1369 System.out.println(); 1370 System.out.println("Actions"); 1371 System.out.println(RULER); 1372 1373 if (verbose) { 1374 System.out.println("ID" + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "Error Code" 1375 + VERBOSE_DELIMITER + "Error Message" + VERBOSE_DELIMITER + "External ID" + VERBOSE_DELIMITER 1376 + "External Status" + VERBOSE_DELIMITER + "Name" + VERBOSE_DELIMITER + "Retries" 1377 + VERBOSE_DELIMITER + "Tracker URI" + VERBOSE_DELIMITER + "Type" + VERBOSE_DELIMITER 1378 + "Started" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Ended"); 1379 System.out.println(RULER); 1380 1381 for (WorkflowAction action : job.getActions()) { 1382 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER 1383 + maskIfNull(action.getConsoleUrl()) + VERBOSE_DELIMITER 1384 + maskIfNull(action.getErrorCode()) + VERBOSE_DELIMITER 1385 + maskIfNull(action.getErrorMessage()) + VERBOSE_DELIMITER 1386 + maskIfNull(action.getExternalId()) + VERBOSE_DELIMITER 1387 + maskIfNull(action.getExternalStatus()) + VERBOSE_DELIMITER + maskIfNull(action.getName()) 1388 + VERBOSE_DELIMITER + action.getRetries() + VERBOSE_DELIMITER 1389 + maskIfNull(action.getTrackerUri()) + VERBOSE_DELIMITER + maskIfNull(action.getType()) 1390 + VERBOSE_DELIMITER + maskDate(action.getStartTime(), timeZoneId, verbose) 1391 + VERBOSE_DELIMITER + action.getStatus() + VERBOSE_DELIMITER 1392 + maskDate(action.getEndTime(), timeZoneId, verbose)); 1393 1394 System.out.println(RULER); 1395 } 1396 } 1397 else { 1398 System.out.println(String.format(WORKFLOW_ACTION_FORMATTER, "ID", "Status", "Ext ID", "Ext Status", 1399 "Err Code")); 1400 1401 System.out.println(RULER); 1402 1403 for (WorkflowAction action : job.getActions()) { 1404 System.out.println(String.format(WORKFLOW_ACTION_FORMATTER, maskIfNull(action.getId()), action 1405 .getStatus(), maskIfNull(action.getExternalId()), maskIfNull(action.getExternalStatus()), 1406 maskIfNull(action.getErrorCode()))); 1407 1408 System.out.println(RULER); 1409 } 1410 } 1411 } 1412 else { 1413 System.out.println(RULER); 1414 } 1415 1416 System.out.println(); 1417 } 1418 1419 private void jobsCommand(CommandLine commandLine) throws IOException, OozieCLIException { 1420 XOozieClient wc = createXOozieClient(commandLine); 1421 1422 String filter = commandLine.getOptionValue(FILTER_OPTION); 1423 String s = commandLine.getOptionValue(OFFSET_OPTION); 1424 int start = Integer.parseInt((s != null) ? s : "0"); 1425 s = commandLine.getOptionValue(LEN_OPTION); 1426 String jobtype = commandLine.getOptionValue(JOBTYPE_OPTION); 1427 String timeZoneId = getTimeZoneId(commandLine); 1428 jobtype = (jobtype != null) ? jobtype : "wf"; 1429 int len = Integer.parseInt((s != null) ? s : "0"); 1430 String bulkFilterString = commandLine.getOptionValue(BULK_OPTION); 1431 1432 try { 1433 if (bulkFilterString != null) { 1434 printBulkJobs(wc.getBulkInfo(bulkFilterString, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION)); 1435 } 1436 else if (jobtype.toLowerCase().contains("wf")) { 1437 printJobs(wc.getJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION)); 1438 } 1439 else if (jobtype.toLowerCase().startsWith("coord")) { 1440 printCoordJobs(wc.getCoordJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION)); 1441 } 1442 else if (jobtype.toLowerCase().startsWith("bundle")) { 1443 printBundleJobs(wc.getBundleJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION)); 1444 } 1445 1446 } 1447 catch (OozieClientException ex) { 1448 throw new OozieCLIException(ex.toString(), ex); 1449 } 1450 } 1451 1452 @VisibleForTesting 1453 void printCoordJobs(List<CoordinatorJob> jobs, String timeZoneId, boolean verbose) throws IOException { 1454 if (jobs != null && jobs.size() > 0) { 1455 if (verbose) { 1456 System.out.println("Job ID" + VERBOSE_DELIMITER + "App Name" + VERBOSE_DELIMITER + "App Path" 1457 + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group" 1458 + VERBOSE_DELIMITER + "Concurrency" + VERBOSE_DELIMITER + "Frequency" + VERBOSE_DELIMITER 1459 + "Time Unit" + VERBOSE_DELIMITER + "Time Zone" + VERBOSE_DELIMITER + "Time Out" 1460 + VERBOSE_DELIMITER + "Started" + VERBOSE_DELIMITER + "Next Materialize" + VERBOSE_DELIMITER 1461 + "Status" + VERBOSE_DELIMITER + "Last Action" + VERBOSE_DELIMITER + "Ended"); 1462 System.out.println(RULER); 1463 1464 for (CoordinatorJob job : jobs) { 1465 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName()) 1466 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER 1467 + maskIfNull(job.getConsoleUrl()) + VERBOSE_DELIMITER + maskIfNull(job.getUser()) 1468 + VERBOSE_DELIMITER + maskIfNull(job.getGroup()) + VERBOSE_DELIMITER + job.getConcurrency() 1469 + VERBOSE_DELIMITER + job.getFrequency() + VERBOSE_DELIMITER + job.getTimeUnit() 1470 + VERBOSE_DELIMITER + maskIfNull(job.getTimeZone()) + VERBOSE_DELIMITER + job.getTimeout() 1471 + VERBOSE_DELIMITER + maskDate(job.getStartTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1472 + maskDate(job.getNextMaterializedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1473 + job.getStatus() + VERBOSE_DELIMITER 1474 + maskDate(job.getLastActionTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1475 + maskDate(job.getEndTime(), timeZoneId, verbose)); 1476 1477 System.out.println(RULER); 1478 } 1479 } 1480 else { 1481 System.out.println(String.format(COORD_JOBS_FORMATTER, "Job ID", "App Name", "Status", "Freq", "Unit", 1482 "Started", "Next Materialized")); 1483 System.out.println(RULER); 1484 1485 for (CoordinatorJob job : jobs) { 1486 System.out.println(String.format(COORD_JOBS_FORMATTER, maskIfNull(job.getId()), maskIfNull(job 1487 .getAppName()), job.getStatus(), job.getFrequency(), job.getTimeUnit(), maskDate(job 1488 .getStartTime(), timeZoneId, verbose), maskDate(job.getNextMaterializedTime(), timeZoneId, verbose))); 1489 1490 System.out.println(RULER); 1491 } 1492 } 1493 } 1494 else { 1495 System.out.println("No Jobs match your criteria!"); 1496 } 1497 } 1498 1499 @VisibleForTesting 1500 void printBulkJobs(List<BulkResponse> jobs, String timeZoneId, boolean verbose) throws IOException { 1501 if (jobs != null && jobs.size() > 0) { 1502 for (BulkResponse response : jobs) { 1503 BundleJob bundle = response.getBundle(); 1504 CoordinatorJob coord = response.getCoordinator(); 1505 CoordinatorAction action = response.getAction(); 1506 if (verbose) { 1507 System.out.println(); 1508 System.out.println("Bundle Name : " + maskIfNull(bundle.getAppName())); 1509 1510 System.out.println(RULER); 1511 1512 System.out.println("Bundle ID : " + maskIfNull(bundle.getId())); 1513 System.out.println("Coordinator Name : " + maskIfNull(coord.getAppName())); 1514 System.out.println("Coord Action ID : " + maskIfNull(action.getId())); 1515 System.out.println("Action Status : " + action.getStatus()); 1516 System.out.println("External ID : " + maskIfNull(action.getExternalId())); 1517 System.out.println("Created Time : " + maskDate(action.getCreatedTime(), timeZoneId, false)); 1518 System.out.println("User : " + maskIfNull(bundle.getUser())); 1519 System.out.println("Error Message : " + maskIfNull(action.getErrorMessage())); 1520 System.out.println(RULER); 1521 } 1522 else { 1523 System.out.println(String.format(BULK_RESPONSE_FORMATTER, "Bundle Name", "Bundle ID", "Coord Name", 1524 "Coord Action ID", "Status", "External ID", "Created Time", "Error Message")); 1525 System.out.println(RULER); 1526 System.out 1527 .println(String.format(BULK_RESPONSE_FORMATTER, maskIfNull(bundle.getAppName()), 1528 maskIfNull(bundle.getId()), maskIfNull(coord.getAppName()), 1529 maskIfNull(action.getId()), action.getStatus(), maskIfNull(action.getExternalId()), 1530 maskDate(action.getCreatedTime(), timeZoneId, false), 1531 maskIfNull(action.getErrorMessage()))); 1532 System.out.println(RULER); 1533 } 1534 } 1535 } 1536 else { 1537 System.out.println("Bulk request criteria did not match any coordinator actions"); 1538 } 1539 } 1540 1541 @VisibleForTesting 1542 void printBundleJobs(List<BundleJob> jobs, String timeZoneId, boolean verbose) throws IOException { 1543 if (jobs != null && jobs.size() > 0) { 1544 if (verbose) { 1545 System.out.println("Job ID" + VERBOSE_DELIMITER + "Bundle Name" + VERBOSE_DELIMITER + "Bundle Path" 1546 + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group" + VERBOSE_DELIMITER + "Status" 1547 + VERBOSE_DELIMITER + "Kickoff" + VERBOSE_DELIMITER + "Pause" + VERBOSE_DELIMITER + "Created" 1548 + VERBOSE_DELIMITER + "Console URL"); 1549 System.out.println(RULER); 1550 1551 for (BundleJob job : jobs) { 1552 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName()) 1553 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER 1554 + maskIfNull(job.getUser()) + VERBOSE_DELIMITER + maskIfNull(job.getGroup()) 1555 + VERBOSE_DELIMITER + job.getStatus() + VERBOSE_DELIMITER 1556 + maskDate(job.getKickoffTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1557 + maskDate(job.getPauseTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1558 + maskDate(job.getCreatedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1559 + maskIfNull(job.getConsoleUrl())); 1560 1561 System.out.println(RULER); 1562 } 1563 } 1564 else { 1565 System.out.println(String.format(BUNDLE_JOBS_FORMATTER, "Job ID", "Bundle Name", "Status", "Kickoff", 1566 "Created", "User", "Group")); 1567 System.out.println(RULER); 1568 1569 for (BundleJob job : jobs) { 1570 System.out.println(String.format(BUNDLE_JOBS_FORMATTER, maskIfNull(job.getId()), 1571 maskIfNull(job.getAppName()), job.getStatus(), 1572 maskDate(job.getKickoffTime(), timeZoneId, verbose), 1573 maskDate(job.getCreatedTime(), timeZoneId, verbose), maskIfNull(job.getUser()), 1574 maskIfNull(job.getGroup()))); 1575 System.out.println(RULER); 1576 } 1577 } 1578 } 1579 else { 1580 System.out.println("No Jobs match your criteria!"); 1581 } 1582 } 1583 1584 private void slaCommand(CommandLine commandLine) throws IOException, OozieCLIException { 1585 XOozieClient wc = createXOozieClient(commandLine); 1586 List<String> options = new ArrayList<String>(); 1587 for (Option option : commandLine.getOptions()) { 1588 options.add(option.getOpt()); 1589 } 1590 1591 String s = commandLine.getOptionValue(OFFSET_OPTION); 1592 int start = Integer.parseInt((s != null) ? s : "0"); 1593 s = commandLine.getOptionValue(LEN_OPTION); 1594 int len = Integer.parseInt((s != null) ? s : "100"); 1595 String filter = commandLine.getOptionValue(FILTER_OPTION); 1596 1597 try { 1598 wc.getSlaInfo(start, len, filter); 1599 } 1600 catch (OozieClientException ex) { 1601 throw new OozieCLIException(ex.toString(), ex); 1602 } 1603 } 1604 1605 private void adminCommand(CommandLine commandLine) throws OozieCLIException { 1606 XOozieClient wc = createXOozieClient(commandLine); 1607 1608 List<String> options = new ArrayList<String>(); 1609 for (Option option : commandLine.getOptions()) { 1610 options.add(option.getOpt()); 1611 } 1612 1613 try { 1614 SYSTEM_MODE status = SYSTEM_MODE.NORMAL; 1615 if (options.contains(VERSION_OPTION)) { 1616 System.out.println("Oozie server build version: " + wc.getServerBuildVersion()); 1617 } 1618 else if (options.contains(SYSTEM_MODE_OPTION)) { 1619 String systemModeOption = commandLine.getOptionValue(SYSTEM_MODE_OPTION).toUpperCase(); 1620 try { 1621 status = SYSTEM_MODE.valueOf(systemModeOption); 1622 } 1623 catch (Exception e) { 1624 throw new OozieCLIException("Invalid input provided for option: " + SYSTEM_MODE_OPTION 1625 + " value given :" + systemModeOption 1626 + " Expected values are: NORMAL/NOWEBSERVICE/SAFEMODE "); 1627 } 1628 wc.setSystemMode(status); 1629 System.out.println("System mode: " + status); 1630 } 1631 else if (options.contains(STATUS_OPTION)) { 1632 status = wc.getSystemMode(); 1633 System.out.println("System mode: " + status); 1634 } 1635 1636 else if (options.contains(UPDATE_SHARELIB_OPTION)) { 1637 System.out.println(wc.updateShareLib()); 1638 } 1639 1640 else if (options.contains(LIST_SHARELIB_LIB_OPTION)) { 1641 String sharelibKey = null; 1642 if (commandLine.getArgList().size() > 0) { 1643 sharelibKey = (String) commandLine.getArgList().get(0); 1644 } 1645 System.out.println(wc.listShareLib(sharelibKey)); 1646 } 1647 1648 else if (options.contains(QUEUE_DUMP_OPTION)) { 1649 1650 List<String> list = wc.getQueueDump(); 1651 if (list != null && list.size() != 0) { 1652 for (String str : list) { 1653 System.out.println(str); 1654 } 1655 } 1656 else { 1657 System.out.println("QueueDump is null!"); 1658 } 1659 } 1660 else if (options.contains(AVAILABLE_SERVERS_OPTION)) { 1661 Map<String, String> availableOozieServers = wc.getAvailableOozieServers(); 1662 for (String key : availableOozieServers.keySet()) { 1663 System.out.println(key + " : " + availableOozieServers.get(key)); 1664 } 1665 } 1666 } 1667 catch (OozieClientException ex) { 1668 throw new OozieCLIException(ex.toString(), ex); 1669 } 1670 } 1671 1672 private void versionCommand() throws OozieCLIException { 1673 System.out.println("Oozie client build version: " 1674 + BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION)); 1675 } 1676 1677 @VisibleForTesting 1678 void printJobs(List<WorkflowJob> jobs, String timeZoneId, boolean verbose) throws IOException { 1679 if (jobs != null && jobs.size() > 0) { 1680 if (verbose) { 1681 System.out.println("Job ID" + VERBOSE_DELIMITER + "App Name" + VERBOSE_DELIMITER + "App Path" 1682 + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group" 1683 + VERBOSE_DELIMITER + "Run" + VERBOSE_DELIMITER + "Created" + VERBOSE_DELIMITER + "Started" 1684 + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified" + VERBOSE_DELIMITER 1685 + "Ended"); 1686 System.out.println(RULER); 1687 1688 for (WorkflowJob job : jobs) { 1689 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName()) 1690 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER 1691 + maskIfNull(job.getConsoleUrl()) + VERBOSE_DELIMITER + maskIfNull(job.getUser()) 1692 + VERBOSE_DELIMITER + maskIfNull(job.getGroup()) + VERBOSE_DELIMITER + job.getRun() 1693 + VERBOSE_DELIMITER + maskDate(job.getCreatedTime(), timeZoneId, verbose) 1694 + VERBOSE_DELIMITER + maskDate(job.getStartTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1695 + job.getStatus() + VERBOSE_DELIMITER 1696 + maskDate(job.getLastModifiedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER 1697 + maskDate(job.getEndTime(), timeZoneId, verbose)); 1698 1699 System.out.println(RULER); 1700 } 1701 } 1702 else { 1703 System.out.println(String.format(WORKFLOW_JOBS_FORMATTER, "Job ID", "App Name", "Status", "User", 1704 "Group", "Started", "Ended")); 1705 System.out.println(RULER); 1706 1707 for (WorkflowJob job : jobs) { 1708 System.out.println(String.format(WORKFLOW_JOBS_FORMATTER, maskIfNull(job.getId()), 1709 maskIfNull(job.getAppName()), job.getStatus(), maskIfNull(job.getUser()), 1710 maskIfNull(job.getGroup()), maskDate(job.getStartTime(), timeZoneId, verbose), 1711 maskDate(job.getEndTime(), timeZoneId, verbose))); 1712 1713 System.out.println(RULER); 1714 } 1715 } 1716 } 1717 else { 1718 System.out.println("No Jobs match your criteria!"); 1719 } 1720 } 1721 1722 void printWfsForCoordAction(List<WorkflowJob> jobs, String timeZoneId) throws IOException { 1723 if (jobs != null && jobs.size() > 0) { 1724 System.out.println(String.format("%-41s%-10s%-24s%-24s", "Job ID", "Status", "Started", "Ended")); 1725 System.out.println(RULER); 1726 1727 for (WorkflowJob job : jobs) { 1728 System.out 1729 .println(String.format("%-41s%-10s%-24s%-24s", maskIfNull(job.getId()), job.getStatus(), 1730 maskDate(job.getStartTime(), timeZoneId, false), 1731 maskDate(job.getEndTime(), timeZoneId, false))); 1732 System.out.println(RULER); 1733 } 1734 } 1735 } 1736 1737 private String maskIfNull(String value) { 1738 if (value != null && value.length() > 0) { 1739 return value; 1740 } 1741 return "-"; 1742 } 1743 1744 private String maskDate(Date date, String timeZoneId, boolean verbose) { 1745 if (date == null) { 1746 return "-"; 1747 } 1748 1749 SimpleDateFormat dateFormater = null; 1750 if (verbose) { 1751 dateFormater = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz", Locale.US); 1752 } 1753 else { 1754 dateFormater = new SimpleDateFormat("yyyy-MM-dd HH:mm zzz", Locale.US); 1755 } 1756 1757 if (timeZoneId != null) { 1758 dateFormater.setTimeZone(TimeZone.getTimeZone(timeZoneId)); 1759 } 1760 String dateString = dateFormater.format(date); 1761 // Most TimeZones are 3 or 4 characters; GMT offsets (e.g. GMT-07:00) are 9, so lets remove the "GMT" part to make it 6 1762 // to fit better 1763 Matcher m = GMT_OFFSET_SHORTEN_PATTERN.matcher(dateString); 1764 if (m.matches() && m.groupCount() == 2) { 1765 dateString = m.group(1) + m.group(2); 1766 } 1767 return dateString; 1768 } 1769 1770 private void validateCommand(CommandLine commandLine) throws OozieCLIException { 1771 String[] args = commandLine.getArgs(); 1772 if (args.length != 1) { 1773 throw new OozieCLIException("One file must be specified"); 1774 } 1775 File file = new File(args[0]); 1776 if (file.exists()) { 1777 try { 1778 List<StreamSource> sources = new ArrayList<StreamSource>(); 1779 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1780 "oozie-workflow-0.1.xsd"))); 1781 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1782 "shell-action-0.1.xsd"))); 1783 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1784 "shell-action-0.2.xsd"))); 1785 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1786 "shell-action-0.3.xsd"))); 1787 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1788 "email-action-0.1.xsd"))); 1789 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1790 "email-action-0.2.xsd"))); 1791 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1792 "distcp-action-0.1.xsd"))); 1793 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1794 "distcp-action-0.2.xsd"))); 1795 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1796 "oozie-workflow-0.2.xsd"))); 1797 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1798 "oozie-workflow-0.2.5.xsd"))); 1799 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1800 "oozie-workflow-0.3.xsd"))); 1801 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1802 "oozie-workflow-0.4.xsd"))); 1803 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1804 "oozie-workflow-0.4.5.xsd"))); 1805 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1806 "oozie-workflow-0.5.xsd"))); 1807 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1808 "oozie-coordinator-0.1.xsd"))); 1809 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1810 "oozie-coordinator-0.2.xsd"))); 1811 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1812 "oozie-coordinator-0.3.xsd"))); 1813 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1814 "oozie-coordinator-0.4.xsd"))); 1815 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1816 "oozie-bundle-0.1.xsd"))); 1817 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1818 "oozie-bundle-0.2.xsd"))); 1819 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1820 "oozie-sla-0.1.xsd"))); 1821 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1822 "oozie-sla-0.2.xsd"))); 1823 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1824 "hive-action-0.2.xsd"))); 1825 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1826 "hive-action-0.3.xsd"))); 1827 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1828 "hive-action-0.4.xsd"))); 1829 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1830 "hive-action-0.5.xsd"))); 1831 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1832 "sqoop-action-0.2.xsd"))); 1833 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1834 "sqoop-action-0.3.xsd"))); 1835 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1836 "sqoop-action-0.4.xsd"))); 1837 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1838 "ssh-action-0.1.xsd"))); 1839 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream( 1840 "ssh-action-0.2.xsd"))); 1841 SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); 1842 Schema schema = factory.newSchema(sources.toArray(new StreamSource[sources.size()])); 1843 Validator validator = schema.newValidator(); 1844 validator.validate(new StreamSource(new FileReader(file))); 1845 System.out.println("Valid workflow-app"); 1846 } 1847 catch (Exception ex) { 1848 throw new OozieCLIException("Invalid app definition, " + ex.toString(), ex); 1849 } 1850 } 1851 else { 1852 throw new OozieCLIException("File does not exists"); 1853 } 1854 } 1855 1856 private void scriptLanguageCommand(CommandLine commandLine, String jobType) throws IOException, OozieCLIException { 1857 List<String> args = commandLine.getArgList(); 1858 if (args.size() > 0) { 1859 // checking if args starts with -X (because CLIParser cannot check this) 1860 if (!args.get(0).equals("-X")) { 1861 throw new OozieCLIException("Unrecognized option: " + args.get(0) + " Expecting -X"); 1862 } 1863 args.remove(0); 1864 } 1865 1866 if (!commandLine.hasOption(SCRIPTFILE_OPTION)) { 1867 throw new OozieCLIException("Need to specify -file <scriptfile>"); 1868 } 1869 1870 if (!commandLine.hasOption(CONFIG_OPTION)) { 1871 throw new OozieCLIException("Need to specify -config <configfile>"); 1872 } 1873 1874 try { 1875 XOozieClient wc = createXOozieClient(commandLine); 1876 Properties conf = getConfiguration(wc, commandLine); 1877 String script = commandLine.getOptionValue(SCRIPTFILE_OPTION); 1878 List<String> paramsList = new ArrayList<String>(); 1879 if (commandLine.hasOption("P")) { 1880 Properties params = commandLine.getOptionProperties("P"); 1881 for (String key : params.stringPropertyNames()) { 1882 paramsList.add(key + "=" + params.getProperty(key)); 1883 } 1884 } 1885 System.out.println(JOB_ID_PREFIX + wc.submitScriptLanguage(conf, script, args.toArray(new String[args.size()]), 1886 paramsList.toArray(new String[paramsList.size()]), jobType)); 1887 } 1888 catch (OozieClientException ex) { 1889 throw new OozieCLIException(ex.toString(), ex); 1890 } 1891 } 1892 1893 private void sqoopCommand(CommandLine commandLine) throws IOException, OozieCLIException { 1894 List<String> args = commandLine.getArgList(); 1895 if (args.size() > 0) { 1896 // checking if args starts with -X (because CLIParser cannot check this) 1897 if (!args.get(0).equals("-X")) { 1898 throw new OozieCLIException("Unrecognized option: " + args.get(0) + " Expecting -X"); 1899 } 1900 args.remove(0); 1901 } 1902 1903 if (!commandLine.hasOption(SQOOP_COMMAND_OPTION)) { 1904 throw new OozieCLIException("Need to specify -command"); 1905 } 1906 1907 if (!commandLine.hasOption(CONFIG_OPTION)) { 1908 throw new OozieCLIException("Need to specify -config <configfile>"); 1909 } 1910 1911 try { 1912 XOozieClient wc = createXOozieClient(commandLine); 1913 Properties conf = getConfiguration(wc, commandLine); 1914 String[] command = commandLine.getOptionValues(SQOOP_COMMAND_OPTION); 1915 System.out.println(JOB_ID_PREFIX + wc.submitSqoop(conf, command, args.toArray(new String[args.size()]))); 1916 } 1917 catch (OozieClientException ex) { 1918 throw new OozieCLIException(ex.toString(), ex); 1919 } 1920 } 1921 1922 private void infoCommand(CommandLine commandLine) throws OozieCLIException { 1923 for (Option option : commandLine.getOptions()) { 1924 String opt = option.getOpt(); 1925 if (opt.equals(INFO_TIME_ZONES_OPTION)) { 1926 printAvailableTimeZones(); 1927 } 1928 } 1929 } 1930 1931 private void printAvailableTimeZones() { 1932 System.out.println("The format is \"SHORT_NAME (ID)\"\nGive the ID to the -timezone argument"); 1933 System.out.println("GMT offsets can also be used (e.g. GMT-07:00, GMT-0700, GMT+05:30, GMT+0530)"); 1934 System.out.println("Available Time Zones:"); 1935 for (String tzId : TimeZone.getAvailableIDs()) { 1936 // skip id's that are like "Etc/GMT+01:00" because their display names are like "GMT-01:00", which is confusing 1937 if (!tzId.startsWith("Etc/GMT")) { 1938 TimeZone tZone = TimeZone.getTimeZone(tzId); 1939 System.out.println(" " + tZone.getDisplayName(false, TimeZone.SHORT) + " (" + tzId + ")"); 1940 } 1941 } 1942 } 1943 1944 1945 private void mrCommand(CommandLine commandLine) throws IOException, OozieCLIException { 1946 try { 1947 XOozieClient wc = createXOozieClient(commandLine); 1948 Properties conf = getConfiguration(wc, commandLine); 1949 1950 String mapper = conf.getProperty(MAPRED_MAPPER, conf.getProperty(MAPRED_MAPPER_2)); 1951 if (mapper == null) { 1952 throw new OozieCLIException("mapper (" + MAPRED_MAPPER + " or " + MAPRED_MAPPER_2 + ") must be specified in conf"); 1953 } 1954 1955 String reducer = conf.getProperty(MAPRED_REDUCER, conf.getProperty(MAPRED_REDUCER_2)); 1956 if (reducer == null) { 1957 throw new OozieCLIException("reducer (" + MAPRED_REDUCER + " or " + MAPRED_REDUCER_2 1958 + ") must be specified in conf"); 1959 } 1960 1961 String inputDir = conf.getProperty(MAPRED_INPUT); 1962 if (inputDir == null) { 1963 throw new OozieCLIException("input dir (" + MAPRED_INPUT +") must be specified in conf"); 1964 } 1965 1966 String outputDir = conf.getProperty(MAPRED_OUTPUT); 1967 if (outputDir == null) { 1968 throw new OozieCLIException("output dir (" + MAPRED_OUTPUT +") must be specified in conf"); 1969 } 1970 1971 System.out.println(JOB_ID_PREFIX + wc.submitMapReduce(conf)); 1972 } 1973 catch (OozieClientException ex) { 1974 throw new OozieCLIException(ex.toString(), ex); 1975 } 1976 } 1977 1978 private String getFirstMissingDependencies(CoordinatorAction action) { 1979 StringBuilder allDeps = new StringBuilder(); 1980 String missingDep = action.getMissingDependencies(); 1981 boolean depExists = false; 1982 if (missingDep != null && !missingDep.isEmpty()) { 1983 allDeps.append(missingDep.split(INSTANCE_SEPARATOR)[0]); 1984 depExists = true; 1985 } 1986 String pushDeps = action.getPushMissingDependencies(); 1987 if (pushDeps != null && !pushDeps.isEmpty()) { 1988 if(depExists) { 1989 allDeps.append(INSTANCE_SEPARATOR); 1990 } 1991 allDeps.append(pushDeps.split(INSTANCE_SEPARATOR)[0]); 1992 } 1993 return allDeps.toString(); 1994 } 1995 1996}