#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # To use this to execute your Eclipse workspace, install as a symbolic # link in your JRE/bin directory, with any name other than 'viaducc', # for instance, java_viaducc,, linking back to the base version of # viaducc in your ducc_runtime. # # Then configre your Eclipse's launch JRE to use java_viaducc in place # of 'java' import sys import os import re global ducc_mem_size global ducc_mem_size_set global ducc_desc_set global default_jvm global ducc_home ducc_class = 'fixed' java_cmd = None ducc_home = None # tbd in a minute ducc_mem_size = None ducc_mem_size_set = False # whether the user passed a flag to manually set DUCC_MEMORY_SIZE ducc_desc_set = False # whether the user passed a flag to manually set DUCC_DESCRIPTION default_jvm = None # get the default DUCC_HOME right away for message realpath = os.path.realpath(sys.argv[0]) ndx = realpath.rindex('/') ndx = realpath.rindex('/', 0, ndx) ducc_home = realpath[:ndx] def error(message): print "Error:", message sys.exit(1) def usage(): print "Usage: viaducc [defines] [command and parameters]" print " -or-" print " java_viaducc [defines] [java-class and parameters]" print "" print "Where defines include:" print " -DDUCC_MEMORY_SIZE=size-in-gb" print " The default is -DDUCC_MEMORY_SIZE="+str(ducc_mem_size) print"" print " -DDUCC_HOME=alternative-DUCC-runtime" print " The default is -DDUCC_HOME=" + ducc_home print "" print " -DDUCC_CLASS=ducc-scheduling-class" print " The default is -DDUCC_CLASS=" + ducc_class print "" print " -DDUCC_ENVIRONMENT=environment-settings" print " The default is no additional environment. The environment string should be" print ' blank-delimeted and quoted, for example: -DDUCC_ENVIRONMENT="A=B C=D"' if (default_jvm != None ): print "" print " -DJAVA_BIN=specific-java-bin-directory" print " For use with java_viaducc. Details follow." print " The default is -DJAVA_BIN=" + default_jvm print "" print " -DDUCC_DESRIPTION=user-description-string" print " The default description is the program name (viaducc or java_viaducc) with arguments" print "" print " -DDUCC_NO_CANCEL_ON_INTERRUPT" print " To disable process cancellation when the submitting process terminates." print "" print " When invoked as 'viaducc' the command is executed in a DUCC-managed resource" print "with the console (bi-directionally) redirected to the submitter's console." print "" print " When invokded as 'java_viaducc' an appropriate JRE is found and the class" print "is executed as a java class in a DUCC-managed resource, with the console" print "bi-directionally redirected to the submitter's console." print "" print "Notes for java_viaducc:" print "" print " If java_viaducc is installed in a JRE/bin directory, it may be invoked as an" print "alternative to java from Eclipse, allowing Eclipse users to execute their workspaces" print "in a DUCC-managed resource. JAVA_BIN should NOT be used in this case. java_viaducc" print "will use the JRE it is installed in." print "" print " If java_viaducc is used as a 'normal' command, the JRE is searched for in this order:" print " 1. Use the java specified by -DJAVA_BIN=" print " 2. Use the java configured for DUCC" print "" print " If -jar is used it must be the last JVM arg before any command line args" print sys.exit(0) # where should I get java from? def read_properties(): ducc_properties = ducc_home + '/resources/ducc.properties' if ( not os.path.exists(ducc_properties) ): error("Cannot access DUCC_HOME at " + ducc_home) ducc_jvm = None # first see if i'm executed out of some JAVA_HOME somewhere jpath = os.path.abspath(sys.argv[0]) ndx = jpath.rindex('/') jdir = jpath[:ndx] java = jdir + '/java' if (os.path.exists(java)): ducc_jvm = java # nope, we'll use the default ducc java props = open(ducc_properties) try: for line in props: line = line.strip() mobj = re.search('[ =:]+', line) # java props allows apace, =, or : as delimeters if ( mobj ): key = line[:mobj.start()].strip() val = line[mobj.end():].strip() if ( (ducc_jvm is None) and (key == 'ducc.jvm') ): # yes! ducc_jvm = val continue if ( key == 'ducc.rm.share.quantum' ): # yes! ducc_mem = 1 # default quantum may not be the same for all classes continue # and the actual allocation will be rounded up to quantum finally: props.close(); return (ducc_jvm, ducc_mem) def parse_memory_string(text): """Given a string (e.g., "5G", "200m"), Returns the memory requested in bytes. For example: parse_memory_string('500m') should return 500 * 1024 * 1024 parse_memory_string('15G') should return 15 * 1024 * 1024 * 1024 """ # this list should cover us for a while suffixes = { 'k': 1024, 'm': 1024 * 1024, 'g': 1024 * 1024 * 1024, 't': 1024 * 1024 * 1024 * 1024, } text = text.lower() if text[-1] in suffixes: multiplier = suffixes[text[-1]] text = text[:-1] # chop off suffix else: multiplier = 1 return multiplier * int(text) def parse_java_command_line(): """Parse the command line and returns a dictionary including information on the maximum amount of memory the job might need, the Java class name, and its arguments.""" maximum_memory_required = 1024.0 ** 3 # our default if -Xmx not specified args = iter(sys.argv[1:]) skip_next_arg = False # scan args until we find the Java class while 1: arg = next(args).lower() if skip_next_arg: skip_next_arg = False continue if arg.startswith('-'): # these are the only options that take an argument and require # a space between the flag and the argument if arg in ('-cp', '-classpath'): skip_next_arg = True elif arg.startswith('-xmx'): # maximum Java heap size maximum_memory_required = parse_memory_string(arg[4:]) continue break return dict(maximum_memory_required=maximum_memory_required, java_class_name=arg, java_class_args=list(args)) def main(): global ducc_mem_size global ducc_mem_size_set global ducc_desc_set global default_jvm global ducc_home if ( len(sys.argv) == 1 ): usage() ducc_class = 'fixed' java_cmd = None ducc_env = '' enable_cancel = True ducc_desc = os.path.basename(sys.argv[0]) # remember to add the '=' at the end if following value p_mem_size = '-DDUCC_MEMORY_SIZE=' p_class = '-DDUCC_CLASS=' p_jvm_dir = '-DJAVA_BIN=' p_ducc_home = '-DDUCC_HOME=' p_env = '-DDUCC_ENVIRONMENT=' p_no_int = '-DDUCC_NO_CANCEL_ON_INTERRUPT' p_desc = '-DDUCC_DESCRIPTION=' need_java = False if ( os.path.basename(sys.argv[0]) != 'viaducc' ): # if invokded as 'java_viaducc we must inject some kind of jvm as the executable # this is more liberal: any link other than 'viaducc' caused injection of the # jvm, to allow multiple, different symlinks (in case there are multiple, # different versions of ducc, not really reccomended, but supported) need_java = True ducc_args = [] for arg in sys.argv[1:]: if (arg in ('-h', '-?', '-help', '--help') ): # have to wait for parsing the args to look up DUCC_HOME so we defer emitting the message usage() elif (arg.startswith(p_mem_size) ): ducc_mem_size = arg[len(p_mem_size):] ducc_mem_size_set = True elif (arg.startswith(p_class) ): ducc_class = arg[len(p_class):] elif (arg.startswith(p_jvm_dir) ): java_cmd = arg[len(p_jvm_dir):] + '/java' elif (arg.startswith(p_ducc_home) ): ducc_home = arg[len(p_ducc_home):] elif (arg.startswith(p_env) ): ducc_env = ' --environment "' + arg[len(p_env):] + '"' elif (arg.startswith(p_desc) ): ducc_desc = arg[len(p_desc):] ducc_desc_set = True elif (arg.startswith(p_no_int) ): enable_cancel = False else: ducc_args.append("'" + arg + "'") p = read_properties() default_jvm = p[0] if (ducc_mem_size is None): ducc_mem_size = p[1] description = os.path.basename(sys.argv[0]) if ( need_java and (java_cmd is None) ): java_cmd = default_jvm if ( (java_cmd is None) or (java_cmd == '') ): error("Cannot figure out where java is") if need_java: java_command_line_info = parse_java_command_line() description += ' %s %s' % (java_command_line_info['java_class_name'], ' '.join(java_command_line_info['java_class_args'])) import math # convert to gigabytes, round up maximum_memory_required = java_command_line_info['maximum_memory_required'] / (1024.0 ** 3) maximum_memory_required = int(math.ceil(maximum_memory_required)) if maximum_memory_required > float(ducc_mem_size): if ducc_mem_size_set: print ("Warning: DUCC memory size specified as %sg but max heap set to %sg.\n" " Consider increasing -DDUCC_MEMORY_SIZE=\n") % \ (ducc_mem_size, maximum_memory_required) else: print ("Info: DUCC memory size is not specified but asking JVM to set max heap to %sg.\n" " Setting DUCC memory size to %sg. Use -DDUCC_MEMORY_SIZE= to specify\n") % \ (maximum_memory_required, maximum_memory_required) ducc_mem_size = maximum_memory_required # DUCC_HOME isn't ususally needed but viaducc is slithery and may end up tryng to # execute from somebody else's ducc so let's be sure we point at the right one os.environ['DUCC_HOME'] = ducc_home CMD = ducc_home + '/bin/ducc_process_submit' CMD = CMD + ' --attach_console' CMD = CMD + ' --process_memory_size ' + str(ducc_mem_size) CMD = CMD + ' --scheduling_class ' + ducc_class if ducc_desc_set: CMD = CMD + ' --description %r' % ducc_desc else: if need_java: CMD = CMD + ' --description %r' % description # %r so it will be quoted else: CMD = CMD + ' --description viaducc' CMD = CMD + ' --wait_for_completion' if (enable_cancel): CMD = CMD + ' --cancel_on_interrupt' CMD = CMD + ' --working_directory ' + os.getcwd() CMD = CMD + ducc_env if ( need_java ): CMD = CMD + ' --process_executable ' + java_cmd if ( len(ducc_args) > 0): CMD = CMD + ' --process_executable_args "' + ' '.join(ducc_args) + '"' else: if ( len(ducc_args) == 0 ): error("No command specified.") CMD = CMD + ' --process_executable ' + ducc_args[0] if ( len(ducc_args[1:]) > 0): CMD = CMD + ' --process_executable_args "' + ' '.join(ducc_args[1:]) + '"' print CMD sys.stdout.flush() # UIMA-4168 rc = os.system(CMD) sys.exit(rc); main()