job_generation: job_generation: run_auto_stack_prerelease

00001 #!/usr/bin/python
00002 
00003 STACK_DIR = 'stack_overlay'
00004 DEPENDS_DIR = 'depends_overlay'
00005 DEPENDS_ON_DIR = 'depends_on_overlay'
00006 
00007 
00008 import roslib; roslib.load_manifest("job_generation")
00009 from roslib import stack_manifest
00010 import rosdistro
00011 from jobs_common import *
00012 from apt_parser import parse_apt
00013 import sys
00014 import os
00015 import optparse 
00016 import subprocess
00017 import traceback
00018 
00019 
00020 def remove(list1, list2):
00021     for l in list2:
00022         if l in list1:
00023             list1.remove(l)
00024 
00025 
00026 def main():
00027     # global try
00028     try:
00029 
00030         # parse command line options
00031         (options, args) = get_options(['stack', 'rosdistro'], ['repeat', 'source-only'])
00032         if not options:
00033             return -1
00034 
00035         # set environment
00036         print "Setting up environment"
00037         env = get_environment()
00038         env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks'%(env['INSTALL_DIR']+'/'+STACK_DIR,
00039                                                                  env['INSTALL_DIR']+'/'+DEPENDS_DIR,
00040                                                                  env['INSTALL_DIR']+'/'+DEPENDS_ON_DIR,
00041                                                                  options.rosdistro)
00042         if 'ros' in options.stack:
00043             env['ROS_ROOT'] = env['INSTALL_DIR']+'/'+STACK_DIR+'/ros'
00044             print "We're building ROS, so setting the ROS_ROOT to %s"%(env['ROS_ROOT'])
00045         else:
00046             env['ROS_ROOT'] = '/opt/ros/%s/ros'%options.rosdistro
00047         env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src'
00048         env['PATH'] = '/opt/ros/%s/ros/bin:%s'%(options.rosdistro, os.environ['PATH'])
00049         print "Environment set to %s"%str(env)
00050 
00051         # Parse distro file
00052         rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
00053         print 'Operating on ROS distro %s'%rosdistro_obj.release_name
00054 
00055 
00056         # Install the stacks to test from source
00057         print 'Installing the stacks to test from source'
00058         rosinstall = ''
00059         for stack in options.stack:
00060             rosinstall += stack_to_rosinstall(rosdistro_obj.stacks[stack], 'devel')
00061         rosinstall_file = '%s.rosinstall'%STACK_DIR
00062         print 'Generating rosinstall file [%s]'%(rosinstall_file)
00063         print 'Contents:\n\n'+rosinstall+'\n\n'
00064         with open(rosinstall_file, 'w') as f:
00065             f.write(rosinstall)
00066             print 'rosinstall file [%s] generated'%(rosinstall_file)
00067         call('rosinstall --rosdep-yes %s /opt/ros/%s %s'%(STACK_DIR, options.rosdistro, rosinstall_file), env,
00068              'Install the stacks to test from source.')
00069 
00070 
00071         # get all stack dependencies of stacks we're testing
00072         print "Computing dependencies of stacks we're testing"
00073         depends_all = []
00074         for stack in options.stack:    
00075             stack_xml = '%s/%s/stack.xml'%(STACK_DIR, stack)
00076             call('ls %s'%stack_xml, env, 'Checking if stack %s contains "stack.xml" file'%stack)
00077             with open(stack_xml) as stack_file:
00078                 depends_one = [str(d) for d in stack_manifest.parse(stack_file.read()).depends]  # convert to list
00079                 print 'Dependencies of stack %s: %s'%(stack, str(depends_one))
00080                 for d in depends_one:
00081                     if not d in options.stack and not d in depends_all:
00082                         print 'Adding dependencies of stack %s'%d
00083                         get_depends_all(rosdistro_obj, d, depends_all)
00084                         print 'Resulting total dependencies of all stacks that get tested: %s'%str(depends_all)
00085 
00086         if len(depends_all) > 0:
00087             if options.source_only:
00088                 # Install dependencies from source
00089                 print 'Installing stack dependencies from source'
00090                 rosinstall = stacks_to_rosinstall(depends_all, rosdistro_obj.released_stacks, 'release-tar')
00091                 rosinstall_file = '%s.rosinstall'%DEPENDS_DIR
00092                 print 'Generating rosinstall file [%s]'%(rosinstall_file)
00093                 print 'Contents:\n\n'+rosinstall+'\n\n'
00094                 with open(rosinstall_file, 'w') as f:
00095                     f.write(rosinstall)
00096                     print 'rosinstall file [%s] generated'%(rosinstall_file)
00097                 call('rosinstall --rosdep-yes %s /opt/ros/%s %s'%(DEPENDS_DIR, options.rosdistro, rosinstall_file), env,
00098                      'Install the stack dependencies from source.')
00099             else:
00100                 # Install Debian packages of stack dependencies
00101                 print 'Installing debian packages of "%s" dependencies: %s'%(stack, str(depends_all))
00102                 call('sudo apt-get update', env)
00103                 call('sudo apt-get install %s --yes'%(stacks_to_debs(depends_all, options.rosdistro)), env)
00104         else:
00105             print 'Stack(s) %s do(es) not have any dependencies, not installing anything now'%str(options.stack)
00106 
00107 
00108         # Install system dependencies of stacks re're testing
00109         print "Installing system dependencies of stacks we're testing"
00110         call('rosmake rosdep', env)
00111         for stack in options.stack:
00112             call('rosdep install -y %s'%stack, env,
00113                  'Install system dependencies of stack %s'%stack)
00114 
00115 
00116         # Run hudson helper for stacks only
00117         print "Running Hudson Helper for stacks we're testing"
00118         res = 0
00119         for r in range(0, int(options.repeat)+1):
00120             env['ROS_TEST_RESULTS_DIR'] = env['ROS_TEST_RESULTS_DIR'] + '/' + STACK_DIR + '_run_' + str(r)
00121             helper = subprocess.Popen(('./hudson_helper --dir-test %s build'%STACK_DIR).split(' '), env=env)
00122             helper.communicate()
00123             if helper.returncode != 0:
00124                 res = helper.returncode
00125         if res != 0:
00126             return res
00127 
00128 
00129         # parse debian repository configuration file to get stack dependencies
00130         arch = 'i386'
00131         if '64' in call('uname -mrs', env):
00132             arch = 'amd64'
00133         ubuntudistro = call('lsb_release -a', env).split('Codename:')[1].strip()
00134         print "Parsing apt repository configuration file to get stack dependencies, for %s machine running %s"%(arch, ubuntudistro)
00135         apt_deps = parse_apt(ubuntudistro, arch, options.rosdistro)
00136         if not apt_deps.has_debian_package(options.stack):
00137             print "Stack does not yet have a Debian package. No need to test dependenies"
00138             return 0
00139 
00140         # all stacks that depends on the tested stacks, excluding the tested stacks.
00141         depends_on_all = apt_deps.depends_on_all(options.stack)
00142         remove(depends_on_all, options.stack)
00143 
00144         # if tested stacks are all in a variant, then only test stacks that are also in a variant
00145         variant_stacks = []
00146         for name, v in rosdistro_obj.variants.iteritems():
00147             variant_stacks = variant_stacks + v.stack_names
00148         all_in_variant = True
00149         for s in options.stack:
00150             if not s in variant_stacks:
00151                 all_in_variant = False
00152         if all_in_variant:
00153             print "Limiting test to stacks that are in a variant"
00154             for s in depends_on_all:
00155                 if not s in variant_stacks:
00156                     depends_on_all.remove(s)
00157 
00158         # all stack dependencies of above stack list, except for the test stack dependencies
00159         depends_all_depends_on_all = apt_deps.depends_all(depends_on_all)
00160         remove(depends_all_depends_on_all, options.stack)
00161         remove(depends_all_depends_on_all, depends_all)
00162 
00163 
00164         # Install dependencies of depends_on_all stacks, excluding dependencies of test stacks.
00165         if len(depends_all_depends_on_all) > 0:
00166             print "Install dependencies of depends_on_all stacks, excluding dependencies of test stacks."
00167             if not options.source_only:
00168                 # Install Debian packages of 'depends_all_depends_on_all' list
00169                 print 'Installing Debian package of %s'%str(depends_all_depends_on_all)
00170                 call('sudo apt-get install %s --yes'%(stacks_to_debs(depends_all_depends_on_all, options.rosdistro)), env)
00171             else:
00172                 # Install source of 'depends_all_depends_on_all' list
00173                 print 'Installing source of %s'%str(depends_all_depends_on_all)
00174                 rosinstall = stacks_to_rosinstall(depends_all_depends_on_all, rosdistro_obj.released_stacks, 'release-tar')
00175                 rosinstall_file = '%s_depends_all_depends_on_all.rosinstall'%DEPENDS_ON_DIR
00176                 print 'Generating rosinstall file [%s]'%(rosinstall_file)
00177                 print 'Contents:\n\n'+rosinstall+'\n\n'
00178                 with open(rosinstall_file, 'w') as f:
00179                     f.write(rosinstall)
00180                     print 'rosinstall file [%s] generated'%(rosinstall_file)
00181                 call('rosinstall --rosdep-yes %s /opt/ros/%s %s %s'%(DEPENDS_ON_DIR, options.rosdistro, STACK_DIR, rosinstall_file), env,
00182                      'Install dependencies of depends_on_all stacks, excluding dependencies of test stacks.')
00183         else:
00184             print "No dependencies of depends_on_all stacks"
00185             
00186 
00187         # Install all stacks that depend on this stack from source
00188         if len(depends_on_all) > 0:
00189             print 'Installing depends_on_all stacks from source: %s'%str(depends_on_all)
00190             rosinstall = stacks_to_rosinstall(depends_on_all, rosdistro_obj.released_stacks, 'release-tar')
00191             rosinstall_file = '%s.rosinstall'%DEPENDS_ON_DIR
00192             print 'Generating rosinstall file [%s]'%(rosinstall_file)
00193             print 'Contents:\n\n'+rosinstall+'\n\n'
00194             with open(rosinstall_file, 'w') as f:
00195                 f.write(rosinstall)
00196                 print 'rosinstall file [%s] generated'%(rosinstall_file)
00197             call('rosinstall --rosdep-yes %s /opt/ros/%s %s %s'%(DEPENDS_ON_DIR, options.rosdistro, STACK_DIR, rosinstall_file), env,
00198                  'Install the stacks that depend on the stacks that are getting tested from source.')
00199 
00200             # Run hudson helper for all stacks
00201             print 'Running Hudson Helper'
00202             env['ROS_TEST_RESULTS_DIR'] = env['ROS_TEST_RESULTS_DIR'] + '/' + DEPENDS_ON_DIR
00203             helper = subprocess.Popen(('./hudson_helper --dir-test %s build'%DEPENDS_ON_DIR).split(' '), env=env)
00204             helper.communicate()
00205             return helper.returncode
00206         else:
00207             print "No stacks depends on this stack. Tests finished"
00208 
00209 
00210     # global except
00211     except Exception, ex:
00212         print "Global exception caught. Generating email"
00213         generate_email("%s. Check the console output for test failure details."%ex, env)
00214         traceback.print_exc(file=sys.stdout)
00215         raise ex
00216 
00217 
00218 if __name__ == '__main__':
00219     try:
00220         res = main()
00221         sys.exit( res )
00222     except Exception, ex:
00223         sys.exit(-1)
00224 
00225 
00226 
00227 
00228 
00229
run_auto_stack_prerelease_devel.py
