job_generation: jobs_common.py Source File

00001 #!/usr/bin/python
00002 
00003 import roslib; roslib.load_manifest("job_generation")
00004 import os
00005 import optparse
00006 import rosdistro
00007 import rospkg
00008 import rospkg.distro
00009 import hudson
00010 import urllib
00011 import time
00012 import subprocess
00013 import yaml
00014 
00015 BOOTSTRAP_SCRIPT = """
00016 cat &gt; $WORKSPACE/script.sh &lt;&lt;DELIM
00017 #!/usr/bin/env bash
00018 set -o errexit
00019 echo "_________________________________BEGIN SCRIPT______________________________________"
00020 sudo apt-get install bzr --yes
00021 sudo apt-get install ros-ROSDISTRO-ros --yes
00022 source /opt/ros/ROSDISTRO/setup.sh
00023 
00024 export INSTALL_DIR=/tmp/install_dir
00025 export WORKSPACE=/tmp/ros
00026 export ROS_TEST_RESULTS_DIR=/tmp/ros/test_results
00027 export JOB_NAME=$JOB_NAME
00028 export BUILD_NUMBER=$BUILD_NUMBER
00029 export HUDSON_URL=$HUDSON_URL
00030 export ROS_PACKAGE_PATH=\$INSTALL_DIR/ros_release:\$ROS_PACKAGE_PATH
00031 
00032 mkdir -p \$INSTALL_DIR
00033 cd \$INSTALL_DIR
00034 
00035 wget  --no-check-certificate http://code.ros.org/svn/ros/installers/trunk/hudson/hudson_helper 
00036 chmod +x  hudson_helper
00037 svn co https://code.ros.org/svn/ros/stacks/ros_release/trunk ros_release
00038 """
00039 
00040 SHUTDOWN_SCRIPT = """
00041 echo "_________________________________END SCRIPT_______________________________________"
00042 DELIM
00043 
00044 set -o errexit
00045 
00046 rm -rf $WORKSPACE/test_results
00047 rm -rf $WORKSPACE/test_output
00048 
00049 wget  --no-check-certificate https://code.ros.org/svn/ros/stacks/ros_release/trunk/hudson/scripts/run_chroot.py -O $WORKSPACE/run_chroot.py
00050 chmod +x $WORKSPACE/run_chroot.py
00051 cd $WORKSPACE &amp;&amp; $WORKSPACE/run_chroot.py --distro=UBUNTUDISTRO --arch=ARCH  --ramdisk --hdd-scratch=/home/rosbuild/install_dir --script=$WORKSPACE/script.sh --ssh-key-file=/home/rosbuild/rosbuild-ssh.tar
00052 """
00053 
00054 BOOTSTRAP_SCRIPT_OSX = """
00055 echo "_________________________________BEGIN SCRIPT______________________________________"
00056 source /Users/rosbuild/ros_bootstrap/setup.bash
00057 export ROS_PACKAGE_PATH=$WORKSPACE/ros_release:$ROS_PACKAGE_PATH
00058 
00059 wget  --no-check-certificate http://code.ros.org/svn/ros/installers/trunk/hudson/hudson_helper -O $WORKSPACE/hudson_helper
00060 chmod +x  $WORKSPACE/hudson_helper
00061 svn co https://code.ros.org/svn/ros/stacks/ros_release/trunk $WORKSPACE/ros_release
00062 """
00063 
00064 SHUTDOWN_SCRIPT_OSX = """
00065 echo "_________________________________END SCRIPT_______________________________________"
00066 """
00067 
00068 
00069 # the supported Ubuntu distro's for each ros distro
00070 ARCHES = ['amd64', 'i386']
00071 
00072 # ubuntu distro mapping to ros distro
00073 UBUNTU_DISTRO_MAP = os_test_platform = {
00074     'testing': ['lucid', 'maverick'],
00075     'unstable': ['lucid', 'oneiric'],
00076     'electric': ['lucid', 'maverick', 'natty', 'oneiric'],
00077     'diamondback': ['lucid', 'maverick', 'natty'],
00078     'cturtle': ['lucid', 'maverick', 'karmic'],
00079 }
00080 
00081 # Path to hudson server
00082 SERVER = 'http://build.willowgarage.com'
00083 #SERVER = 'http://hudson.willowgarage.com:8080'
00084 
00085 # config path
00086 CONFIG_PATH = 'http://wgs24.willowgarage.com/hudson-html/hds.xml'
00087 
00088 
00089 EMAIL_TRIGGER="""
00090         <hudson.plugins.emailext.plugins.trigger.WHENTrigger> 
00091           <email> 
00092             <recipientList></recipientList> 
00093             <subject>$PROJECT_DEFAULT_SUBJECT</subject> 
00094             <body>$PROJECT_DEFAULT_CONTENT</body> 
00095             <sendToDevelopers>SEND_DEVEL</sendToDevelopers> 
00096             <sendToRecipientList>true</sendToRecipientList> 
00097             <contentTypeHTML>false</contentTypeHTML> 
00098             <script>true</script> 
00099           </email> 
00100         </hudson.plugins.emailext.plugins.trigger.WHENTrigger> 
00101 """
00102 
00103 
00104 hudson_scm_managers = {'svn':"""
00105   <scm class="hudson.scm.SubversionSCM"> 
00106     <locations> 
00107       <hudson.scm.SubversionSCM_-ModuleLocation> 
00108         <remote>STACKURI</remote> 
00109         <local>STACKNAME</local> 
00110       </hudson.scm.SubversionSCM_-ModuleLocation> 
00111     </locations> 
00112     <useUpdate>false</useUpdate> 
00113     <doRevert>false</doRevert> 
00114     <excludedRegions></excludedRegions> 
00115     <includedRegions></includedRegions> 
00116     <excludedUsers></excludedUsers> 
00117     <excludedRevprop></excludedRevprop> 
00118     <excludedCommitMessages></excludedCommitMessages> 
00119   </scm> 
00120 """,
00121                        'hg':"""
00122   <scm class="hudson.plugins.mercurial.MercurialSCM">
00123     <source>STACKURI</source>
00124     <modules></modules>
00125     <subdir>STACKNAME</subdir>
00126     <clean>false</clean>
00127     <forest>false</forest>
00128     <branch>STACKBRANCH</branch>
00129   </scm>
00130 """,
00131                        'bzr':"""
00132   <scm class="hudson.plugins.bazaar.BazaarSCM"> 
00133     <source>STACKURI STACKNAME</source> 
00134     <clean>false</clean> 
00135   </scm> 
00136 """,
00137                        'git':"""
00138 
00139   <scm class="hudson.plugins.git.GitSCM">
00140     <configVersion>1</configVersion>
00141     <remoteRepositories>
00142       <org.spearce.jgit.transport.RemoteConfig>
00143         <string>origin</string>
00144         <int>5</int>
00145 
00146         <string>fetch</string>
00147         <string>+refs/heads/*:refs/remotes/origin/*</string>
00148         <string>receivepack</string>
00149         <string>git-upload-pack</string>
00150         <string>uploadpack</string>
00151         <string>git-upload-pack</string>
00152 
00153         <string>url</string>
00154         <string>STACKURI</string>
00155         <string>tagopt</string>
00156         <string></string>
00157       </org.spearce.jgit.transport.RemoteConfig>
00158     </remoteRepositories>
00159     <branches>
00160 
00161       <hudson.plugins.git.BranchSpec>
00162         <name>STACKBRANCH</name>
00163       </hudson.plugins.git.BranchSpec>
00164     </branches>
00165     <localBranch></localBranch>
00166     <mergeOptions/>
00167     <recursiveSubmodules>false</recursiveSubmodules>
00168     <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
00169 
00170     <authorOrCommitter>Hudson</authorOrCommitter>
00171     <clean>false</clean>
00172     <wipeOutWorkspace>false</wipeOutWorkspace>
00173     <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
00174     <gitTool>Default</gitTool>
00175     <submoduleCfg class="list"/>
00176     <relativeTargetDir>STACKNAME</relativeTargetDir>
00177 
00178     <excludedRegions></excludedRegions>
00179     <excludedUsers></excludedUsers>
00180   </scm>
00181 """
00182 }
00183 
00184 def stack_to_deb(stack, rosdistro):
00185     return '-'.join(['ros', rosdistro, str(stack).replace('_','-')])
00186 
00187 def stacks_to_debs(stack_list, rosdistro):
00188     if not stack_list or len(stack_list) == 0:
00189         return ''
00190     return ' '.join([stack_to_deb(s, rosdistro) for s in stack_list])
00191 
00192 
00193 def stack_to_rosinstall(stack_obj, branch):
00194     try:
00195         return yaml.dump(rosdistro.stack_to_rosinstall(stack_obj, branch, anonymous=True))
00196     except rosdistro.DistroException, ex:
00197         print str(ex)
00198         return ''
00199 
00200 
00201 def stacks_to_rosinstall(stack_list, stack_map, branch):
00202     res = ''
00203     for s in stack_list:
00204         if s in stack_map:
00205             res += stack_to_rosinstall(stack_map[s], branch)
00206         else:
00207             print 'Stack "%s" is not in stack list. Not adding this stack to rosinstall file'%s
00208     return res
00209 
00210     
00211 
00212 def get_depends_one(stack):
00213     name = '%s-%s'%(stack.name, stack.version)
00214     url = urllib.urlopen('https://code.ros.org/svn/release/download/stacks/%s/%s/%s.yaml'%(stack.name, name, name))
00215     conf = url.read()
00216     if '404 Not Found' in conf:
00217         print 'Could not get dependencies of stack %s'%stack.name
00218         return []
00219     depends = yaml.load(conf)['depends']
00220     if depends:
00221         return depends
00222     else:
00223         print 'Stack %s does not have any dependencies'%stack.name
00224         return []
00225 
00226 def get_depends_all(distro_obj, stack_name, depends_all):
00227     start_depth = len(depends_all)
00228     print start_depth, " depends all ", stack_name
00229     if not stack_name in depends_all:
00230         depends_all.append(stack_name)
00231         try:
00232             for d in get_depends_one(distro_obj.stacks[stack_name]):
00233                 get_depends_all(distro_obj, d, depends_all)
00234         except KeyError, ex:
00235             print "Exception when processing %s.  Key %s is not in distro_obj.stacks: %s"%(stack_name, ex, ", ".join([s for s in distro_obj.stacks]))
00236             print "depends_all is %s"%(', '.join(depends_all))
00237             raise ex
00238     print start_depth, " DEPENDS_ALL ", stack_name, " end depth ", len(depends_all)
00239 
00240 def get_environment():
00241     my_env = os.environ
00242     my_env['WORKSPACE'] = os.getenv('WORKSPACE', '')
00243     my_env['INSTALL_DIR'] = os.getenv('INSTALL_DIR', '')
00244     #my_env['HOME'] = os.getenv('HOME', '')
00245     my_env['HOME'] = os.path.expanduser('~')
00246     my_env['JOB_NAME'] = os.getenv('JOB_NAME', '')
00247     my_env['BUILD_NUMBER'] = os.getenv('BUILD_NUMBER', '')
00248     my_env['ROS_TEST_RESULTS_DIR'] = os.getenv('ROS_TEST_RESULTS_DIR', my_env['WORKSPACE']+'/test_results')
00249     my_env['PWD'] = os.getenv('WORKSPACE', '')
00250     return my_env
00251 
00252 
00253 def get_options(required, optional):
00254     parser = optparse.OptionParser()
00255     print "required: ", required, "optional: ", optional
00256     ops = required + optional
00257     print "parsing opts:", ops, "done"
00258     
00259     if 'os' in ops:
00260         parser.add_option('--os', dest = 'os', default='ubuntu', action='store',
00261                           help='OS name')
00262     if 'rosdistro' in ops:
00263         parser.add_option('--rosdistro', dest = 'rosdistro', default=None, action='store',
00264                           help='Ros distro name')
00265     if 'stack' in ops:
00266         parser.add_option('--stack', dest = 'stack', default=None, action='append',
00267                           help='Stack name')
00268     if 'email' in ops:
00269         parser.add_option('--email', dest = 'email', default=None, action='store',
00270                           help='Email address to send results to')
00271     if 'arch' in ops:
00272         parser.add_option('--arch', dest = 'arch', default=None, action='append',
00273                           help='Architecture to test')
00274     if 'ubuntu' in ops:
00275         parser.add_option('--ubuntu', dest = 'ubuntu', default=None, action='append',
00276                           help='Ubuntu distribution to test')
00277     if 'repeat' in ops:
00278         parser.add_option('--repeat', dest = 'repeat', default=0, action='store',
00279                           help='How many times to repeat the test')
00280     if 'source-only' in ops:
00281         parser.add_option('--source-only', dest = 'source_only', default=False, action='store_true',
00282                           help="Build everything from source, don't use Debian packages")
00283     if 'delete' in ops:
00284         parser.add_option('--delete', dest = 'delete', default=False, action='store_true',
00285                           help='Delete jobs from Hudson')    
00286     if 'wait' in ops:
00287         parser.add_option('--wait', dest = 'wait', default=False, action='store_true',
00288                           help='Wait for running jobs to finish to reconfigure them')    
00289     if 'rosinstall' in ops:
00290         parser.add_option('--rosinstall', dest = 'rosinstall', default=None, action='store',
00291                           help="Specify the rosinstall file that refers to unreleased code.")
00292     if 'overlay' in ops:
00293         parser.add_option('--overlay', dest = 'overlay', default=None, action='store',
00294                           help='Create overlay file')
00295     if 'variant' in ops:
00296         parser.add_option('--variant', dest = 'variant', default=None, action='store',
00297                           help="Specify the variant to create a rosinstall for")
00298     if 'database' in ops:
00299         parser.add_option('--database', dest = 'database', default=None, action='store',
00300                           help="Specify database file")
00301 
00302     print "done setting up parser"
00303 
00304     (options, args) = parser.parse_args()
00305     print "run parser"
00306 
00307     # make repeat an int
00308     if 'repeat' in ops:
00309         options.repeat = int(options.repeat)
00310 
00311     # check if required arguments are there
00312     for r in required:
00313         if not eval('options.%s'%r):
00314             print 'You need to specify "--%s"'%r
00315             return (None, args)
00316 
00317     # postprocessing
00318     if 'email' in ops and options.email and not '@' in options.email:
00319         options.email = options.email + '@willowgarage.com'        
00320 
00321 
00322     print "check if the rosdistro exists:"
00323     # check if rosdistro exists
00324     if 'rosdistro' in ops and (not options.rosdistro or not options.rosdistro in UBUNTU_DISTRO_MAP.keys()):
00325         print 'You provided an invalid "--rosdistro %s" argument. Options are %s'%(options.rosdistro, UBUNTU_DISTRO_MAP.keys())
00326         return (None, args)
00327 
00328     print "check if the stack exists"
00329     # check if stacks exist
00330     if 'stack' in ops and options.stack:
00331         distro_obj = rospkg.distro.load_distro(rospkg.distro.distro_uri(options.rosdistro))
00332         for s in options.stack:
00333             if not s in distro_obj.stacks:
00334                 print 'Stack "%s" does not exist in the %s distro file.'%(s, options.rosdistro)
00335                 print 'You need to add this stack to the rosdistro file'
00336                 return (None, args)
00337 
00338     print "checking for variants"
00339     # check if variant exists
00340     if 'variant' in ops and options.variant:
00341         distro_obj = rospkg.distro.load_distro(rospkg.distro.distro_uri(options.rosdistro))
00342         if not options.variant in distro_obj.variants:
00343                 print 'Variant "%s" does not exist in the %s distro file.'%(options.variant, options.rosdistro)
00344                 return (None, args)
00345 
00346     return (options, args)
00347 
00348 
00349 def schedule_jobs(jobs, wait=False, delete=False, start=False, hudson_obj=None):
00350     # create hudson instance
00351     if not hudson_obj:
00352         info = urllib.urlopen(CONFIG_PATH).read().split(',')
00353         hudson_obj = hudson.Hudson(SERVER, info[0], info[1])
00354 
00355     finished = False
00356     while not finished:
00357         jobs_todo = {}
00358         for job_name in jobs:
00359             exists = hudson_obj.job_exists(job_name)
00360 
00361             # job is already running
00362             if exists and hudson_obj.job_is_running(job_name):
00363                 jobs_todo[job_name] = jobs[job_name]
00364                 print "Not reconfiguring running job %s because it is still running"%job_name
00365 
00366 
00367             # delete old job
00368             elif delete:
00369                 if exists:
00370                     hudson_obj.delete_job(job_name)
00371                     print " - Deleting job %s"%job_name
00372 
00373             # reconfigure job
00374             elif exists:
00375                 hudson_obj.reconfig_job(job_name, jobs[job_name])
00376                 if start:
00377                     hudson_obj.build_job(job_name)
00378                 print " - %s"%job_name
00379 
00380             # create job
00381             elif not exists:
00382                 hudson_obj.create_job(job_name, jobs[job_name])
00383                 if start:
00384                     hudson_obj.build_job(job_name)
00385                 print " - %s"%job_name
00386 
00387         if wait and len(jobs_todo) > 0:
00388             jobs = jobs_todo
00389             jobs_todo = {}
00390             time.sleep(10.0)
00391         else:
00392             finished = True
00393 
00394 
00395 
00396 def get_rosdistro_file(rosdistro):
00397     return 'https://code.ros.org/svn/release/trunk/distros/%s.rosdistro'%rosdistro
00398 
00399 
00400 
00401 def get_email_triggers(when, send_devel=True):
00402     triggers = ''
00403     for w in when:
00404         trigger = EMAIL_TRIGGER
00405         trigger = trigger.replace('WHEN', w)
00406         if send_devel:
00407             trigger = trigger.replace('SEND_DEVEL', 'true')
00408         else:
00409             trigger = trigger.replace('SEND_DEVEL', 'false')
00410         triggers += trigger
00411     return triggers
00412 
00413 
00414 def get_job_name(jobtype, rosdistro, stack_name, ubuntu, arch):
00415     if len(stack_name) > 50:
00416         stack_name = stack_name[0:46]+'_...'
00417     return "_".join([jobtype, rosdistro, stack_name, ubuntu, arch])
00418 
00419 
00420 def ensure_dir(f):
00421     d = os.path.dirname(f)
00422     if not os.path.exists(d):
00423         os.makedirs(d)
00424 
00425 def write_file(filename, msg):
00426     ensure_dir(filename)
00427     with open(filename, 'w') as f:
00428         f.write(msg)
00429     
00430 
00431 def generate_email(message, env):
00432     print message
00433     write_file(env['WORKSPACE']+'/build_output/buildfailures.txt', message)
00434     write_file(env['WORKSPACE']+'/test_output/testfailures.txt', '')
00435     write_file(env['WORKSPACE']+'/build_output/buildfailures-with-context.txt', '')
00436     
00437 
00438 
00439 def call(command, env=None, message='', ignore_fail=False):
00440     res = ''
00441     err = ''
00442     try:
00443         print message+'\nExecuting command "%s"'%command
00444         helper = subprocess.Popen(command.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, env=env)
00445         res, err = helper.communicate()
00446         print str(res)
00447         print str(err)
00448         if helper.returncode != 0:
00449             raise Exception
00450         return res
00451     except Exception:
00452         if not ignore_fail:
00453             message += "\n=========================================\n"
00454             message += "Failed to execute '%s'"%command
00455             message += "\n=========================================\n"
00456             message += str(res)
00457             message += "\n=========================================\n"
00458             message += str(err)
00459             message += "\n=========================================\n"
00460             if env:
00461                 message += "ROS_PACKAGE_PATH = %s\n"%env['ROS_PACKAGE_PATH']
00462                 message += "ROS_ROOT = %s\n"%env['ROS_ROOT']
00463                 message += "PYTHONPATH = %s\n"%env['PYTHONPATH']
00464                 message += "\n=========================================\n"
00465                 generate_email(message, env)
00466             raise Exception
00467 
00468         
00469 def get_sys_info():
00470     arch = 'i386'
00471     if '64' in call('uname -mrs'):
00472         arch = 'amd64'
00473     ubuntudistro = call('lsb_release -a').split('Codename:')[1].strip()
00474     return (arch, ubuntudistro)