rosdistro

1 # Software License Agreement (BSD License) 2 # 3 # Copyright (c) 2010, Willow Garage, Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions 8 # are met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following 14 # disclaimer in the documentation and/or other materials provided 15 # with the distribution. 16 # * Neither the name of Willow Garage, Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived 18 # from this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 30 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 # POSSIBILITY OF SUCH DAMAGE. 32 # 33 # Revision $Id: distro.py 10301 2010-07-09 01:21:23Z kwc $ 34 35 """ 36 Library for process rosdistro files. 37 38 New in ROS C-Turtle 39 """ 40 41 import os 42 import sys 43 import yaml 44 import string 45 import subprocess 46

47 -class DistroException(Exception): pass

48

49 -def distro_uri(distro_name):

50 """ 51 @param distro_name: name of distro, e.g. 'diamondback' 52 @return: the SVN/HTTP URL of the specified distro. This function should only be used 53 with the main distros. 54 """ 55 return "https://code.ros.org/svn/release/trunk/distros/%s.rosdistro"%(distro_name)

56

57 -def distro_version(version_val):

58 """Parse distro version value, converting SVN revision to version value if necessary""" 59 import re 60 version_val = str(version_val) 61 m = re.search('\$Revision:\s*([0-9]*)\s*\$', version_val) 62 if m is not None: 63 version_val = 'r'+m.group(1) 64 65 # Check that is a valid version string 66 valid = string.ascii_letters + string.digits + '.+~' 67 if False in (c in valid for c in version_val): 68 raise DistroException("Version string %s not valid"%version_val) 69 return version_val

70

71 -def expand_rule(rule, stack_name, stack_ver, release_name, revision=None):

72 s = rule.replace('$STACK_NAME', stack_name) 73 if stack_ver: 74 s = s.replace('$STACK_VERSION', stack_ver) 75 s = s.replace('$RELEASE_NAME', release_name) 76 if s.find('$REVISION') > 0 and not revision: 77 raise DistroException("revision specified but not supplied by build_release") 78 elif revision: 79 s = s.replace('$REVISION', revision) 80 return s

81

82 -def load_vcs_config(rules, rule_eval):

83 vcs_config = None 84 if 'svn' in rules or 'dev-svn' in rules: 85 #legacy support 86 if 'dev-svn' in rules: 87 vcs_config = SvnConfig() 88 vcs_config.dev = rule_eval(rules['dev-svn']) 89 vcs_config.distro_tag = rule_eval(rules['distro-svn']) 90 vcs_config.release_tag = rule_eval(rules['release-svn']) 91 vcs_config.anon_dev = vcs_config.dev 92 vcs_config.anon_distro_tag = vcs_config.distro_tag 93 vcs_config.anon_release_tag = vcs_config.release_tag 94 95 elif 'svn' in rules: 96 vcs_config = SvnConfig() 97 r = rules['svn'] 98 for k in ['dev', 'distro-tag', 'release-tag']: 99 if not k in r: 100 raise KeyError("svn rules missing required %s key: %s"%(k, r)) 101 vcs_config.dev = rule_eval(r['dev']) 102 vcs_config.distro_tag = rule_eval(r['distro-tag']) 103 vcs_config.release_tag = rule_eval(r['release-tag']) 104 105 # specify urls that are safe to anonymously read 106 # from. Users must supply a complete set. 107 if 'anon-dev' in r: 108 vcs_config.anon_dev = rule_eval(r['anon-dev']) 109 vcs_config.anon_distro_tag = rule_eval(r['anon-distro-tag']) 110 vcs_config.anon_release_tag = rule_eval(r['anon-release-tag']) 111 else: 112 # if no login credentials, assume that anonymous is 113 # same as normal keys. 114 vcs_config.anon_dev = vcs_config.dev 115 vcs_config.anon_distro_tag = vcs_config.distro_tag 116 vcs_config.anon_release_tag = vcs_config.release_tag 117 118 119 120 121 elif 'hg' in rules or 'git' in rules or 'bzr' in rules: 122 r = None 123 if 'hg' in rules: 124 vcs_config = HgConfig() 125 r = rules['hg'] 126 127 elif 'git' in rules: 128 vcs_config = GitConfig() 129 r = rules['git'] 130 131 elif 'bzr' in rules: 132 vcs_config = BZRConfig() 133 r = rules['bzr'] 134 135 if not r: 136 raise NotImplementedError("Rules %s not implemented"%rules) 137 138 vcs_config.repo_uri = rule_eval(r['uri']) 139 140 if 'anon-uri' in r: 141 vcs_config.anon_repo_uri = rule_eval(r['anon-uri']) 142 else: 143 vcs_config.anon_repo_uri = vcs_config.repo_uri 144 145 vcs_config.dev_branch = rule_eval(r['dev-branch']) 146 vcs_config.distro_tag = rule_eval(r['distro-tag']) 147 vcs_config.release_tag = rule_eval(r['release-tag']) 148 149 return vcs_config

150

151 -def get_variants(distro, stack_name):

152 """ 153 Retrieve names of variants that stack is present in. This operates 154 on the raw distro dictionary document. 155 156 @param distro: rosdistro document 157 @type distro: dict 158 """ 159 if stack_name == 'ROS': 160 stack_name = 'ros' 161 162 retval = [] 163 variants = distro.get('variants', {}) 164 165 for variant_d in variants: 166 try: 167 variant = variant_d.keys()[0] 168 variant_props = variant_d[variant] 169 if stack_name in variant_props['stacks']: 170 if variant not in retval: 171 retval.append(variant) 172 except: 173 pass 174 175 # process extends 176 for variant_d in variants: 177 try: 178 variant = variant_d.keys()[0] 179 variant_props = variant_d[variant] 180 if 'extends' in variant_props: 181 extends = variant_props['extends'] 182 if type(extends) in (str, unicode): 183 # single variant: backwards compat 184 if extends in retval and variant not in retval: 185 retval.append(variant) 186 187 else: 188 # list of variants 189 if set(extends) ^ set(retval) and variant not in retval: 190 retval.append(variant) 191 except: 192 pass 193 return retval

194 195 # TODO: integrate with Distro

196 -def get_rules(distro, stack_name):

197 """ 198 Retrieve rules from distro for specified stack This operates on 199 the raw distro dictionary document. 200 201 @param distro: rosdistro document 202 @type distro: dict 203 @param stack_name: name of stack to get rules for 204 @type stack_name: str 205 """ 206 207 if stack_name == 'ROS': 208 stack_name = 'ros' 209 210 # _rules: named section 211 named_rules_d = distro.get('_rules', {}) 212 213 # there are three tiers of dictionaries that we look in for uri rules 214 rules_d = [distro.get('stacks', {}), 215 distro.get('stacks', {}).get(stack_name, {})] 216 rules_d = [d for d in rules_d if d] 217 218 # load the '_rules' from the dictionaries, in order 219 props = {} 220 for d in rules_d: 221 if type(d) == dict: 222 update_r = d.get('_rules', {}) 223 if type(update_r) == str: 224 try: 225 update_r = named_rules_d[update_r] 226 except KeyError: 227 raise DistroException("no _rules named [%s]"%(update_r)) 228 229 new_style = True 230 for k in ['distro-svn', 'release-svn', 'dev-svn']: 231 if k in update_r: 232 new_style = False 233 if new_style: 234 # in new style, we do not do additive rules 235 if not type(update_r) == dict: 236 raise Exception("invalid rules: %s %s"%(d, type(d))) 237 # ignore empty definition 238 if update_r: 239 props = update_r 240 else: 241 # legacy: rules overlay higher level rules 242 if not type(update_r) == dict: 243 raise Exception("invalid rules: %s %s"%(d, type(d))) 244 props.update(update_r) 245 246 if not props: 247 raise Exception("cannot load _rules") 248 return props

249 250

251 -def load_distro_stacks(distro_doc, stack_names, release_name=None, version=None):

252 """ 253 @param distro_doc: dictionary form of rosdistro file 254 @type distro_doc: dict 255 @param stack_names: names of stacks to load 256 @type stack_names: [str] 257 @param release_name: (optional) name of distro release to override distro_doc spec. 258 @type release_name: str 259 @param version: (optional) distro version to override distro_doc spec. 260 @type version: str 261 @return: dictionary of stack names to DistroStack instances 262 @rtype: {str : DistroStack} 263 @raise DistroException: if distro_doc format is invalid 264 """ 265 266 # load stacks and expand out uri rules 267 stacks = {} 268 # we pass these in mostly for small performance reasons, as well as testing 269 if version is None: 270 version = distro_version(distro_doc.get('version', '0')) 271 if release_name is None: 272 release_name = distro_doc['release'] 273 274 try: 275 stack_props = distro_doc['stacks'] 276 except KeyError: 277 raise DistroException("distro is missing required 'stacks' key") 278 for stack_name in stack_names: 279 # ignore private keys like _rules 280 if stack_name[0] == '_': 281 continue 282 283 stack_version = stack_props[stack_name].get('version', None) 284 rules = get_rules(distro_doc, stack_name) 285 stacks[stack_name] = DistroStack(stack_name, rules, stack_version, release_name, version) 286 return stacks

287

288 -class DistroStack(object):

289 """Stores information about a stack release""" 290

291 - def __init__(self, stack_name, rules, stack_version, release_name, release_version):

292 self.name = stack_name 293 self.release_name = release_name 294 self.release_version = release_version 295 296 self._rules = rules 297 298 self.update_version(stack_version) 299 self.repo = rules.get('repo', None)

300 301

302 - def update_version(self, stack_version):

303 rules = self._rules 304 self.version = stack_version 305 306 #rosdistro key 307 # - for future SCM rules, we need to put them in a more 308 # general representation. Leaving the SVN representation 309 # as-is so as to not disturb existing scripts. 310 self.dev_svn = self.distro_svn = self.release_svn = None 311 self.vcs_config = load_vcs_config(rules, self.expand_rule) 312 313 # legacy support, remove once we stop building box turtle 314 if 'svn' in rules or 'dev-svn' in rules: 315 self.dev_svn = self.vcs_config.dev 316 self.distro_svn = self.vcs_config.distro_tag 317 self.release_svn = self.vcs_config.release_tag

318 319

320 - def expand_rule(self, rule):

321 """ 322 Perform variable substitution on stack rule. 323 """ 324 return expand_rule(rule, self.name, self.version, self.release_name)

325

326 - def __eq__(self, other):

327 if not isinstance(other, DistroStack): 328 return False 329 return self.name == other.name and \ 330 self.version == other.version and \ 331 self.vcs_config == other.vcs_config

332

333 -class Variant(object):

334 """ 335 A variant defines a specific set of stacks ("metapackage", in Debian 336 parlance). For example, "base", "pr2". These variants can extend 337 another variant. 338 """ 339

340 - def __init__(self, variant_name, variants_props):

341 """ 342 @param variant_name: name of variant to load from distro file 343 @type variant_name: str 344 @param variants_props: dictionary mapping variant names to the rosdistro map for that variant 345 """ 346 self.name = variant_name 347 self.parents = [] 348 349 # save the properties for our particular variant 350 try: 351 props = variants_props[variant_name] 352 except: 353 raise DistroException("distro does not define a '%s' variant"%variant_name) 354 355 # load in variant properties from distro spec 356 if not 'stacks' in props and not 'extends' in props: 357 raise DistroException("variant properties must define 'stacks' or 'extends':\n%s"%(props)) 358 359 # stack_names accumulates the full expanded list 360 self.stack_names = list(props.get('stacks', [])) 361 # stack_names_explicit is only the stack names directly specified 362 self.stack_names_explicit = self.stack_names 363 364 # check to see if we extend another distro, in which case we prepend their props 365 if 'extends' in props: 366 extends = props['extends'] 367 if type(extends) == str: 368 extends = [extends] 369 # store parents property for debian metapackages 370 self.parents = extends 371 372 for e in extends: 373 parent_variant = Variant(e, variants_props) 374 self.stack_names = parent_variant.stack_names + self.stack_names 375 self.props = props

376

377 -class Distro(object):

378 """ 379 Store information in a rosdistro file. 380 """ 381

382 - def __init__(self, source_uri):

383 """ 384 @param source_uri: source URI of distro file, or path to distro file 385 """ 386 # initialize members 387 self.source_uri = source_uri 388 389 self.ros = None 390 self.stacks = {} # {str: DistroStack} 391 self.stack_names = [] 392 self.released_stacks = {} # {str: DistroStack} 393 self.variants = {} 394 self.distro_props = None 395 396 try: 397 # parse rosdistro yaml 398 if os.path.isfile(source_uri): 399 # load rosdistro file 400 with open(source_uri) as f: 401 y = yaml.load(f.read()) 402 elif 'code.ros.org/svn' in source_uri: 403 # Create a temp directory and fetch via svn export so 404 # that keywords are evaluated. The test for 405 # code.ros.org is a big hack. 406 import shutil 407 import tempfile 408 409 tmp_dir = tempfile.mkdtemp() 410 tmp_distro_file = os.path.join(tmp_dir, os.path.split(source_uri)[-1]) 411 p = subprocess.Popen(['svn','export',source_uri,tmp_distro_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 412 p.communicate() 413 if p.returncode != 0: 414 raise Exception("svn export [%s] failed"%(source_uri)) 415 with open(tmp_distro_file) as f: 416 y = yaml.load(f.read()) 417 shutil.rmtree(tmp_dir) 418 else: 419 import urllib2 420 y = yaml.load(urllib2.urlopen(source_uri)) 421 422 self.distro_props = y 423 424 stack_props = y['stacks'] 425 self.stack_names = [x for x in stack_props.keys() if not x[0] == '_'] 426 self.version = distro_version(y.get('version', '0')) 427 self.release_name = y['release'] 428 429 variants = {} 430 for props in y['variants']: 431 if len(props.keys()) != 1: 432 raise DistroException("invalid variant spec: %s"%props) 433 n = props.keys()[0] 434 variants[n] = props[n] 435 436 except KeyError, e: 437 raise DistroException("this program assumes the yaml has a '%s' map"%(str(e))) 438 439 # load variants 440 for v in variants.iterkeys(): 441 self.variants[v] = Variant(v, variants) 442 443 if not 'ros' in stack_props: 444 raise DistroException("this program assumes that ros is in your variant") 445 446 self.stacks = load_distro_stacks(self.distro_props, self.stack_names, release_name=self.release_name, version=self.version) 447 for s, obj in self.stacks.iteritems(): 448 if obj.version: 449 self.released_stacks[s] = obj 450 self.ros = self.stacks.get('ros', None)

451 452 453 # TODO Ken's suggested restructuring template 454 #mappings_dvcs = {} 455 #vcs_maps = { 456 # 'svn-anon': mappings_svn_anon, 457 # 'svn': mappings_svn, 458 # 'git': mappings_dvcs, 459 # 'hg': mappings_dvcs, 460 #} 461 #if vcs.type == 'svn': 462 # if anon: 463 # mapping 464 #else: 465 # mappigns = vcs_maps[vcs.type] 466 #uri = getattr(vcs, d[branch]) 467

468 -def stack_to_rosinstall(stack, branch, anonymous=True):

469 """ 470 Generate the rosinstall dictionary entry for a stack in the 471 rosdistro. 472 473 @param stack: A DistroStack for a particular stack 474 @type stack: L{DistroStack} 475 @param branch: Select the branch or tag from 'devel', 'release' or 'distro' of the stack to checkout 476 @type branch: str 477 @param anonymous: If True, use anonymous-access URLs if available (optional, default True). 478 @type anonymous: bool 479 """ 480 result = [] 481 482 uri = None 483 version = stack.version 484 if not version and not branch == 'devel': 485 print "Stack %s at version null, skipping"%stack.name 486 print "Can only get 'deve' branch from a stack at version null." 487 return result 488 489 version_tag = None # to be conditionally filled later 490 491 492 vcs = stack.vcs_config 493 if not branch in ['devel', 'release', 'distro', 'release-tar']: 494 raise DistroException('Unsupported branch type %s for stack %s'%(branch, stack.name)) 495 if not vcs.type in ['svn', 'git', 'bzr', 'hg']: 496 raise DistroException( 'Unsupported vcs type %s for stack %s'%(vcs.type, stack.name)) 497 498 499 if branch == 'release-tar': 500 def get_tar(stack): 501 name = '%s-%s'%(stack.name, stack.version) 502 return 'https://code.ros.org/svn/release/download/stacks/%s/%s/%s.tar.bz2'%(stack.name, name, name)

503 uri = get_tar(stack) 504 version_tag = '%s-%s'%(stack.name, stack.version) 505 vcs_type = 'tar' 506 507 else: 508 vcs_type = vcs.type 509 if vcs.type in ['svn']: 510 if branch == 'devel': 511 if anonymous: 512 uri = vcs.anon_dev 513 else: 514 uri = vcs.dev 515 #return "- svn: {uri: '%s', local-name: '%s'}\n"%(vcs.anon_dev, stack.name) 516 elif branch == 'distro': 517 if anonymous: 518 uri = vcs.anon_distro_tag 519 else: 520 uri = vcs.distro_tag 521 elif branch == 'release': 522 if anonymous: 523 uri = vcs.anon_release_tag 524 else: 525 uri = vcs.release_tag 526 527 elif vcs.type == 'hg' or vcs.type == 'git' or vcs.type == 'bzr': 528 if anonymous and hasattr(vcs, 'anon_repo_uri'): 529 uri = vcs.anon_repo_uri 530 else: 531 uri = vcs.repo_uri 532 if branch == 'devel': 533 version_tag = vcs.dev_branch 534 elif branch == 'distro': 535 version_tag = vcs.distro_tag 536 elif branch == 'release': 537 version_tag = vcs.release_tag 538 539 540 541 if version_tag: 542 result.append({vcs_type: {"uri": uri, 'local-name': stack.name, 'version': version_tag} } ) 543 else: 544 result.append({vcs_type: {"uri": uri, 'local-name': stack.name} } ) 545 return result 546

547 -def variant_to_rosinstall(variant_name, distro, branch, anonymous=True):

548 rosinstall_dict = [] 549 variant = distro.variants.get(variant_name, None) 550 if not variant: 551 return [] 552 done = [] 553 for s in variant.stack_names_explicit: 554 if s in distro.released_stacks and not s in done: 555 done.append(s) 556 rosinstall_dict.extend(stack_to_rosinstall(distro.stacks[s], branch, anonymous)) 557 return rosinstall_dict

558

559 -def extended_variant_to_rosinstall(variant_name, distro, branch, anonymous=True):

560 rosinstall_dict = [] 561 variant = distro.variants.get(variant_name, None) 562 if not variant: 563 return [] 564 done = [] # avoid duplicates 565 for s in variant.stack_names: 566 if s in distro.released_stacks and not s in done: 567 done.append(s) 568 rosinstall_dict.extend(stack_to_rosinstall(distro.stacks[s], branch, anonymous)) 569 return rosinstall_dict

570 571

572 -def distro_to_rosinstall(distro, branch, anonymous=True):

573 rosinstall_dict = [] 574 for s in distro.released_stacks.itervalues(): 575 rosinstall_dict.extend(stack_to_rosinstall(s, branch, anonymous)) 576 return rosinstall_dict

577 578

579 -class BZRConfig(object):

580 """ 581 Configuration information about an BZR repository for a component 582 of code. The configuration we maintain is specific to ROS 583 toolchain concepts and is not a general notion of BZR configuration. 584 585 * repo_uri: base URI of repo 586 * dev_branch: hg branch the code is developed 587 * distro_tag: a tag of the latest released code for a specific ROS distribution 588 * release_tag: a tag of the code for a specific release 589 """ 590

591 - def __init__(self):

592 self.type = 'bzr' 593 self.repo_uri = None 594 self.anon_repo_uri = None 595 self.dev_branch = None 596 self.distro_tag = None 597 self.release_tag = None

598

599 - def __eq__(self, other):

600 return self.repo_uri == other.repo_uri and \ 601 self.anon_repo_uri == other.anon_repo_uri and \ 602 self.dev_branch == other.dev_branch and \ 603 self.release_tag == other.release_tag and \ 604 self.distro_tag == other.distro_tag

605

606 -class HgConfig(object):

607 """ 608 Configuration information about an SVN repository for a component 609 of code. The configuration we maintain is specific to ROS 610 toolchain concepts and is not a general notion of SVN configuration. 611 612 * repo_uri: base URI of repo 613 * dev_branch: hg branch the code is developed 614 * distro_tag: a tag of the latest released code for a specific ROS distribution 615 * release_tag: a tag of the code for a specific release 616 """ 617

618 - def __init__(self):

619 self.type = 'hg' 620 self.repo_uri = None 621 self.anon_repo_uri = None 622 self.dev_branch = None 623 self.distro_tag = None 624 self.release_tag = None

625

626 - def __eq__(self, other):

627 return self.repo_uri == other.repo_uri and \ 628 self.anon_repo_uri == other.anon_repo_uri and \ 629 self.dev_branch == other.dev_branch and \ 630 self.release_tag == other.release_tag and \ 631 self.distro_tag == other.distro_tag

632

633 -class GitConfig(object):

634 """ 635 Configuration information about an SVN repository for a component 636 of code. The configuration we maintain is specific to ROS 637 toolchain concepts and is not a general notion of SVN configuration. 638 639 * repo_uri: base URI of repo 640 * dev_branch: git branch the code is developed 641 * distro_tag: a tag of the latest released code for a specific ROS distribution 642 * release_tag: a tag of the code for a specific release 643 """ 644

645 - def __init__(self):

646 self.type = 'git' 647 self.repo_uri = None 648 self.anon_repo_uri = None 649 self.dev_branch = None 650 self.distro_tag = None 651 self.release_tag = None

652

653 - def __eq__(self, other):

654 return self.repo_uri == other.repo_uri and \ 655 self.anon_repo_uri == other.anon_repo_uri and \ 656 self.dev_branch == other.dev_branch and \ 657 self.release_tag == other.release_tag and \ 658 self.distro_tag == other.distro_tag

659

660 -class SvnConfig(object):

661 """ 662 Configuration information about an SVN repository for a component 663 of code. The configuration we maintain is specific to ROS 664 toolchain concepts and is not a general notion of SVN configuration. 665 666 * dev: where the code is developed 667 * distro_tag: a tag of the code for a specific ROS distribution 668 * release_tag: a tag of the code for a specific release 669 """ 670

671 - def __init__(self):

672 self.type = 'svn' 673 self.dev = None 674 self.distro_tag = None 675 self.release_tag = None 676 677 # anonymously readable version of URLs above. Some repos have 678 # separate URLs for read-only vs. writable versions of repo 679 # and many tools need to be able to read repos without 680 # providing credentials. 681 self.anon_dev = None 682 self.anon_distro_tag = None 683 self.anon_release_tag = None

684

685 - def __eq__(self, other):

686 return self.dev == other.dev and \ 687 self.distro_tag == other.distro_tag and \ 688 self.release_tag == other.release_tag and \ 689 self.anon_dev == other.anon_dev and \ 690 self.anon_distro_tag == other.anon_distro_tag and \ 691 self.anon_release_tag == other.anon_release_tag

692

Source Code for Module rosdistro