rosdeb.rosutil

1 # Software License Agreement (BSD License) 2 # 3 # Copyright (c) 2010, Willow Garage, Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions 8 # are met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following 14 # disclaimer in the documentation and/or other materials provided 15 # with the distribution. 16 # * Neither the name of Willow Garage, Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived 18 # from this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 30 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 # POSSIBILITY OF SUCH DAMAGE. 32 # 33 # Revision $Id: rosutil.py 16945 2012-08-31 16:45:33Z tfoote $ 34 # $Author: tfoote $ 35 36 import os 37 import subprocess 38 import tempfile 39 40 if __name__ == '__main__': 41 import roslib; roslib.load_manifest('rosdeb') 42 43 import roslib.manifest 44 import roslib.stack_manifest 45 import roslib.packages 46 import vcstools 47 48 import rosdeb 49 50 # due to migration issues, have to boostrap rosdep manually 51 roslib.load_manifest('rosdep') 52 import rosdep 53 from rosdep.core import RosdepLookupPackage, YamlCache 54 55 IMPLICIT_DEPS = ['libc6','build-essential','cmake','python-yaml','subversion'] 56

57 -def _text_only(soup):

58 return ''.join(soup.findAll(text=True))

59

60 -def checkout_svn_to_tmp(name, uri):

61 """ 62 Checkout an SVN tree to the tmp dir. 63 64 Utility routine -- need to replace with vcs 65 66 @return: temporary directory that contains checkout of SVN tree in 67 directory 'name'. temporary directory will be a subdirectory of 68 OS-provided temporary space. 69 @rtype: str 70 """ 71 tmp_dir = tempfile.mkdtemp() 72 dest = os.path.join(tmp_dir, name) 73 print 'Checking out a fresh copy of %s from %s to %s...'%(name, uri, dest) 74 subprocess.check_call(['svn', 'co', uri, dest]) 75 return tmp_dir

76

77 -def checkout_dev_to_tmp(name, distro_stack):

78 """ 79 Checkout an VCS-based 'dev' code tree to the tmp dir. 80 81 @return: temporary directory that contains checkout of SVN tree in 82 directory 'name'. temporary directory will be a subdirectory of 83 OS-provided temporary space. 84 @rtype: str 85 """ 86 for key in ['svn', 'git', 'hg', 'bzr']: 87 if key in distro_stack._rules: 88 break 89 else: 90 raise Exception("stack [%s] does not have any supported checkout rules"%(name)) 91 92 try: 93 if key == 'svn': 94 uri = distro_stack.expand_rule(distro_stack._rules[key]['dev']) 95 version = '' 96 elif key in ['hg', 'git', 'bzr']: 97 uri = distro_stack.expand_rule(distro_stack._rules[key]['uri']) 98 version = distro_stack.expand_rule(distro_stack._rules[key]['dev-branch']) 99 else: 100 raise Exception ("key %s not implemented"%key) 101 102 except KeyError: 103 raise Exception("cannot checkout stack dev tree to tmp: %s rules have no 'dev' key"%(key)) 104 105 tmp_dir = tempfile.mkdtemp() 106 dest = os.path.join(tmp_dir, name) 107 print 'Checking out a fresh copy of %s from %s to %s...'%(name, uri, dest) 108 vcs_client = vcstools.VcsClient(key, dest) 109 vcs_client.checkout(uri, version) 110 return tmp_dir

111

112 -def convert_html_to_text(d):

113 """ 114 Convert a HTML description to plain text. This routine still has 115 much work to do, but appears to handle the common uses of HTML in 116 our current manifests. 117 """ 118 # check for presence of tags 119 if '<' in d: 120 from rosdeb.BeautifulSoup import BeautifulSoup 121 soup = BeautifulSoup(d) 122 123 # first, target formatting tags with a straight replace 124 for t in ['b', 'strong', 'em', 'i', 'tt', 'a']: 125 tags = soup.findAll(t) 126 for x in tags: 127 x.replaceWith(_text_only(x)) 128 129 # second, target low-level container tags 130 tags = soup.findAll('li') 131 for x in tags: 132 x.replaceWith('* '+_text_only(x)+'\n') 133 134 # convert all high-level containers to line breaks 135 for t in ['p', 'div']: 136 tags = soup.findAll(t) 137 for t in tags: 138 t.replaceWith(_text_only(t)+'\n') 139 140 # findAll text strips remaining tags 141 d = ''.join(soup.findAll(text=True)) 142 143 # reduce the whitespace as the debian parsers have strict rules 144 # about what is a paragraph and what is verbose based on leading 145 # whitespace. 146 d = '\n'.join([x.strip() for x in d.split('\n')]) 147 148 d_reduced = '' 149 last = None 150 for x in d.split('\n'): 151 if last is None: 152 d_reduced = x 153 else: 154 if x == '': 155 if last == '': 156 pass 157 else: 158 d_reduced += '\n' 159 else: 160 d_reduced += x + ' ' 161 last = x 162 return d_reduced

163 164 # based on code in roslib.stacks

165 -def package_manifests_of(stack_dir):

166 """ 167 @return: list of package names and manifest file paths for stack 168 dir. These will be returned as a list of (name, path) pairs. 169 @rtype: [(str, str)] 170 """ 171 # Unary stack check 172 m_file = os.path.join(stack_dir, 'manifest.xml') 173 if os.path.isfile(m_file): 174 return [(os.path.basename(os.path.abspath(stack_dir)), m_file),] 175 176 l = [os.path.join(stack_dir, d) for d in os.listdir(stack_dir)] 177 manifests = [] 178 packages = [] 179 while l: 180 d = l.pop() 181 if os.path.isdir(d): 182 if roslib.packages.is_pkg_dir(d): 183 p = os.path.basename(d) 184 m_file = os.path.join(d, 'manifest.xml') 185 # this is sometimes true if we've descended into a build directory 186 if not p in packages: 187 packages.append(p) 188 manifests.append((p, m_file)) 189 elif os.path.exists(os.path.join(d, 'rospack_nosubdirs')): 190 # don't descend 191 pass 192 elif os.path.basename(d) not in ['build', '.svn', '.git']: #recurse 193 l.extend([os.path.join(d, e) for e in os.listdir(d)]) 194 return manifests

195

196 -def stack_rosdeps(stack_name, stack_dir, platform):

197 """ 198 Calculate dependencies of stack on an 'ubuntu' OS, including both 199 ROS stacks and their rosdep dependencies, for the specified 200 ubuntu release version. 201 202 NOTE: one flaw in this implementation is that it uses the rosdep 203 view from the *active environment* to generate the rosdeps. It 204 does not generate them from specific versions of stacks. The hope 205 is that rosdeps improve monotonically over time, so that this will 206 not be a major issue. 207 208 @param platform: platform name (e.g. lucid) 209 210 @return: list of debian package deps 211 @rtype: [str] 212 @raise Exception: if stack rosdeps cannot be fully resolved 213 """ 214 215 # - implicit deps of all ROS packages 216 deb_deps = IMPLICIT_DEPS[:] 217 218 # hardcode OS for now as we don't build straight debian 219 os_name = 'ubuntu' 220 # reverse lookup version number, which is the key for rosdep 221 os_version = [k for k, v in rosdeb.get_ubuntu_map().iteritems() if v == platform][0] 222 223 try: 224 # REP 111 API 225 import rosdep.installers 226 installers = {'apt': rosdep.installers.AptInstaller, 'source': rosdep.installers.SourceInstaller} 227 os_version = platform 228 yc = YamlCache(os_name, os_version, installers) 229 230 except ImportError: 231 yc = YamlCache(os_name, os_version) 232 233 package_manifests = package_manifests_of(stack_dir) 234 for p, m_file in package_manifests: 235 m = roslib.manifest.parse_file(m_file) 236 rosdeps = [d.name for d in m.rosdeps] 237 if not rosdeps: 238 continue 239 240 rdlp = RosdepLookupPackage(os_name, os_version, p, yc) 241 for r in rosdeps: 242 value = rdlp.lookup_rosdep(r) 243 if value is False: 244 raise Exception("cannot generate rosdeps for stack [%s] on platform [%s]:\n\trosdep lookup of [%s] failed"%(stack_name, os_version, r)) 245 if type(value) == dict: 246 if 'apt' in value: 247 packages = value['apt']['packages'] 248 if type(packages) == list: 249 deb_deps.extend(packages) 250 else: 251 deb_deps.append(packages) 252 else: 253 if '\n' in value: 254 raise Exception("cannot generate rosdeps for stack [%s] on platform [%s]:\n\trosdep [%s] has a script binding"%(stack_name, os_version, r)) 255 256 deb_deps.extend([x for x in value.split(' ') if x.strip()]) 257 258 return list(set(deb_deps))

259

260 -def missing_stack_rosdeps(stack_name, stack_dir, platform):

261 """ 262 Calculate list of rosdeps that are missing definitions on platform. 263 264 NOTE: one flaw in this implementation is that it uses the rosdep 265 view from the *active environment* to generate the rosdeps. It 266 does not generate them from specific versions of stacks. The hope 267 is that rosdeps improve monotonically over time, so that this will 268 not be a major issue. 269 270 @param platform: platform name (e.g. lucid) 271 @return: dictionary mapping packages to their missing rosdep mappings 272 @rtype: {str: [str]} 273 """ 274 275 # hardcode OS for now as we don't build straight debian 276 os_name = 'ubuntu' 277 # reverse lookup version number, which is the key for rosdep 278 os_version = [k for k, v in rosdeb.get_ubuntu_map().iteritems() if v == platform][0] 279 280 yc = YamlCache(os_name, os_version) 281 package_manifests = package_manifests_of(stack_dir) 282 packages = {} 283 for p, m_file in package_manifests: 284 missing = [] 285 packages[p] = missing 286 m = roslib.manifest.parse_file(m_file) 287 rosdeps = [d.name for d in m.rosdeps] 288 if not rosdeps: 289 continue 290 291 rdlp = RosdepLookupPackage(os_name, os_version, p, yc) 292 for r in rosdeps: 293 value = rdlp.lookup_rosdep(r) 294 if not value or '\n' in value: 295 missing.append(r) 296 return packages

297

298 -def send_email(smtp_server, from_addr, to_addrs, subject, text):

299 import smtplib 300 from email.mime.text import MIMEText 301 302 msg = MIMEText(text) 303 304 msg['From'] = from_addr 305 msg['To'] = to_addrs 306 msg['Subject'] = subject 307 308 s = smtplib.SMTP(smtp_server) 309 print 'Sending mail to %s'%(to_addrs) 310 try: 311 s.sendmail(msg['From'], [msg['To']], msg.as_string()) 312 except Exception, ex: 313 print "Sending email failed with exception: %s" % ex 314 s.quit()

315 316 if __name__ == '__main__': 317 import roslib.stacks 318 from rosdeb.rosutil import convert_html_to_text 319 from roslib.stack_manifest import parse_file, stack_file 320 for stack_name in roslib.stacks.list_stacks(): 321 stack_xml = stack_file(stack_name) 322 m = roslib.stack_manifest.parse_file(stack_xml) 323 if stack_name in ['ros_release']: 324 print '='*80 325 print stack_name 326 print convert_html_to_text(m.description) 327

Source Code for Module rosdeb.rosutil