rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id: junitxml.py 15026 2011-09-22 18:34:28Z kwc $ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import os 43 import sys 44 import cStringIO 45 import string 46 import codecs 47 import re 48 49 from xml.dom.minidom import parse, parseString 50 from xml.dom import Node as DomNode 51 52 import roslib.rosenv 53

54 -class TestInfo(object):

55 """ 56 Common container for 'error' and 'failure' results 57 """ 58

59 - def __init__(self, type_, text):

60 """ 61 @param type_: type attribute from xml 62 @type type_: str 63 @param text: text property from xml 64 @type text: str 65 """ 66 self.type = type_ 67 self.text = text

68

69 -class TestError(TestInfo):

70 """ 71 'error' result container 72 """

73 - def xml(self):

74 return u'<error type="%s"><![CDATA[%s]]></error>'%(self.type, self.text)

75

76 -class TestFailure(TestInfo):

77 """ 78 'failure' result container 79 """

80 - def xml(self):

81 return u'<failure type="%s"><![CDATA[%s]]></failure>'%(self.type, self.text)

82 83

84 -class TestCaseResult(object):

85 """ 86 'testcase' result container 87 """ 88

89 - def __init__(self, name):

90 """ 91 @param name: name of testcase 92 @type name: str 93 """ 94 self.name = name 95 self.failures = [] 96 self.errors = [] 97 self.time = 0.0 98 self.classname = ''

99

100 - def _passed(self):

101 """ 102 @return: True if test passed 103 @rtype: bool 104 """ 105 return not self.errors and not self.failures

106 ## bool: True if test passed without errors or failures 107 passed = property(_passed) 108

109 - def _failure_description(self):

110 """ 111 @return: description of testcase failure 112 @rtype: str 113 """ 114 if self.failures: 115 tmpl = "[%s][FAILURE]"%self.name 116 tmpl = tmpl + '-'*(80-len(tmpl)) 117 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 118 return '\n'.join(tmpl%x.text for x in self.failures) 119 return ''

120

121 - def _error_description(self):

122 """ 123 @return: description of testcase error 124 @rtype: str 125 """ 126 if self.errors: 127 tmpl = "[%s][ERROR]"%self.name 128 tmpl = tmpl + '-'*(80-len(tmpl)) 129 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 130 return '\n'.join(tmpl%x.text for x in self.errors) 131 return ''

132

133 - def _description(self):

134 """ 135 @return: description of testcase result 136 @rtype: str 137 """ 138 if self.passed: 139 return "[%s][passed]\n"%self.name 140 else: 141 return self._failure_description()+\ 142 self._error_description()

143 ## str: printable description of testcase result 144 description = property(_description)

145 - def add_failure(self, failure):

146 """ 147 @param failure TestFailure 148 """ 149 self.failures.append(failure)

150

151 - def add_error(self, error):

152 """ 153 @param failure TestError 154 """ 155 self.errors.append(error)

156

157 - def xml(self):

158 return u' <testcase classname="%s" name="%s" time="%s">\n'%(self.classname, self.name, self.time)+\ 159 '\n '.join([f.xml() for f in self.failures])+\ 160 '\n '.join([e.xml() for e in self.errors])+\ 161 ' </testcase>'

162

163 -class Result(object):

164 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 165 'test_case_results', 'system_out', 'system_err', 'time']

166 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

167 self.name = name 168 self.num_errors = num_errors 169 self.num_failures = num_failures 170 self.num_tests = num_tests 171 self.test_case_results = [] 172 self.system_out = '' 173 self.system_err = '' 174 self.time = 0.0

175

176 - def accumulate(self, r):

177 """ 178 Add results from r to this result 179 @param r: results to aggregate with this result 180 @type r: Result 181 """ 182 self.num_errors += r.num_errors 183 self.num_failures += r.num_failures 184 self.num_tests += r.num_tests 185 self.time += r.time 186 self.test_case_results.extend(r.test_case_results) 187 if r.system_out: 188 self.system_out += '\n'+r.system_out 189 if r.system_err: 190 self.system_err += '\n'+r.system_err

191

192 - def add_test_case_result(self, r):

193 """ 194 Add results from a testcase to this result container 195 @param r: TestCaseResult 196 @type r: TestCaseResult 197 """ 198 self.test_case_results.append(r)

199

200 - def xml(self):

201 """ 202 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 203 """ 204 return u'<?xml version="1.0" encoding="utf-8"?>'+\ 205 '<testsuite name="%s" tests="%s" errors="%s" failures="%s" time="%s">'%\ 206 (self.name, self.num_tests, self.num_errors, self.num_failures, self.time)+\ 207 '\n'.join([tc.xml() for tc in self.test_case_results])+\ 208 ' <system-out><![CDATA[%s]]></system-out>'%self.system_out+\ 209 ' <system-err><![CDATA[%s]]></system-err>'%self.system_err+\ 210 '</testsuite>'

211

212 -def _text(tag):

213 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

214

215 -def _load_suite_results(test_suite_name, test_suite, result):

216 nodes = [n for n in test_suite.childNodes \ 217 if n.nodeType == DomNode.ELEMENT_NODE] 218 for node in nodes: 219 name = node.tagName 220 if name == 'testsuite': 221 # for now we flatten this hierarchy 222 _load_suite_results(test_suite_name, node, result) 223 elif name == 'system-out': 224 if _text(node): 225 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 226 system_out += '\n'+_text(node) 227 result.system_out += system_out 228 elif name == 'system-err': 229 if _text(node): 230 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 231 system_err += '\n'+_text(node) 232 result.system_err += system_err 233 elif name == 'testcase': 234 name = node.getAttribute('name') or 'unknown' 235 classname = node.getAttribute('classname') or 'unknown' 236 237 # mangle the classname for some sense of uniformity 238 # between rostest/unittest/gtest 239 if '__main__.' in classname: 240 classname = classname[classname.find('__main__.')+9:] 241 if classname == 'rostest.rostest.RosTest': 242 classname = 'rostest' 243 elif not classname.startswith(result.name): 244 classname = "%s.%s"%(result.name,classname) 245 246 time = float(node.getAttribute('time')) or 0.0 247 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 248 tc_result.classname = classname 249 tc_result.time = time 250 result.add_test_case_result(tc_result) 251 for d in [n for n in node.childNodes \ 252 if n.nodeType == DomNode.ELEMENT_NODE]: 253 # convert 'message' attributes to text elements to keep 254 # python unittest and gtest consistent 255 if d.tagName == 'failure': 256 message = d.getAttribute('message') or '' 257 text = _text(d) or message 258 x = TestFailure(d.getAttribute('type') or '', text) 259 tc_result.add_failure(x) 260 elif d.tagName == 'error': 261 message = d.getAttribute('message') or '' 262 text = _text(d) or message 263 x = TestError(d.getAttribute('type') or '', text) 264 tc_result.add_error(x)

265 266 ## #603: unit test suites are not good about screening out illegal 267 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 268 ## screens these out 269 RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ 270 u'|' + \ 271 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ 272 (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 273 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 274 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff)) 275 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 276

277 -def _read_file_safe_xml(test_file):

278 """ 279 read in file, screen out unsafe unicode characters 280 """ 281 f = None 282 try: 283 # this is ugly, but the files in question that are problematic 284 # do not declare unicode type. 285 if not os.path.isfile(test_file): 286 raise Exception("test file does not exist") 287 try: 288 f = codecs.open(test_file, "r", "utf-8" ) 289 x = f.read() 290 except: 291 if f is not None: 292 f.close() 293 f = codecs.open(test_file, "r", "iso8859-1" ) 294 x = f.read() 295 296 for match in _safe_xml_regex.finditer(x): 297 x = x[:match.start()] + "?" + x[match.end():] 298 return x.encode("utf-8") 299 finally: 300 if f is not None: 301 f.close()

302

303 -def read(test_file, test_name):

304 """ 305 Read in the test_result file 306 @param test_file: test file path 307 @type test_file: str 308 @param test_name: name of test 309 @type test_name: str 310 @return: test results 311 @rtype: Result 312 """ 313 try: 314 xml_str = _read_file_safe_xml(test_file) 315 if not xml_str.strip(): 316 print("WARN: test result file is empty [%s]"%(test_file)) 317 return Result(test_name, 0, 0, 0) 318 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 319 except Exception as e: 320 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 321 return Result(test_name, 0, 0, 0) 322 if not test_suites: 323 print("WARN: test result file [%s] contains no results"%(test_file)) 324 return Result(test_name, 0, 0, 0) 325 326 results = Result(test_name, 0, 0, 0) 327 for test_suite in test_suites: 328 #test_suite = test_suite[0] 329 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 330 vals = [v or 0 for v in vals] 331 err, fail, tests = [string.atoi(val) for val in vals] 332 333 result = Result(test_name, err, fail, tests) 334 result.time = float(test_suite.getAttribute('time')) or 0.0 335 336 # Create a prefix based on the test result filename. The idea is to 337 # disambiguate the case when tests of the same name are provided in 338 # different .xml files. We use the name of the parent directory 339 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 340 fname = os.path.basename(test_file) 341 if fname.startswith('TEST-'): 342 fname = fname[5:] 343 if fname.endswith('.xml'): 344 fname = fname[:-4] 345 test_file_base = "%s.%s"%(test_file_base, fname) 346 _load_suite_results(test_file_base, test_suite, result) 347 results.accumulate(result) 348 return results

349

350 -def read_all(filter_=[]):

351 """ 352 Read in the test_results and aggregate into a single Result object 353 @param filter_: list of packages that should be processed 354 @type filter_: [str] 355 @return: aggregated result 356 @rtype: L{Result} 357 """ 358 dir_ = roslib.rosenv.get_test_results_dir() 359 root_result = Result('ros', 0, 0, 0) 360 if not os.path.exists(dir_): 361 return root_result 362 for d in os.listdir(dir_): 363 if filter_ and not d in filter_: 364 continue 365 subdir = os.path.join(dir_, d) 366 if os.path.isdir(subdir): 367 for filename in os.listdir(subdir): 368 if filename.endswith('.xml'): 369 filename = os.path.join(subdir, filename) 370 result = read(filename, os.path.basename(subdir)) 371 root_result.accumulate(result) 372 return root_result

373 374

375 -def test_failure_junit_xml(test_name, message, stdout=None):

376 """ 377 Generate JUnit XML file for a unary test suite where the test failed 378 379 @param test_name: Name of test that failed 380 @type test_name: str 381 @param message: failure message 382 @type message: str 383 @param stdout: stdout data to include in report 384 @type stdout: str 385 """ 386 if not stdout: 387 return """<?xml version="1.0" encoding="UTF-8"?> 388 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 389 <testcase name="test_ran" status="run" time="1" classname="Results"> 390 <failure message="%s" type=""/> 391 </testcase> 392 </testsuite>"""%(test_name, message) 393 else: 394 return """<?xml version="1.0" encoding="UTF-8"?> 395 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 396 <testcase name="test_ran" status="run" time="1" classname="Results"> 397 <failure message="%s" type=""/> 398 </testcase> 399 <system-out><![CDATA[[ 400 %s 401 ]]></system-out> 402 </testsuite>"""%(test_name, message, stdout)

403

404 -def test_success_junit_xml(test_name):

405 """ 406 Generate JUnit XML file for a unary test suite where the test succeeded. 407 408 @param test_name: Name of test that passed 409 @type test_name: str 410 """ 411 return """<?xml version="1.0" encoding="UTF-8"?> 412 <testsuite tests="1" failures="0" time="1" errors="0" name="%s"> 413 <testcase name="test_ran" status="run" time="1" classname="Results"> 414 </testcase> 415 </testsuite>"""%(test_name)

416

417 -def print_summary(junit_results, runner_name='ROSUNIT'):

418 """ 419 Print summary of junitxml results to stdout. 420 """ 421 # we have two separate result objects, which can be a bit 422 # confusing. 'result' counts successful _running_ of tests 423 # (i.e. doesn't check for actual test success). The 'r' result 424 # object contains results of the actual tests. 425 426 buff = cStringIO.StringIO() 427 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 428 for tc_result in junit_results.test_case_results: 429 buff.write(tc_result.description) 430 431 buff.write('\nSUMMARY\n') 432 if (junit_results.num_errors + junit_results.num_failures) == 0: 433 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 434 else: 435 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 436 437 # TODO: still some issues with the numbers adding up if tests fail to launch 438 439 # number of errors from the inner tests, plus add in count for tests 440 # that didn't run properly ('result' object). 441 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 442 num_errors = junit_results.num_errors 443 if num_errors: 444 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 445 else: 446 buff.write(" * ERRORS: 0\n") 447 num_failures = junit_results.num_failures 448 if num_failures: 449 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 450 else: 451 buff.write(" * FAILURES: 0\n") 452 453 print(buff.getvalue())

454

Source Code for Module rosunit.junitxml