rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import codecs 43 import os 44 try: 45 from cStringIO import StringIO 46 python2 = True 47 except ImportError: 48 from io import StringIO 49 python2 = False 50 import re 51 import xml.etree.ElementTree as ET 52 from functools import reduce 53 from xml.dom import Node as DomNode 54 from xml.dom.minidom import parseString 55 56 import rospkg 57 58 pattern = r'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF\u0100-\uD7FF\uE000-\uFDCF\uFDE0-\uFFFD]' 59 if python2: 60 pattern = pattern.decode('unicode_escape') 61 else: 62 pattern = codecs.decode(pattern, 'unicode_escape') 63 invalid_chars = re.compile(pattern) 64 65

66 -def invalid_char_replacer(m):

67 return '&#x' + ('%04X' % ord(m.group(0))) + ';'

68 69

70 -def filter_nonprintable_text(text):

71 return re.sub(invalid_chars, invalid_char_replacer, text)

72 73

74 -def cdata(cdata_text):

75 return '<![CDATA[\n{}\n]]>'.format(cdata_text)

76 77

78 -class TestInfo(object):

79 """ 80 Common container for 'error' and 'failure' results 81 """ 82

83 - def __init__(self, type_, text):

84 """ 85 @param type_: type attribute from xml 86 @type type_: str 87 @param text: text property from xml 88 @type text: str 89 """ 90 self.type = type_ 91 self.text = text

92 93

94 -class TestError(TestInfo):

95 """ 96 'error' result container 97 """

98 - def xml(self):

99 """ 100 @return XML tag representing the object, with non-XML text filtered out 101 @rtype: str 102 """ 103 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

104

105 - def xml_element(self):

106 """ 107 @return XML tag representing the object, with non-XML text filtered out 108 @rtype: xml.etree.ElementTree.Element 109 """ 110 error = ET.Element('error') 111 error.set('type', self.type) 112 error.text = cdata(filter_nonprintable_text(self.text)) 113 return error

114 115

116 -class TestFailure(TestInfo):

117 """ 118 'failure' result container 119 """

120 - def xml(self):

121 """ 122 @return XML tag representing the object, with non-XML text filtered out 123 @rtype: str 124 """ 125 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

126

127 - def xml_element(self):

128 """ 129 @return XML tag representing the object, with non-XML text filtered out 130 @rtype: xml.etree.ElementTree.Element 131 """ 132 error = ET.Element('failure') 133 error.set('type', self.type) 134 error.text = cdata(filter_nonprintable_text(self.text)) 135 return error

136 137

138 -class TestCaseResult(object):

139 """ 140 'testcase' result container 141 """ 142

143 - def __init__(self, name):

144 """ 145 @param name: name of testcase 146 @type name: str 147 """ 148 self.name = name 149 self.failures = [] 150 self.errors = [] 151 self.time = 0.0 152 self.classname = ''

153

154 - def _passed(self):

155 """ 156 @return: True if test passed 157 @rtype: bool 158 """ 159 return not self.errors and not self.failures

160 # bool: True if test passed without errors or failures 161 passed = property(_passed) 162

163 - def _failure_description(self):

164 """ 165 @return: description of testcase failure 166 @rtype: str 167 """ 168 if self.failures: 169 tmpl = '[%s][FAILURE]' % self.name 170 tmpl = tmpl + '-'*(80 - len(tmpl)) 171 tmpl = tmpl+'\n%s\n' + '-' * 80 + '\n\n' 172 return '\n'.join(tmpl % x.text for x in self.failures) 173 return ''

174

175 - def _error_description(self):

176 """ 177 @return: description of testcase error 178 @rtype: str 179 """ 180 if self.errors: 181 tmpl = '[%s][ERROR]' % self.name 182 tmpl = tmpl + '-' * (80 - len(tmpl)) 183 tmpl = tmpl+'\n%s\n' + '-' * 80 + '\n\n' 184 return '\n'.join(tmpl % x.text for x in self.errors) 185 return ''

186

187 - def _description(self):

188 """ 189 @return: description of testcase result 190 @rtype: str 191 """ 192 if self.passed: 193 return '[%s][passed]\n' % self.name 194 else: 195 return self._failure_description() + \ 196 self._error_description()

197 # str: printable description of testcase result 198 description = property(_description) 199

200 - def add_failure(self, failure):

201 """ 202 @param failure TestFailure 203 """ 204 self.failures.append(failure)

205

206 - def add_error(self, error):

207 """ 208 @param failure TestError 209 """ 210 self.errors.append(error)

211

212 - def xml(self):

213 """ 214 @return XML tag representing the object, with non-XML text filtered out 215 @rtype: str 216 """ 217 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

218

219 - def xml_element(self):

220 """ 221 @return XML tag representing the object, with non-XML text filtered out 222 @rtype: xml.etree.ElementTree.Element 223 """ 224 testcase = ET.Element('testcase') 225 testcase.set('classname', self.classname) 226 testcase.set('name', self.name) 227 testcase.set('time', str(self.time)) 228 for f in self.failures: 229 testcase.append(f.xml_element()) 230 for e in self.errors: 231 testcase.append(e.xml_element()) 232 return testcase

233 234

235 -class Result(object):

236 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', 237 'test_case_results', 'system_out', 'system_err', 'time'] 238

239 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

240 self.name = name 241 self.num_errors = num_errors 242 self.num_failures = num_failures 243 self.num_tests = num_tests 244 self.test_case_results = [] 245 self.system_out = '' 246 self.system_err = '' 247 self.time = 0.0

248

249 - def accumulate(self, r):

250 """ 251 Add results from r to this result 252 @param r: results to aggregate with this result 253 @type r: Result 254 """ 255 self.num_errors += r.num_errors 256 self.num_failures += r.num_failures 257 self.num_tests += r.num_tests 258 self.time += r.time 259 self.test_case_results.extend(r.test_case_results) 260 if r.system_out: 261 self.system_out += '\n'+r.system_out 262 if r.system_err: 263 self.system_err += '\n'+r.system_err

264

265 - def add_test_case_result(self, r):

266 """ 267 Add results from a testcase to this result container 268 @param r: TestCaseResult 269 @type r: TestCaseResult 270 """ 271 self.test_case_results.append(r)

272

273 - def xml_element(self):

274 """ 275 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 276 """ 277 testsuite = ET.Element('testsuite') 278 testsuite.set('tests', str(self.num_tests)) 279 testsuite.set('failures', str(self.num_failures)) 280 testsuite.set('time', str(self.time)) 281 testsuite.set('errors', str(self.num_errors)) 282 testsuite.set('name', self.name) 283 for tc in self.test_case_results: 284 tc.xml(testsuite) 285 system_out = ET.SubElement(testsuite, 'system-out') 286 system_out.text = cdata(filter_nonprintable_text(self.system_out)) 287 system_err = ET.SubElement(testsuite, 'system-err') 288 system_err.text = cdata(filter_nonprintable_text(self.system_err)) 289 return ET.tostring(testsuite, encoding='utf-8', method='xml')

290 291

292 -def _text(tag):

293 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], '').strip()

294 295

296 -def _load_suite_results(test_suite_name, test_suite, result):

297 nodes = [n for n in test_suite.childNodes 298 if n.nodeType == DomNode.ELEMENT_NODE] 299 for node in nodes: 300 name = node.tagName 301 if name == 'testsuite': 302 # for now we flatten this hierarchy 303 _load_suite_results(test_suite_name, node, result) 304 elif name == 'system-out': 305 if _text(node): 306 system_out = '[%s] stdout' % test_suite_name + '-' * (71 - len(test_suite_name)) 307 system_out += '\n'+_text(node) 308 result.system_out += system_out 309 elif name == 'system-err': 310 if _text(node): 311 system_err = '[%s] stderr' % test_suite_name + '-' * (71 - len(test_suite_name)) 312 system_err += '\n'+_text(node) 313 result.system_err += system_err 314 elif name == 'testcase': 315 name = node.getAttribute('name') or 'unknown' 316 classname = node.getAttribute('classname') or 'unknown' 317 318 # mangle the classname for some sense of uniformity 319 # between rostest/unittest/gtest 320 if '__main__.' in classname: 321 classname = classname[classname.find('__main__.') + 9:] 322 if classname == 'rostest.rostest.RosTest': 323 classname = 'rostest' 324 elif not classname.startswith(result.name): 325 classname = '%s.%s' % (result.name, classname) 326 327 try: 328 time = float(node.getAttribute('time')) 329 except ValueError: 330 time = 0.0 331 tc_result = TestCaseResult('%s/%s' % (test_suite_name, name)) 332 tc_result.classname = classname 333 tc_result.time = time 334 result.add_test_case_result(tc_result) 335 for d in [n for n in node.childNodes 336 if n.nodeType == DomNode.ELEMENT_NODE]: 337 # convert 'message' attributes to text elements to keep 338 # python unittest and gtest consistent 339 if d.tagName == 'failure': 340 message = d.getAttribute('message') or '' 341 text = _text(d) or message 342 x = TestFailure(d.getAttribute('type') or '', text) 343 tc_result.add_failure(x) 344 elif d.tagName == 'error': 345 message = d.getAttribute('message') or '' 346 text = _text(d) or message 347 x = TestError(d.getAttribute('type') or '', text) 348 tc_result.add_error(x)

349 350 351 # #603: unit test suites are not good about screening out illegal 352 # unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 353 # screens these out 354 try: 355 char = unichr 356 except NameError: 357 char = chr 358 RE_XML_ILLEGAL = '([%s-%s%s-%s%s-%s%s-%s])' + \ 359 '|' + \ 360 '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' 361 try: 362 RE_XML_ILLEGAL = unicode(RE_XML_ILLEGAL) 363 except NameError: 364 pass 365 RE_XML_ILLEGAL = RE_XML_ILLEGAL % \ 366 (char(0x0000), char(0x0008), char(0x000b), char(0x000c), 367 char(0x000e), char(0x001f), char(0xfffe), char(0xffff), 368 char(0xd800), char(0xdbff), char(0xdc00), char(0xdfff), 369 char(0xd800), char(0xdbff), char(0xdc00), char(0xdfff), 370 char(0xd800), char(0xdbff), char(0xdc00), char(0xdfff)) 371 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 372 373

374 -def _read_file_safe_xml(test_file, write_back_sanitized=True):

375 """ 376 read in file, screen out unsafe unicode characters 377 """ 378 f = None 379 try: 380 # this is ugly, but the files in question that are problematic 381 # do not declare unicode type. 382 if not os.path.isfile(test_file): 383 raise Exception('test file does not exist') 384 try: 385 f = codecs.open(test_file, 'r', 'utf-8') 386 x = f.read() 387 except Exception: 388 if f is not None: 389 f.close() 390 f = codecs.open(test_file, 'r', 'iso8859-1') 391 x = f.read() 392 393 for match in _safe_xml_regex.finditer(x): 394 x = x[:match.start()] + '?' + x[match.end():] 395 x = x.encode('utf-8') 396 if write_back_sanitized: 397 with open(test_file, 'wb') as h: 398 h.write(x) 399 return x 400 finally: 401 if f is not None: 402 f.close()

403 404

405 -def read(test_file, test_name):

406 """ 407 Read in the test_result file 408 @param test_file: test file path 409 @type test_file: str 410 @param test_name: name of test 411 @type test_name: str 412 @return: test results 413 @rtype: Result 414 """ 415 try: 416 xml_str = _read_file_safe_xml(test_file) 417 if not xml_str.strip(): 418 print('WARN: test result file is empty [%s]' % (test_file)) 419 return Result(test_name, 0, 0, 0) 420 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 421 except Exception as e: 422 print('WARN: cannot read test result file [%s]: %s' % (test_file, str(e))) 423 return Result(test_name, 0, 0, 0) 424 if not test_suites: 425 print('WARN: test result file [%s] contains no results' % (test_file)) 426 return Result(test_name, 0, 0, 0) 427 428 results = Result(test_name, 0, 0, 0) 429 for index, test_suite in enumerate(test_suites): 430 # skip test suites which are already covered by a parent test suite 431 if index > 0 and test_suite.parentNode in test_suites[0:index]: 432 continue 433 434 # test_suite = test_suite[0] 435 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 436 vals = [v or 0 for v in vals] 437 err, fail, tests = [int(val) for val in vals] 438 439 result = Result(test_name, err, fail, tests) 440 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 441 442 # Create a prefix based on the test result filename. The idea is to 443 # disambiguate the case when tests of the same name are provided in 444 # different .xml files. We use the name of the parent directory 445 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 446 fname = os.path.basename(test_file) 447 if fname.startswith('TEST-'): 448 fname = fname[5:] 449 if fname.endswith('.xml'): 450 fname = fname[:-4] 451 test_file_base = '%s.%s' % (test_file_base, fname) 452 _load_suite_results(test_file_base, test_suite, result) 453 results.accumulate(result) 454 return results

455 456

457 -def read_all(filter_=[]):

458 """ 459 Read in the test_results and aggregate into a single Result object 460 @param filter_: list of packages that should be processed 461 @type filter_: [str] 462 @return: aggregated result 463 @rtype: L{Result} 464 """ 465 dir_ = rospkg.get_test_results_dir() 466 root_result = Result('ros', 0, 0, 0) 467 if not os.path.exists(dir_): 468 return root_result 469 for d in os.listdir(dir_): 470 if filter_ and d not in filter_: 471 continue 472 subdir = os.path.join(dir_, d) 473 if os.path.isdir(subdir): 474 for filename in os.listdir(subdir): 475 if filename.endswith('.xml'): 476 filename = os.path.join(subdir, filename) 477 result = read(filename, os.path.basename(subdir)) 478 root_result.accumulate(result) 479 return root_result

480 481

482 -def test_failure_junit_xml(test_name, message, stdout=None, class_name='Results', testcase_name='test_ran'):

483 """ 484 Generate JUnit XML file for a unary test suite where the test failed 485 486 @param test_name: Name of test that failed 487 @type test_name: str 488 @param message: failure message 489 @type message: str 490 @param stdout: stdout data to include in report 491 @type stdout: str 492 """ 493 testsuite = ET.Element('testsuite') 494 testsuite.set('tests', '1') 495 testsuite.set('failures', '1') 496 testsuite.set('time', '1') 497 testsuite.set('errors', '0') 498 testsuite.set('name', test_name) 499 testcase = ET.SubElement(testsuite, 'testcase') 500 testcase.set('name', testcase_name) 501 testcase.set('status', 'run') 502 testcase.set('time', '1') 503 testcase.set('classname', class_name) 504 failure = ET.SubElement(testcase, 'failure') 505 failure.set('message', message) 506 failure.set('type', '') 507 if stdout: 508 system_out = ET.SubElement(testsuite, 'system-out') 509 system_out.text = cdata(filter_nonprintable_text(stdout)) 510 return ET.tostring(testsuite, encoding='utf-8', method='xml')

511 512

513 -def test_success_junit_xml(test_name, class_name='Results', testcase_name='test_ran'):

514 """ 515 Generate JUnit XML file for a unary test suite where the test succeeded. 516 517 @param test_name: Name of test that passed 518 @type test_name: str 519 """ 520 testsuite = ET.Element('testsuite') 521 testsuite.set('tests', '1') 522 testsuite.set('failures', '0') 523 testsuite.set('time', '1') 524 testsuite.set('errors', '0') 525 testsuite.set('name', test_name) 526 testcase = ET.SubElement(testsuite, 'testcase') 527 testcase.set('name', testcase_name) 528 testcase.set('status', 'run') 529 testcase.set('time', '1') 530 testcase.set('classname', class_name) 531 return ET.tostring(testsuite, encoding='utf-8', method='xml')

532 533

534 -def print_summary(junit_results, runner_name='ROSUNIT'):

535 """ 536 Print summary of junitxml results to stdout. 537 """ 538 # we have two separate result objects, which can be a bit 539 # confusing. 'result' counts successful _running_ of tests 540 # (i.e. doesn't check for actual test success). The 'r' result 541 # object contains results of the actual tests. 542 543 buff = StringIO() 544 buff.write('[%s]' % runner_name + '-' * 71 + '\n\n') 545 for tc_result in junit_results.test_case_results: 546 buff.write(tc_result.description) 547 548 buff.write('\nSUMMARY\n') 549 if (junit_results.num_errors + junit_results.num_failures) == 0: 550 buff.write('\033[32m * RESULT: SUCCESS\033[0m\n') 551 else: 552 buff.write('\033[1;31m * RESULT: FAIL\033[0m\n') 553 554 # TODO: still some issues with the numbers adding up if tests fail to launch 555 556 # number of errors from the inner tests, plus add in count for tests 557 # that didn't run properly ('result' object). 558 buff.write(' * TESTS: %s\n' % junit_results.num_tests) 559 num_errors = junit_results.num_errors 560 if num_errors: 561 buff.write('\033[1;31m * ERRORS: %s\033[0m\n' % num_errors) 562 else: 563 buff.write(' * ERRORS: 0\n') 564 num_failures = junit_results.num_failures 565 if num_failures: 566 buff.write('\033[1;31m * FAILURES: %s\033[0m\n' % num_failures) 567 else: 568 buff.write(' * FAILURES: 0\n') 569 570 print(buff.getvalue())

571

Source Code for Module rosunit.junitxml