rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import codecs 43 import os 44 import sys 45 try: 46 from cStringIO import StringIO 47 python2 = True 48 except ImportError: 49 from io import StringIO 50 python2 = False 51 import string 52 import codecs 53 import re 54 55 import xml.etree.ElementTree as ET 56 from xml.dom.minidom import parse, parseString 57 from xml.dom import Node as DomNode 58 59 from functools import reduce 60 import rospkg 61 62 pattern = r'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF\u0100-\uD7FF\uE000-\uFDCF\uFDE0-\uFFFD]' 63 if python2: 64 pattern = pattern.decode('unicode_escape') 65 else: 66 pattern = codecs.decode(pattern, 'unicode_escape') 67 invalid_chars = re.compile(pattern) 68 69

70 -def invalid_char_replacer(m):

71 return "&#x"+('%04X' % ord(m.group(0)))+";"

72 -def filter_nonprintable_text(text):

73 return re.sub(invalid_chars, invalid_char_replacer, text)

74

75 -def cdata(cdata_text):

76 return '<![CDATA[\n{}\n]]>'.format(cdata_text)

77

78 -class TestInfo(object):

79 """ 80 Common container for 'error' and 'failure' results 81 """ 82

83 - def __init__(self, type_, text):

84 """ 85 @param type_: type attribute from xml 86 @type type_: str 87 @param text: text property from xml 88 @type text: str 89 """ 90 self.type = type_ 91 self.text = text

92

93 -class TestError(TestInfo):

94 """ 95 'error' result container 96 """

97 - def xml(self):

98 """ 99 @return XML tag representing the object, with non-XML text filtered out 100 @rtype: str 101 """ 102 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

103

104 - def xml_element(self):

105 """ 106 @return XML tag representing the object, with non-XML text filtered out 107 @rtype: xml.etree.ElementTree.Element 108 """ 109 error = ET.Element('error') 110 error.set('type', self.type) 111 error.text = cdata(filter_nonprintable_text(self.text)) 112 return error

113

114 -class TestFailure(TestInfo):

115 """ 116 'failure' result container 117 """

118 - def xml(self):

119 """ 120 @return XML tag representing the object, with non-XML text filtered out 121 @rtype: str 122 """ 123 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

124

125 - def xml_element(self):

126 """ 127 @return XML tag representing the object, with non-XML text filtered out 128 @rtype: xml.etree.ElementTree.Element 129 """ 130 error = ET.Element('failure') 131 error.set('type', self.type) 132 error.text = cdata(filter_nonprintable_text(self.text)) 133 return error

134

135 -class TestCaseResult(object):

136 """ 137 'testcase' result container 138 """ 139

140 - def __init__(self, name):

141 """ 142 @param name: name of testcase 143 @type name: str 144 """ 145 self.name = name 146 self.failures = [] 147 self.errors = [] 148 self.time = 0.0 149 self.classname = ''

150

151 - def _passed(self):

152 """ 153 @return: True if test passed 154 @rtype: bool 155 """ 156 return not self.errors and not self.failures

157 ## bool: True if test passed without errors or failures 158 passed = property(_passed) 159

160 - def _failure_description(self):

161 """ 162 @return: description of testcase failure 163 @rtype: str 164 """ 165 if self.failures: 166 tmpl = "[%s][FAILURE]"%self.name 167 tmpl = tmpl + '-'*(80-len(tmpl)) 168 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 169 return '\n'.join(tmpl%x.text for x in self.failures) 170 return ''

171

172 - def _error_description(self):

173 """ 174 @return: description of testcase error 175 @rtype: str 176 """ 177 if self.errors: 178 tmpl = "[%s][ERROR]"%self.name 179 tmpl = tmpl + '-'*(80-len(tmpl)) 180 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 181 return '\n'.join(tmpl%x.text for x in self.errors) 182 return ''

183

184 - def _description(self):

185 """ 186 @return: description of testcase result 187 @rtype: str 188 """ 189 if self.passed: 190 return "[%s][passed]\n"%self.name 191 else: 192 return self._failure_description()+\ 193 self._error_description()

194 ## str: printable description of testcase result 195 description = property(_description)

196 - def add_failure(self, failure):

197 """ 198 @param failure TestFailure 199 """ 200 self.failures.append(failure)

201

202 - def add_error(self, error):

203 """ 204 @param failure TestError 205 """ 206 self.errors.append(error)

207

208 - def xml(self):

209 """ 210 @return XML tag representing the object, with non-XML text filtered out 211 @rtype: str 212 """ 213 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

214

215 - def xml_element(self):

216 """ 217 @return XML tag representing the object, with non-XML text filtered out 218 @rtype: xml.etree.ElementTree.Element 219 """ 220 testcase = ET.Element('testcase') 221 testcase.set('classname', self.classname) 222 testcase.set('name', self.name) 223 testcase.set('time', str(self.time)) 224 for f in self.failures: 225 testcase.append(f.xml_element()) 226 for e in self.errors: 227 testcase.append(e.xml_element()) 228 return testcase

229

230 -class Result(object):

231 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 232 'test_case_results', 'system_out', 'system_err', 'time']

233 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

234 self.name = name 235 self.num_errors = num_errors 236 self.num_failures = num_failures 237 self.num_tests = num_tests 238 self.test_case_results = [] 239 self.system_out = '' 240 self.system_err = '' 241 self.time = 0.0

242

243 - def accumulate(self, r):

244 """ 245 Add results from r to this result 246 @param r: results to aggregate with this result 247 @type r: Result 248 """ 249 self.num_errors += r.num_errors 250 self.num_failures += r.num_failures 251 self.num_tests += r.num_tests 252 self.time += r.time 253 self.test_case_results.extend(r.test_case_results) 254 if r.system_out: 255 self.system_out += '\n'+r.system_out 256 if r.system_err: 257 self.system_err += '\n'+r.system_err

258

259 - def add_test_case_result(self, r):

260 """ 261 Add results from a testcase to this result container 262 @param r: TestCaseResult 263 @type r: TestCaseResult 264 """ 265 self.test_case_results.append(r)

266

267 - def xml_element(self):

268 """ 269 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 270 """ 271 testsuite = ET.Element('testsuite') 272 testsuite.set('tests', str(self.num_tests)) 273 testsuite.set('failures', str(self.num_failures)) 274 testsuite.set('time', str(self.time)) 275 testsuite.set('errors', str(self.num_errors)) 276 testsuite.set('name', self.name) 277 for tc in self.test_case_results: 278 tc.xml(testsuite) 279 system_out = ET.SubElement(testsuite, 'system-out') 280 system_out.text = cdata(filter_nonprintable_text(self.system_out)) 281 system_err = ET.SubElement(testsuite, 'system-err') 282 system_err.text = cdata(filter_nonprintable_text(self.system_err)) 283 return ET.tostring(testsuite, encoding='utf-8', method='xml')

284

285 -def _text(tag):

286 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

287

288 -def _load_suite_results(test_suite_name, test_suite, result):

289 nodes = [n for n in test_suite.childNodes \ 290 if n.nodeType == DomNode.ELEMENT_NODE] 291 for node in nodes: 292 name = node.tagName 293 if name == 'testsuite': 294 # for now we flatten this hierarchy 295 _load_suite_results(test_suite_name, node, result) 296 elif name == 'system-out': 297 if _text(node): 298 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 299 system_out += '\n'+_text(node) 300 result.system_out += system_out 301 elif name == 'system-err': 302 if _text(node): 303 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 304 system_err += '\n'+_text(node) 305 result.system_err += system_err 306 elif name == 'testcase': 307 name = node.getAttribute('name') or 'unknown' 308 classname = node.getAttribute('classname') or 'unknown' 309 310 # mangle the classname for some sense of uniformity 311 # between rostest/unittest/gtest 312 if '__main__.' in classname: 313 classname = classname[classname.find('__main__.')+9:] 314 if classname == 'rostest.rostest.RosTest': 315 classname = 'rostest' 316 elif not classname.startswith(result.name): 317 classname = "%s.%s"%(result.name,classname) 318 319 time = float(node.getAttribute('time')) or 0.0 320 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 321 tc_result.classname = classname 322 tc_result.time = time 323 result.add_test_case_result(tc_result) 324 for d in [n for n in node.childNodes \ 325 if n.nodeType == DomNode.ELEMENT_NODE]: 326 # convert 'message' attributes to text elements to keep 327 # python unittest and gtest consistent 328 if d.tagName == 'failure': 329 message = d.getAttribute('message') or '' 330 text = _text(d) or message 331 x = TestFailure(d.getAttribute('type') or '', text) 332 tc_result.add_failure(x) 333 elif d.tagName == 'error': 334 message = d.getAttribute('message') or '' 335 text = _text(d) or message 336 x = TestError(d.getAttribute('type') or '', text) 337 tc_result.add_error(x)

338 339 ## #603: unit test suites are not good about screening out illegal 340 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 341 ## screens these out 342 try: 343 char = unichr 344 except NameError: 345 char = chr 346 RE_XML_ILLEGAL = '([%s-%s%s-%s%s-%s%s-%s])' + \ 347 '|' + \ 348 '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' 349 try: 350 RE_XML_ILLEGAL = unicode(RE_XML_ILLEGAL) 351 except NameError: 352 pass 353 RE_XML_ILLEGAL = RE_XML_ILLEGAL % \ 354 (char(0x0000),char(0x0008),char(0x000b),char(0x000c), 355 char(0x000e),char(0x001f),char(0xfffe),char(0xffff), 356 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 357 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 358 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff)) 359 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 360

361 -def _read_file_safe_xml(test_file, write_back_sanitized=True):

362 """ 363 read in file, screen out unsafe unicode characters 364 """ 365 f = None 366 try: 367 # this is ugly, but the files in question that are problematic 368 # do not declare unicode type. 369 if not os.path.isfile(test_file): 370 raise Exception("test file does not exist") 371 try: 372 f = codecs.open(test_file, "r", "utf-8" ) 373 x = f.read() 374 except: 375 if f is not None: 376 f.close() 377 f = codecs.open(test_file, "r", "iso8859-1" ) 378 x = f.read() 379 380 for match in _safe_xml_regex.finditer(x): 381 x = x[:match.start()] + "?" + x[match.end():] 382 x = x.encode("utf-8") 383 if write_back_sanitized: 384 with open(test_file, 'wb') as h: 385 h.write(x) 386 return x 387 finally: 388 if f is not None: 389 f.close()

390

391 -def read(test_file, test_name):

392 """ 393 Read in the test_result file 394 @param test_file: test file path 395 @type test_file: str 396 @param test_name: name of test 397 @type test_name: str 398 @return: test results 399 @rtype: Result 400 """ 401 try: 402 xml_str = _read_file_safe_xml(test_file) 403 if not xml_str.strip(): 404 print("WARN: test result file is empty [%s]"%(test_file)) 405 return Result(test_name, 0, 0, 0) 406 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 407 except Exception as e: 408 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 409 return Result(test_name, 0, 0, 0) 410 if not test_suites: 411 print("WARN: test result file [%s] contains no results"%(test_file)) 412 return Result(test_name, 0, 0, 0) 413 414 results = Result(test_name, 0, 0, 0) 415 for index, test_suite in enumerate(test_suites): 416 # skip test suites which are already covered by a parent test suite 417 if index > 0 and test_suite.parentNode in test_suites[0:index]: 418 continue 419 420 #test_suite = test_suite[0] 421 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 422 vals = [v or 0 for v in vals] 423 err, fail, tests = [int(val) for val in vals] 424 425 result = Result(test_name, err, fail, tests) 426 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 427 428 # Create a prefix based on the test result filename. The idea is to 429 # disambiguate the case when tests of the same name are provided in 430 # different .xml files. We use the name of the parent directory 431 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 432 fname = os.path.basename(test_file) 433 if fname.startswith('TEST-'): 434 fname = fname[5:] 435 if fname.endswith('.xml'): 436 fname = fname[:-4] 437 test_file_base = "%s.%s"%(test_file_base, fname) 438 _load_suite_results(test_file_base, test_suite, result) 439 results.accumulate(result) 440 return results

441

442 -def read_all(filter_=[]):

443 """ 444 Read in the test_results and aggregate into a single Result object 445 @param filter_: list of packages that should be processed 446 @type filter_: [str] 447 @return: aggregated result 448 @rtype: L{Result} 449 """ 450 dir_ = rospkg.get_test_results_dir() 451 root_result = Result('ros', 0, 0, 0) 452 if not os.path.exists(dir_): 453 return root_result 454 for d in os.listdir(dir_): 455 if filter_ and not d in filter_: 456 continue 457 subdir = os.path.join(dir_, d) 458 if os.path.isdir(subdir): 459 for filename in os.listdir(subdir): 460 if filename.endswith('.xml'): 461 filename = os.path.join(subdir, filename) 462 result = read(filename, os.path.basename(subdir)) 463 root_result.accumulate(result) 464 return root_result

465 466

467 -def test_failure_junit_xml(test_name, message, stdout=None, class_name="Results", testcase_name="test_ran"):

468 """ 469 Generate JUnit XML file for a unary test suite where the test failed 470 471 @param test_name: Name of test that failed 472 @type test_name: str 473 @param message: failure message 474 @type message: str 475 @param stdout: stdout data to include in report 476 @type stdout: str 477 """ 478 testsuite = ET.Element('testsuite') 479 testsuite.set('tests', '1') 480 testsuite.set('failures', '1') 481 testsuite.set('time', '1') 482 testsuite.set('errors', '0') 483 testsuite.set('name', test_name) 484 testcase = ET.SubElement(testsuite, 'testcase') 485 testcase.set('name', testcase_name) 486 testcase.set('status', 'run') 487 testcase.set('time', '1') 488 testcase.set('classname', class_name) 489 failure = ET.SubElement(testcase, 'failure') 490 failure.set('message', message) 491 failure.set('type', '') 492 if stdout: 493 system_out = ET.SubElement(testsuite, 'system-out') 494 system_out.text = cdata(filter_nonprintable_text(stdout)) 495 return ET.tostring(testsuite, encoding='utf-8', method='xml')

496

497 -def test_success_junit_xml(test_name, class_name="Results", testcase_name="test_ran"):

498 """ 499 Generate JUnit XML file for a unary test suite where the test succeeded. 500 501 @param test_name: Name of test that passed 502 @type test_name: str 503 """ 504 testsuite = ET.Element('testsuite') 505 testsuite.set('tests', '1') 506 testsuite.set('failures', '0') 507 testsuite.set('time', '1') 508 testsuite.set('errors', '0') 509 testsuite.set('name', test_name) 510 testcase = ET.SubElement(testsuite, 'testcase') 511 testcase.set('name', testcase_name) 512 testcase.set('status', 'run') 513 testcase.set('time', '1') 514 testcase.set('classname', class_name) 515 return ET.tostring(testsuite, encoding='utf-8', method='xml')

516

517 -def print_summary(junit_results, runner_name='ROSUNIT'):

518 """ 519 Print summary of junitxml results to stdout. 520 """ 521 # we have two separate result objects, which can be a bit 522 # confusing. 'result' counts successful _running_ of tests 523 # (i.e. doesn't check for actual test success). The 'r' result 524 # object contains results of the actual tests. 525 526 buff = StringIO() 527 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 528 for tc_result in junit_results.test_case_results: 529 buff.write(tc_result.description) 530 531 buff.write('\nSUMMARY\n') 532 if (junit_results.num_errors + junit_results.num_failures) == 0: 533 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 534 else: 535 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 536 537 # TODO: still some issues with the numbers adding up if tests fail to launch 538 539 # number of errors from the inner tests, plus add in count for tests 540 # that didn't run properly ('result' object). 541 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 542 num_errors = junit_results.num_errors 543 if num_errors: 544 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 545 else: 546 buff.write(" * ERRORS: 0\n") 547 num_failures = junit_results.num_failures 548 if num_failures: 549 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 550 else: 551 buff.write(" * FAILURES: 0\n") 552 553 print(buff.getvalue())

554

Source Code for Module rosunit.junitxml