rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import os 43 import sys 44 try: 45 from cStringIO import StringIO 46 except ImportError: 47 from io import StringIO 48 import string 49 import codecs 50 import re 51 52 import xml.etree.ElementTree as ET 53 from xml.dom.minidom import parse, parseString 54 from xml.dom import Node as DomNode 55 56 from functools import reduce 57 import rospkg 58 59 invalid_chars = re.compile(ur'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF\u0100-\uD7FF\uE000-\uFDCF\uFDE0-\uFFFD]')

60 -def invalid_char_replacer(m):

61 return "&#x"+('%04X' % ord(m.group(0)))+";"

62 -def filter_nonprintable_text(text):

63 return re.sub(invalid_chars, invalid_char_replacer, text)

64

65 -def cdata(cdata_text):

66 return '<![CDATA[\n{}\n]]>'.format(cdata_text)

67

68 -class TestInfo(object):

69 """ 70 Common container for 'error' and 'failure' results 71 """ 72

73 - def __init__(self, type_, text):

74 """ 75 @param type_: type attribute from xml 76 @type type_: str 77 @param text: text property from xml 78 @type text: str 79 """ 80 self.type = type_ 81 self.text = text

82

83 -class TestError(TestInfo):

84 """ 85 'error' result container 86 """

87 - def xml(self):

88 """ 89 @return XML tag representing the object, with non-XML text filtered out 90 @rtype: str 91 """ 92 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

93

94 - def xml_element(self):

95 """ 96 @return XML tag representing the object, with non-XML text filtered out 97 @rtype: xml.etree.ElementTree.Element 98 """ 99 error = ET.Element('error') 100 error.set('type', self.type) 101 error.text = cdata(filter_nonprintable_text(self.text)) 102 return error

103

104 -class TestFailure(TestInfo):

105 """ 106 'failure' result container 107 """

108 - def xml(self):

109 """ 110 @return XML tag representing the object, with non-XML text filtered out 111 @rtype: str 112 """ 113 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

114

115 - def xml_element(self):

116 """ 117 @return XML tag representing the object, with non-XML text filtered out 118 @rtype: xml.etree.ElementTree.Element 119 """ 120 error = ET.Element('failure') 121 error.set('type', self.type) 122 error.text = cdata(filter_nonprintable_text(self.text)) 123 return error

124

125 -class TestCaseResult(object):

126 """ 127 'testcase' result container 128 """ 129

130 - def __init__(self, name):

131 """ 132 @param name: name of testcase 133 @type name: str 134 """ 135 self.name = name 136 self.failures = [] 137 self.errors = [] 138 self.time = 0.0 139 self.classname = ''

140

141 - def _passed(self):

142 """ 143 @return: True if test passed 144 @rtype: bool 145 """ 146 return not self.errors and not self.failures

147 ## bool: True if test passed without errors or failures 148 passed = property(_passed) 149

150 - def _failure_description(self):

151 """ 152 @return: description of testcase failure 153 @rtype: str 154 """ 155 if self.failures: 156 tmpl = "[%s][FAILURE]"%self.name 157 tmpl = tmpl + '-'*(80-len(tmpl)) 158 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 159 return '\n'.join(tmpl%x.text for x in self.failures) 160 return ''

161

162 - def _error_description(self):

163 """ 164 @return: description of testcase error 165 @rtype: str 166 """ 167 if self.errors: 168 tmpl = "[%s][ERROR]"%self.name 169 tmpl = tmpl + '-'*(80-len(tmpl)) 170 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 171 return '\n'.join(tmpl%x.text for x in self.errors) 172 return ''

173

174 - def _description(self):

175 """ 176 @return: description of testcase result 177 @rtype: str 178 """ 179 if self.passed: 180 return "[%s][passed]\n"%self.name 181 else: 182 return self._failure_description()+\ 183 self._error_description()

184 ## str: printable description of testcase result 185 description = property(_description)

186 - def add_failure(self, failure):

187 """ 188 @param failure TestFailure 189 """ 190 self.failures.append(failure)

191

192 - def add_error(self, error):

193 """ 194 @param failure TestError 195 """ 196 self.errors.append(error)

197

198 - def xml(self):

199 """ 200 @return XML tag representing the object, with non-XML text filtered out 201 @rtype: str 202 """ 203 return ET.tostring(self.xml_element(), encoding='utf-8', method='xml')

204

205 - def xml_element(self):

206 """ 207 @return XML tag representing the object, with non-XML text filtered out 208 @rtype: xml.etree.ElementTree.Element 209 """ 210 testcase = ET.Element('testcase') 211 testcase.set('classname', self.classname) 212 testcase.set('name', self.name) 213 testcase.set('time', str(self.time)) 214 for f in self.failures: 215 testcase.append(f.xml_element()) 216 for e in self.errors: 217 testcase.append(e.xml_element()) 218 return testcase

219

220 -class Result(object):

221 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 222 'test_case_results', 'system_out', 'system_err', 'time']

223 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

224 self.name = name 225 self.num_errors = num_errors 226 self.num_failures = num_failures 227 self.num_tests = num_tests 228 self.test_case_results = [] 229 self.system_out = '' 230 self.system_err = '' 231 self.time = 0.0

232

233 - def accumulate(self, r):

234 """ 235 Add results from r to this result 236 @param r: results to aggregate with this result 237 @type r: Result 238 """ 239 self.num_errors += r.num_errors 240 self.num_failures += r.num_failures 241 self.num_tests += r.num_tests 242 self.time += r.time 243 self.test_case_results.extend(r.test_case_results) 244 if r.system_out: 245 self.system_out += '\n'+r.system_out 246 if r.system_err: 247 self.system_err += '\n'+r.system_err

248

249 - def add_test_case_result(self, r):

250 """ 251 Add results from a testcase to this result container 252 @param r: TestCaseResult 253 @type r: TestCaseResult 254 """ 255 self.test_case_results.append(r)

256

257 - def xml_element(self):

258 """ 259 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 260 """ 261 testsuite = ET.Element('testsuite') 262 testsuite.set('tests', str(self.num_tests)) 263 testsuite.set('failures', str(self.num_failures)) 264 testsuite.set('time', str(self.time)) 265 testsuite.set('errors', str(self.num_errors)) 266 testsuite.set('name', self.name) 267 for tc in self.test_case_results: 268 tc.xml(testsuite) 269 system_out = ET.SubElement(testsuite, 'system-out') 270 system_out.text = cdata(filter_nonprintable_text(self.system_out)) 271 system_err = ET.SubElement(testsuite, 'system-err') 272 system_err.text = cdata(filter_nonprintable_text(self.system_err)) 273 return ET.tostring(testsuite, encoding='utf-8', method='xml')

274

275 -def _text(tag):

276 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

277

278 -def _load_suite_results(test_suite_name, test_suite, result):

279 nodes = [n for n in test_suite.childNodes \ 280 if n.nodeType == DomNode.ELEMENT_NODE] 281 for node in nodes: 282 name = node.tagName 283 if name == 'testsuite': 284 # for now we flatten this hierarchy 285 _load_suite_results(test_suite_name, node, result) 286 elif name == 'system-out': 287 if _text(node): 288 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 289 system_out += '\n'+_text(node) 290 result.system_out += system_out 291 elif name == 'system-err': 292 if _text(node): 293 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 294 system_err += '\n'+_text(node) 295 result.system_err += system_err 296 elif name == 'testcase': 297 name = node.getAttribute('name') or 'unknown' 298 classname = node.getAttribute('classname') or 'unknown' 299 300 # mangle the classname for some sense of uniformity 301 # between rostest/unittest/gtest 302 if '__main__.' in classname: 303 classname = classname[classname.find('__main__.')+9:] 304 if classname == 'rostest.rostest.RosTest': 305 classname = 'rostest' 306 elif not classname.startswith(result.name): 307 classname = "%s.%s"%(result.name,classname) 308 309 time = float(node.getAttribute('time')) or 0.0 310 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 311 tc_result.classname = classname 312 tc_result.time = time 313 result.add_test_case_result(tc_result) 314 for d in [n for n in node.childNodes \ 315 if n.nodeType == DomNode.ELEMENT_NODE]: 316 # convert 'message' attributes to text elements to keep 317 # python unittest and gtest consistent 318 if d.tagName == 'failure': 319 message = d.getAttribute('message') or '' 320 text = _text(d) or message 321 x = TestFailure(d.getAttribute('type') or '', text) 322 tc_result.add_failure(x) 323 elif d.tagName == 'error': 324 message = d.getAttribute('message') or '' 325 text = _text(d) or message 326 x = TestError(d.getAttribute('type') or '', text) 327 tc_result.add_error(x)

328 329 ## #603: unit test suites are not good about screening out illegal 330 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 331 ## screens these out 332 try: 333 char = unichr 334 except NameError: 335 char = chr 336 RE_XML_ILLEGAL = '([%s-%s%s-%s%s-%s%s-%s])' + \ 337 '|' + \ 338 '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' 339 try: 340 RE_XML_ILLEGAL = unicode(RE_XML_ILLEGAL) 341 except NameError: 342 pass 343 RE_XML_ILLEGAL = RE_XML_ILLEGAL % \ 344 (char(0x0000),char(0x0008),char(0x000b),char(0x000c), 345 char(0x000e),char(0x001f),char(0xfffe),char(0xffff), 346 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 347 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 348 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff)) 349 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 350

351 -def _read_file_safe_xml(test_file, write_back_sanitized=True):

352 """ 353 read in file, screen out unsafe unicode characters 354 """ 355 f = None 356 try: 357 # this is ugly, but the files in question that are problematic 358 # do not declare unicode type. 359 if not os.path.isfile(test_file): 360 raise Exception("test file does not exist") 361 try: 362 f = codecs.open(test_file, "r", "utf-8" ) 363 x = f.read() 364 except: 365 if f is not None: 366 f.close() 367 f = codecs.open(test_file, "r", "iso8859-1" ) 368 x = f.read() 369 370 for match in _safe_xml_regex.finditer(x): 371 x = x[:match.start()] + "?" + x[match.end():] 372 x = x.encode("utf-8") 373 if write_back_sanitized: 374 with open(test_file, 'wb') as h: 375 h.write(x) 376 return x 377 finally: 378 if f is not None: 379 f.close()

380

381 -def read(test_file, test_name):

382 """ 383 Read in the test_result file 384 @param test_file: test file path 385 @type test_file: str 386 @param test_name: name of test 387 @type test_name: str 388 @return: test results 389 @rtype: Result 390 """ 391 try: 392 xml_str = _read_file_safe_xml(test_file) 393 if not xml_str.strip(): 394 print("WARN: test result file is empty [%s]"%(test_file)) 395 return Result(test_name, 0, 0, 0) 396 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 397 except Exception as e: 398 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 399 return Result(test_name, 0, 0, 0) 400 if not test_suites: 401 print("WARN: test result file [%s] contains no results"%(test_file)) 402 return Result(test_name, 0, 0, 0) 403 404 results = Result(test_name, 0, 0, 0) 405 for index, test_suite in enumerate(test_suites): 406 # skip test suites which are already covered by a parent test suite 407 if index > 0 and test_suite.parentNode in test_suites[0:index]: 408 continue 409 410 #test_suite = test_suite[0] 411 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 412 vals = [v or 0 for v in vals] 413 err, fail, tests = [int(val) for val in vals] 414 415 result = Result(test_name, err, fail, tests) 416 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 417 418 # Create a prefix based on the test result filename. The idea is to 419 # disambiguate the case when tests of the same name are provided in 420 # different .xml files. We use the name of the parent directory 421 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 422 fname = os.path.basename(test_file) 423 if fname.startswith('TEST-'): 424 fname = fname[5:] 425 if fname.endswith('.xml'): 426 fname = fname[:-4] 427 test_file_base = "%s.%s"%(test_file_base, fname) 428 _load_suite_results(test_file_base, test_suite, result) 429 results.accumulate(result) 430 return results

431

432 -def read_all(filter_=[]):

433 """ 434 Read in the test_results and aggregate into a single Result object 435 @param filter_: list of packages that should be processed 436 @type filter_: [str] 437 @return: aggregated result 438 @rtype: L{Result} 439 """ 440 dir_ = rospkg.get_test_results_dir() 441 root_result = Result('ros', 0, 0, 0) 442 if not os.path.exists(dir_): 443 return root_result 444 for d in os.listdir(dir_): 445 if filter_ and not d in filter_: 446 continue 447 subdir = os.path.join(dir_, d) 448 if os.path.isdir(subdir): 449 for filename in os.listdir(subdir): 450 if filename.endswith('.xml'): 451 filename = os.path.join(subdir, filename) 452 result = read(filename, os.path.basename(subdir)) 453 root_result.accumulate(result) 454 return root_result

455 456

457 -def test_failure_junit_xml(test_name, message, stdout=None):

458 """ 459 Generate JUnit XML file for a unary test suite where the test failed 460 461 @param test_name: Name of test that failed 462 @type test_name: str 463 @param message: failure message 464 @type message: str 465 @param stdout: stdout data to include in report 466 @type stdout: str 467 """ 468 testsuite = ET.Element('testsuite') 469 testsuite.set('tests', '1') 470 testsuite.set('failures', '1') 471 testsuite.set('time', '1') 472 testsuite.set('errors', '0') 473 testsuite.set('name', test_name) 474 testcase = ET.SubElement(testsuite, 'testcase') 475 testcase.set('name', 'test_ran') 476 testcase.set('status', 'run') 477 testcase.set('time', '1') 478 testcase.set('classname', 'Results') 479 failure = ET.SubElement(testcase, 'failure') 480 failure.set('message', message) 481 failure.set('type', '') 482 if stdout: 483 system_out = ET.SubElement(testsuite, 'system-out') 484 system_out.text = cdata(filter_nonprintable_text(stdout)) 485 return ET.tostring(testsuite, encoding='utf-8', method='xml')

486

487 -def test_success_junit_xml(test_name):

488 """ 489 Generate JUnit XML file for a unary test suite where the test succeeded. 490 491 @param test_name: Name of test that passed 492 @type test_name: str 493 """ 494 testsuite = ET.Element('testsuite') 495 testsuite.set('tests', '1') 496 testsuite.set('failures', '0') 497 testsuite.set('time', '1') 498 testsuite.set('errors', '0') 499 testsuite.set('name', test_name) 500 testcase = ET.SubElement(testsuite, 'testcase') 501 testcase.set('name', 'test_ran') 502 testcase.set('status', 'run') 503 testcase.set('time', '1') 504 testcase.set('classname', 'Results') 505 return ET.tostring(testsuite, encoding='utf-8', method='xml')

506

507 -def print_summary(junit_results, runner_name='ROSUNIT'):

508 """ 509 Print summary of junitxml results to stdout. 510 """ 511 # we have two separate result objects, which can be a bit 512 # confusing. 'result' counts successful _running_ of tests 513 # (i.e. doesn't check for actual test success). The 'r' result 514 # object contains results of the actual tests. 515 516 buff = StringIO() 517 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 518 for tc_result in junit_results.test_case_results: 519 buff.write(tc_result.description) 520 521 buff.write('\nSUMMARY\n') 522 if (junit_results.num_errors + junit_results.num_failures) == 0: 523 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 524 else: 525 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 526 527 # TODO: still some issues with the numbers adding up if tests fail to launch 528 529 # number of errors from the inner tests, plus add in count for tests 530 # that didn't run properly ('result' object). 531 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 532 num_errors = junit_results.num_errors 533 if num_errors: 534 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 535 else: 536 buff.write(" * ERRORS: 0\n") 537 num_failures = junit_results.num_failures 538 if num_failures: 539 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 540 else: 541 buff.write(" * FAILURES: 0\n") 542 543 print(buff.getvalue())

544

Source Code for Module rosunit.junitxml