rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import os 43 import sys 44 try: 45 from cStringIO import StringIO 46 except ImportError: 47 from io import StringIO 48 import string 49 import codecs 50 import re 51 52 from xml.dom.minidom import parse, parseString 53 from xml.dom import Node as DomNode 54 55 from functools import reduce 56 import rospkg 57

58 -class TestInfo(object):

59 """ 60 Common container for 'error' and 'failure' results 61 """ 62

63 - def __init__(self, type_, text):

64 """ 65 @param type_: type attribute from xml 66 @type type_: str 67 @param text: text property from xml 68 @type text: str 69 """ 70 self.type = type_ 71 self.text = text

72

73 -class TestError(TestInfo):

74 """ 75 'error' result container 76 """

77 - def xml(self):

78 data = '<error type="%s"><![CDATA[%s]]></error>' % (self.type, self.text) 79 try: 80 data = unicode(data) 81 except NameError: 82 pass 83 return data

84

85 -class TestFailure(TestInfo):

86 """ 87 'failure' result container 88 """

89 - def xml(self):

90 data = '<failure type="%s"><![CDATA[%s]]></failure>' % (self.type, self.text) 91 try: 92 data = unicode(data) 93 except NameError: 94 pass 95 return data

96 97

98 -class TestCaseResult(object):

99 """ 100 'testcase' result container 101 """ 102

103 - def __init__(self, name):

104 """ 105 @param name: name of testcase 106 @type name: str 107 """ 108 self.name = name 109 self.failures = [] 110 self.errors = [] 111 self.time = 0.0 112 self.classname = ''

113

114 - def _passed(self):

115 """ 116 @return: True if test passed 117 @rtype: bool 118 """ 119 return not self.errors and not self.failures

120 ## bool: True if test passed without errors or failures 121 passed = property(_passed) 122

123 - def _failure_description(self):

124 """ 125 @return: description of testcase failure 126 @rtype: str 127 """ 128 if self.failures: 129 tmpl = "[%s][FAILURE]"%self.name 130 tmpl = tmpl + '-'*(80-len(tmpl)) 131 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 132 return '\n'.join(tmpl%x.text for x in self.failures) 133 return ''

134

135 - def _error_description(self):

136 """ 137 @return: description of testcase error 138 @rtype: str 139 """ 140 if self.errors: 141 tmpl = "[%s][ERROR]"%self.name 142 tmpl = tmpl + '-'*(80-len(tmpl)) 143 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 144 return '\n'.join(tmpl%x.text for x in self.errors) 145 return ''

146

147 - def _description(self):

148 """ 149 @return: description of testcase result 150 @rtype: str 151 """ 152 if self.passed: 153 return "[%s][passed]\n"%self.name 154 else: 155 return self._failure_description()+\ 156 self._error_description()

157 ## str: printable description of testcase result 158 description = property(_description)

159 - def add_failure(self, failure):

160 """ 161 @param failure TestFailure 162 """ 163 self.failures.append(failure)

164

165 - def add_error(self, error):

166 """ 167 @param failure TestError 168 """ 169 self.errors.append(error)

170

171 - def xml(self):

172 data = ' <testcase classname="%s" name="%s" time="%s">\n' % (self.classname, self.name, self.time) + \ 173 '\n '.join([f.xml() for f in self.failures]) + \ 174 '\n '.join([e.xml() for e in self.errors]) + \ 175 ' </testcase>' 176 try: 177 data = unicode(data) 178 except NameError: 179 pass 180 return data

181

182 -class Result(object):

183 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 184 'test_case_results', 'system_out', 'system_err', 'time']

185 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

186 self.name = name 187 self.num_errors = num_errors 188 self.num_failures = num_failures 189 self.num_tests = num_tests 190 self.test_case_results = [] 191 self.system_out = '' 192 self.system_err = '' 193 self.time = 0.0

194

195 - def accumulate(self, r):

196 """ 197 Add results from r to this result 198 @param r: results to aggregate with this result 199 @type r: Result 200 """ 201 self.num_errors += r.num_errors 202 self.num_failures += r.num_failures 203 self.num_tests += r.num_tests 204 self.time += r.time 205 self.test_case_results.extend(r.test_case_results) 206 if r.system_out: 207 self.system_out += '\n'+r.system_out 208 if r.system_err: 209 self.system_err += '\n'+r.system_err

210

211 - def add_test_case_result(self, r):

212 """ 213 Add results from a testcase to this result container 214 @param r: TestCaseResult 215 @type r: TestCaseResult 216 """ 217 self.test_case_results.append(r)

218

219 - def xml(self):

220 """ 221 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 222 """ 223 data = '<?xml version="1.0" encoding="utf-8"?>' + \ 224 '<testsuite name="%s" tests="%s" errors="%s" failures="%s" time="%s">' % \ 225 (self.name, self.num_tests, self.num_errors, self.num_failures, self.time) + \ 226 '\n'.join([tc.xml() for tc in self.test_case_results]) + \ 227 ' <system-out><![CDATA[%s]]></system-out>' % self.system_out + \ 228 ' <system-err><![CDATA[%s]]></system-err>' % self.system_err + \ 229 '</testsuite>' 230 try: 231 data = unicode(data) 232 except NameError: 233 pass 234 return data

235

236 -def _text(tag):

237 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

238

239 -def _load_suite_results(test_suite_name, test_suite, result):

240 nodes = [n for n in test_suite.childNodes \ 241 if n.nodeType == DomNode.ELEMENT_NODE] 242 for node in nodes: 243 name = node.tagName 244 if name == 'testsuite': 245 # for now we flatten this hierarchy 246 _load_suite_results(test_suite_name, node, result) 247 elif name == 'system-out': 248 if _text(node): 249 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 250 system_out += '\n'+_text(node) 251 result.system_out += system_out 252 elif name == 'system-err': 253 if _text(node): 254 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 255 system_err += '\n'+_text(node) 256 result.system_err += system_err 257 elif name == 'testcase': 258 name = node.getAttribute('name') or 'unknown' 259 classname = node.getAttribute('classname') or 'unknown' 260 261 # mangle the classname for some sense of uniformity 262 # between rostest/unittest/gtest 263 if '__main__.' in classname: 264 classname = classname[classname.find('__main__.')+9:] 265 if classname == 'rostest.rostest.RosTest': 266 classname = 'rostest' 267 elif not classname.startswith(result.name): 268 classname = "%s.%s"%(result.name,classname) 269 270 time = float(node.getAttribute('time')) or 0.0 271 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 272 tc_result.classname = classname 273 tc_result.time = time 274 result.add_test_case_result(tc_result) 275 for d in [n for n in node.childNodes \ 276 if n.nodeType == DomNode.ELEMENT_NODE]: 277 # convert 'message' attributes to text elements to keep 278 # python unittest and gtest consistent 279 if d.tagName == 'failure': 280 message = d.getAttribute('message') or '' 281 text = _text(d) or message 282 x = TestFailure(d.getAttribute('type') or '', text) 283 tc_result.add_failure(x) 284 elif d.tagName == 'error': 285 message = d.getAttribute('message') or '' 286 text = _text(d) or message 287 x = TestError(d.getAttribute('type') or '', text) 288 tc_result.add_error(x)

289 290 ## #603: unit test suites are not good about screening out illegal 291 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 292 ## screens these out 293 try: 294 char = unichr 295 except NameError: 296 char = chr 297 RE_XML_ILLEGAL = '([%s-%s%s-%s%s-%s%s-%s])' + \ 298 '|' + \ 299 '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' 300 try: 301 RE_XML_ILLEGAL = unicode(RE_XML_ILLEGAL) 302 except NameError: 303 pass 304 RE_XML_ILLEGAL = RE_XML_ILLEGAL % \ 305 (char(0x0000),char(0x0008),char(0x000b),char(0x000c), 306 char(0x000e),char(0x001f),char(0xfffe),char(0xffff), 307 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 308 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff), 309 char(0xd800),char(0xdbff),char(0xdc00),char(0xdfff)) 310 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 311

312 -def _read_file_safe_xml(test_file, write_back_sanitized=True):

313 """ 314 read in file, screen out unsafe unicode characters 315 """ 316 f = None 317 try: 318 # this is ugly, but the files in question that are problematic 319 # do not declare unicode type. 320 if not os.path.isfile(test_file): 321 raise Exception("test file does not exist") 322 try: 323 f = codecs.open(test_file, "r", "utf-8" ) 324 x = f.read() 325 except: 326 if f is not None: 327 f.close() 328 f = codecs.open(test_file, "r", "iso8859-1" ) 329 x = f.read() 330 331 for match in _safe_xml_regex.finditer(x): 332 x = x[:match.start()] + "?" + x[match.end():] 333 x = x.encode("utf-8") 334 if write_back_sanitized: 335 with open(test_file, 'wb') as h: 336 h.write(x) 337 return x 338 finally: 339 if f is not None: 340 f.close()

341

342 -def read(test_file, test_name):

343 """ 344 Read in the test_result file 345 @param test_file: test file path 346 @type test_file: str 347 @param test_name: name of test 348 @type test_name: str 349 @return: test results 350 @rtype: Result 351 """ 352 try: 353 xml_str = _read_file_safe_xml(test_file) 354 if not xml_str.strip(): 355 print("WARN: test result file is empty [%s]"%(test_file)) 356 return Result(test_name, 0, 0, 0) 357 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 358 except Exception as e: 359 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 360 return Result(test_name, 0, 0, 0) 361 if not test_suites: 362 print("WARN: test result file [%s] contains no results"%(test_file)) 363 return Result(test_name, 0, 0, 0) 364 365 results = Result(test_name, 0, 0, 0) 366 for index, test_suite in enumerate(test_suites): 367 # skip test suites which are already covered by a parent test suite 368 if index > 0 and test_suite.parentNode in test_suites[0:index]: 369 continue 370 371 #test_suite = test_suite[0] 372 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 373 vals = [v or 0 for v in vals] 374 err, fail, tests = [int(val) for val in vals] 375 376 result = Result(test_name, err, fail, tests) 377 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 378 379 # Create a prefix based on the test result filename. The idea is to 380 # disambiguate the case when tests of the same name are provided in 381 # different .xml files. We use the name of the parent directory 382 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 383 fname = os.path.basename(test_file) 384 if fname.startswith('TEST-'): 385 fname = fname[5:] 386 if fname.endswith('.xml'): 387 fname = fname[:-4] 388 test_file_base = "%s.%s"%(test_file_base, fname) 389 _load_suite_results(test_file_base, test_suite, result) 390 results.accumulate(result) 391 return results

392

393 -def read_all(filter_=[]):

394 """ 395 Read in the test_results and aggregate into a single Result object 396 @param filter_: list of packages that should be processed 397 @type filter_: [str] 398 @return: aggregated result 399 @rtype: L{Result} 400 """ 401 dir_ = rospkg.get_test_results_dir() 402 root_result = Result('ros', 0, 0, 0) 403 if not os.path.exists(dir_): 404 return root_result 405 for d in os.listdir(dir_): 406 if filter_ and not d in filter_: 407 continue 408 subdir = os.path.join(dir_, d) 409 if os.path.isdir(subdir): 410 for filename in os.listdir(subdir): 411 if filename.endswith('.xml'): 412 filename = os.path.join(subdir, filename) 413 result = read(filename, os.path.basename(subdir)) 414 root_result.accumulate(result) 415 return root_result

416 417

418 -def test_failure_junit_xml(test_name, message, stdout=None):

419 """ 420 Generate JUnit XML file for a unary test suite where the test failed 421 422 @param test_name: Name of test that failed 423 @type test_name: str 424 @param message: failure message 425 @type message: str 426 @param stdout: stdout data to include in report 427 @type stdout: str 428 """ 429 if not stdout: 430 return """<?xml version="1.0" encoding="UTF-8"?> 431 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 432 <testcase name="test_ran" status="run" time="1" classname="Results"> 433 <failure message="%s" type=""/> 434 </testcase> 435 </testsuite>"""%(test_name, message) 436 else: 437 return """<?xml version="1.0" encoding="UTF-8"?> 438 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 439 <testcase name="test_ran" status="run" time="1" classname="Results"> 440 <failure message="%s" type=""/> 441 </testcase> 442 <system-out><![CDATA[[ 443 %s 444 ]]></system-out> 445 </testsuite>"""%(test_name, message, stdout)

446

447 -def test_success_junit_xml(test_name):

448 """ 449 Generate JUnit XML file for a unary test suite where the test succeeded. 450 451 @param test_name: Name of test that passed 452 @type test_name: str 453 """ 454 return """<?xml version="1.0" encoding="UTF-8"?> 455 <testsuite tests="1" failures="0" time="1" errors="0" name="%s"> 456 <testcase name="test_ran" status="run" time="1" classname="Results"> 457 </testcase> 458 </testsuite>"""%(test_name)

459

460 -def print_summary(junit_results, runner_name='ROSUNIT'):

461 """ 462 Print summary of junitxml results to stdout. 463 """ 464 # we have two separate result objects, which can be a bit 465 # confusing. 'result' counts successful _running_ of tests 466 # (i.e. doesn't check for actual test success). The 'r' result 467 # object contains results of the actual tests. 468 469 buff = StringIO() 470 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 471 for tc_result in junit_results.test_case_results: 472 buff.write(tc_result.description) 473 474 buff.write('\nSUMMARY\n') 475 if (junit_results.num_errors + junit_results.num_failures) == 0: 476 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 477 else: 478 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 479 480 # TODO: still some issues with the numbers adding up if tests fail to launch 481 482 # number of errors from the inner tests, plus add in count for tests 483 # that didn't run properly ('result' object). 484 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 485 num_errors = junit_results.num_errors 486 if num_errors: 487 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 488 else: 489 buff.write(" * ERRORS: 0\n") 490 num_failures = junit_results.num_failures 491 if num_failures: 492 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 493 else: 494 buff.write(" * FAILURES: 0\n") 495 496 print(buff.getvalue())

497

Source Code for Module rosunit.junitxml