rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import os 43 import sys 44 import cStringIO 45 import string 46 import codecs 47 import re 48 49 from xml.dom.minidom import parse, parseString 50 from xml.dom import Node as DomNode 51 52 import rospkg 53

54 -class TestInfo(object):

55 """ 56 Common container for 'error' and 'failure' results 57 """ 58

59 - def __init__(self, type_, text):

60 """ 61 @param type_: type attribute from xml 62 @type type_: str 63 @param text: text property from xml 64 @type text: str 65 """ 66 self.type = type_ 67 self.text = text

68

69 -class TestError(TestInfo):

70 """ 71 'error' result container 72 """

73 - def xml(self):

74 return u'<error type="%s"><![CDATA[%s]]></error>'%(self.type, self.text)

75

76 -class TestFailure(TestInfo):

77 """ 78 'failure' result container 79 """

80 - def xml(self):

81 return u'<failure type="%s"><![CDATA[%s]]></failure>'%(self.type, self.text)

82 83

84 -class TestCaseResult(object):

85 """ 86 'testcase' result container 87 """ 88

89 - def __init__(self, name):

90 """ 91 @param name: name of testcase 92 @type name: str 93 """ 94 self.name = name 95 self.failures = [] 96 self.errors = [] 97 self.time = 0.0 98 self.classname = ''

99

100 - def _passed(self):

101 """ 102 @return: True if test passed 103 @rtype: bool 104 """ 105 return not self.errors and not self.failures

106 ## bool: True if test passed without errors or failures 107 passed = property(_passed) 108

109 - def _failure_description(self):

110 """ 111 @return: description of testcase failure 112 @rtype: str 113 """ 114 if self.failures: 115 tmpl = "[%s][FAILURE]"%self.name 116 tmpl = tmpl + '-'*(80-len(tmpl)) 117 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 118 return '\n'.join(tmpl%x.text for x in self.failures) 119 return ''

120

121 - def _error_description(self):

122 """ 123 @return: description of testcase error 124 @rtype: str 125 """ 126 if self.errors: 127 tmpl = "[%s][ERROR]"%self.name 128 tmpl = tmpl + '-'*(80-len(tmpl)) 129 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 130 return '\n'.join(tmpl%x.text for x in self.errors) 131 return ''

132

133 - def _description(self):

134 """ 135 @return: description of testcase result 136 @rtype: str 137 """ 138 if self.passed: 139 return "[%s][passed]\n"%self.name 140 else: 141 return self._failure_description()+\ 142 self._error_description()

143 ## str: printable description of testcase result 144 description = property(_description)

145 - def add_failure(self, failure):

146 """ 147 @param failure TestFailure 148 """ 149 self.failures.append(failure)

150

151 - def add_error(self, error):

152 """ 153 @param failure TestError 154 """ 155 self.errors.append(error)

156

157 - def xml(self):

158 return u' <testcase classname="%s" name="%s" time="%s">\n'%(self.classname, self.name, self.time)+\ 159 '\n '.join([f.xml() for f in self.failures])+\ 160 '\n '.join([e.xml() for e in self.errors])+\ 161 ' </testcase>'

162

163 -class Result(object):

164 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 165 'test_case_results', 'system_out', 'system_err', 'time']

166 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

167 self.name = name 168 self.num_errors = num_errors 169 self.num_failures = num_failures 170 self.num_tests = num_tests 171 self.test_case_results = [] 172 self.system_out = '' 173 self.system_err = '' 174 self.time = 0.0

175

176 - def accumulate(self, r):

177 """ 178 Add results from r to this result 179 @param r: results to aggregate with this result 180 @type r: Result 181 """ 182 self.num_errors += r.num_errors 183 self.num_failures += r.num_failures 184 self.num_tests += r.num_tests 185 self.time += r.time 186 self.test_case_results.extend(r.test_case_results) 187 if r.system_out: 188 self.system_out += '\n'+r.system_out 189 if r.system_err: 190 self.system_err += '\n'+r.system_err

191

192 - def add_test_case_result(self, r):

193 """ 194 Add results from a testcase to this result container 195 @param r: TestCaseResult 196 @type r: TestCaseResult 197 """ 198 self.test_case_results.append(r)

199

200 - def xml(self):

201 """ 202 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 203 """ 204 return u'<?xml version="1.0" encoding="utf-8"?>'+\ 205 '<testsuite name="%s" tests="%s" errors="%s" failures="%s" time="%s">'%\ 206 (self.name, self.num_tests, self.num_errors, self.num_failures, self.time)+\ 207 '\n'.join([tc.xml() for tc in self.test_case_results])+\ 208 ' <system-out><![CDATA[%s]]></system-out>'%self.system_out+\ 209 ' <system-err><![CDATA[%s]]></system-err>'%self.system_err+\ 210 '</testsuite>'

211

212 -def _text(tag):

213 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

214

215 -def _load_suite_results(test_suite_name, test_suite, result):

216 nodes = [n for n in test_suite.childNodes \ 217 if n.nodeType == DomNode.ELEMENT_NODE] 218 for node in nodes: 219 name = node.tagName 220 if name == 'testsuite': 221 # for now we flatten this hierarchy 222 _load_suite_results(test_suite_name, node, result) 223 elif name == 'system-out': 224 if _text(node): 225 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 226 system_out += '\n'+_text(node) 227 result.system_out += system_out 228 elif name == 'system-err': 229 if _text(node): 230 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 231 system_err += '\n'+_text(node) 232 result.system_err += system_err 233 elif name == 'testcase': 234 name = node.getAttribute('name') or 'unknown' 235 classname = node.getAttribute('classname') or 'unknown' 236 237 # mangle the classname for some sense of uniformity 238 # between rostest/unittest/gtest 239 if '__main__.' in classname: 240 classname = classname[classname.find('__main__.')+9:] 241 if classname == 'rostest.rostest.RosTest': 242 classname = 'rostest' 243 elif not classname.startswith(result.name): 244 classname = "%s.%s"%(result.name,classname) 245 246 time = float(node.getAttribute('time')) or 0.0 247 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 248 tc_result.classname = classname 249 tc_result.time = time 250 result.add_test_case_result(tc_result) 251 for d in [n for n in node.childNodes \ 252 if n.nodeType == DomNode.ELEMENT_NODE]: 253 # convert 'message' attributes to text elements to keep 254 # python unittest and gtest consistent 255 if d.tagName == 'failure': 256 message = d.getAttribute('message') or '' 257 text = _text(d) or message 258 x = TestFailure(d.getAttribute('type') or '', text) 259 tc_result.add_failure(x) 260 elif d.tagName == 'error': 261 message = d.getAttribute('message') or '' 262 text = _text(d) or message 263 x = TestError(d.getAttribute('type') or '', text) 264 tc_result.add_error(x)

265 266 ## #603: unit test suites are not good about screening out illegal 267 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 268 ## screens these out 269 RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ 270 u'|' + \ 271 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ 272 (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 273 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 274 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff)) 275 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 276

277 -def _read_file_safe_xml(test_file, write_back_sanitized=True):

278 """ 279 read in file, screen out unsafe unicode characters 280 """ 281 f = None 282 try: 283 # this is ugly, but the files in question that are problematic 284 # do not declare unicode type. 285 if not os.path.isfile(test_file): 286 raise Exception("test file does not exist") 287 try: 288 f = codecs.open(test_file, "r", "utf-8" ) 289 x = f.read() 290 except: 291 if f is not None: 292 f.close() 293 f = codecs.open(test_file, "r", "iso8859-1" ) 294 x = f.read() 295 296 for match in _safe_xml_regex.finditer(x): 297 x = x[:match.start()] + "?" + x[match.end():] 298 x = x.encode("utf-8") 299 if write_back_sanitized: 300 with open(test_file, 'w') as h: 301 h.write(x) 302 return x 303 finally: 304 if f is not None: 305 f.close()

306

307 -def read(test_file, test_name):

308 """ 309 Read in the test_result file 310 @param test_file: test file path 311 @type test_file: str 312 @param test_name: name of test 313 @type test_name: str 314 @return: test results 315 @rtype: Result 316 """ 317 try: 318 xml_str = _read_file_safe_xml(test_file) 319 if not xml_str.strip(): 320 print("WARN: test result file is empty [%s]"%(test_file)) 321 return Result(test_name, 0, 0, 0) 322 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 323 except Exception as e: 324 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 325 return Result(test_name, 0, 0, 0) 326 if not test_suites: 327 print("WARN: test result file [%s] contains no results"%(test_file)) 328 return Result(test_name, 0, 0, 0) 329 330 results = Result(test_name, 0, 0, 0) 331 for index, test_suite in enumerate(test_suites): 332 # skip test suites which are already covered by a parent test suite 333 if index > 0 and test_suite.parentNode in test_suites[0:index]: 334 continue 335 336 #test_suite = test_suite[0] 337 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 338 vals = [v or 0 for v in vals] 339 err, fail, tests = [string.atoi(val) for val in vals] 340 341 result = Result(test_name, err, fail, tests) 342 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 343 344 # Create a prefix based on the test result filename. The idea is to 345 # disambiguate the case when tests of the same name are provided in 346 # different .xml files. We use the name of the parent directory 347 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 348 fname = os.path.basename(test_file) 349 if fname.startswith('TEST-'): 350 fname = fname[5:] 351 if fname.endswith('.xml'): 352 fname = fname[:-4] 353 test_file_base = "%s.%s"%(test_file_base, fname) 354 _load_suite_results(test_file_base, test_suite, result) 355 results.accumulate(result) 356 return results

357

358 -def read_all(filter_=[]):

359 """ 360 Read in the test_results and aggregate into a single Result object 361 @param filter_: list of packages that should be processed 362 @type filter_: [str] 363 @return: aggregated result 364 @rtype: L{Result} 365 """ 366 dir_ = rospkg.get_test_results_dir() 367 root_result = Result('ros', 0, 0, 0) 368 if not os.path.exists(dir_): 369 return root_result 370 for d in os.listdir(dir_): 371 if filter_ and not d in filter_: 372 continue 373 subdir = os.path.join(dir_, d) 374 if os.path.isdir(subdir): 375 for filename in os.listdir(subdir): 376 if filename.endswith('.xml'): 377 filename = os.path.join(subdir, filename) 378 result = read(filename, os.path.basename(subdir)) 379 root_result.accumulate(result) 380 return root_result

381 382

383 -def test_failure_junit_xml(test_name, message, stdout=None):

384 """ 385 Generate JUnit XML file for a unary test suite where the test failed 386 387 @param test_name: Name of test that failed 388 @type test_name: str 389 @param message: failure message 390 @type message: str 391 @param stdout: stdout data to include in report 392 @type stdout: str 393 """ 394 if not stdout: 395 return """<?xml version="1.0" encoding="UTF-8"?> 396 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 397 <testcase name="test_ran" status="run" time="1" classname="Results"> 398 <failure message="%s" type=""/> 399 </testcase> 400 </testsuite>"""%(test_name, message) 401 else: 402 return """<?xml version="1.0" encoding="UTF-8"?> 403 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 404 <testcase name="test_ran" status="run" time="1" classname="Results"> 405 <failure message="%s" type=""/> 406 </testcase> 407 <system-out><![CDATA[[ 408 %s 409 ]]></system-out> 410 </testsuite>"""%(test_name, message, stdout)

411

412 -def test_success_junit_xml(test_name):

413 """ 414 Generate JUnit XML file for a unary test suite where the test succeeded. 415 416 @param test_name: Name of test that passed 417 @type test_name: str 418 """ 419 return """<?xml version="1.0" encoding="UTF-8"?> 420 <testsuite tests="1" failures="0" time="1" errors="0" name="%s"> 421 <testcase name="test_ran" status="run" time="1" classname="Results"> 422 </testcase> 423 </testsuite>"""%(test_name)

424

425 -def print_summary(junit_results, runner_name='ROSUNIT'):

426 """ 427 Print summary of junitxml results to stdout. 428 """ 429 # we have two separate result objects, which can be a bit 430 # confusing. 'result' counts successful _running_ of tests 431 # (i.e. doesn't check for actual test success). The 'r' result 432 # object contains results of the actual tests. 433 434 buff = cStringIO.StringIO() 435 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 436 for tc_result in junit_results.test_case_results: 437 buff.write(tc_result.description) 438 439 buff.write('\nSUMMARY\n') 440 if (junit_results.num_errors + junit_results.num_failures) == 0: 441 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 442 else: 443 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 444 445 # TODO: still some issues with the numbers adding up if tests fail to launch 446 447 # number of errors from the inner tests, plus add in count for tests 448 # that didn't run properly ('result' object). 449 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 450 num_errors = junit_results.num_errors 451 if num_errors: 452 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 453 else: 454 buff.write(" * ERRORS: 0\n") 455 num_failures = junit_results.num_failures 456 if num_failures: 457 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 458 else: 459 buff.write(" * FAILURES: 0\n") 460 461 print(buff.getvalue())

462

Source Code for Module rosunit.junitxml