rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id$ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 from __future__ import print_function 41 42 import os 43 import sys 44 import cStringIO 45 import string 46 import codecs 47 import re 48 49 from xml.dom.minidom import parse, parseString 50 from xml.dom import Node as DomNode 51 52 import rospkg 53

54 -class TestInfo(object):

55 """ 56 Common container for 'error' and 'failure' results 57 """ 58

59 - def __init__(self, type_, text):

60 """ 61 @param type_: type attribute from xml 62 @type type_: str 63 @param text: text property from xml 64 @type text: str 65 """ 66 self.type = type_ 67 self.text = text

68

69 -class TestError(TestInfo):

70 """ 71 'error' result container 72 """

73 - def xml(self):

74 return u'<error type="%s"><![CDATA[%s]]></error>'%(self.type, self.text)

75

76 -class TestFailure(TestInfo):

77 """ 78 'failure' result container 79 """

80 - def xml(self):

81 return u'<failure type="%s"><![CDATA[%s]]></failure>'%(self.type, self.text)

82 83

84 -class TestCaseResult(object):

85 """ 86 'testcase' result container 87 """ 88

89 - def __init__(self, name):

90 """ 91 @param name: name of testcase 92 @type name: str 93 """ 94 self.name = name 95 self.failures = [] 96 self.errors = [] 97 self.time = 0.0 98 self.classname = ''

99

100 - def _passed(self):

101 """ 102 @return: True if test passed 103 @rtype: bool 104 """ 105 return not self.errors and not self.failures

106 ## bool: True if test passed without errors or failures 107 passed = property(_passed) 108

109 - def _failure_description(self):

110 """ 111 @return: description of testcase failure 112 @rtype: str 113 """ 114 if self.failures: 115 tmpl = "[%s][FAILURE]"%self.name 116 tmpl = tmpl + '-'*(80-len(tmpl)) 117 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 118 return '\n'.join(tmpl%x.text for x in self.failures) 119 return ''

120

121 - def _error_description(self):

122 """ 123 @return: description of testcase error 124 @rtype: str 125 """ 126 if self.errors: 127 tmpl = "[%s][ERROR]"%self.name 128 tmpl = tmpl + '-'*(80-len(tmpl)) 129 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 130 return '\n'.join(tmpl%x.text for x in self.errors) 131 return ''

132

133 - def _description(self):

134 """ 135 @return: description of testcase result 136 @rtype: str 137 """ 138 if self.passed: 139 return "[%s][passed]\n"%self.name 140 else: 141 return self._failure_description()+\ 142 self._error_description()

143 ## str: printable description of testcase result 144 description = property(_description)

145 - def add_failure(self, failure):

146 """ 147 @param failure TestFailure 148 """ 149 self.failures.append(failure)

150

151 - def add_error(self, error):

152 """ 153 @param failure TestError 154 """ 155 self.errors.append(error)

156

157 - def xml(self):

158 return u' <testcase classname="%s" name="%s" time="%s">\n'%(self.classname, self.name, self.time)+\ 159 '\n '.join([f.xml() for f in self.failures])+\ 160 '\n '.join([e.xml() for e in self.errors])+\ 161 ' </testcase>'

162

163 -class Result(object):

164 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 165 'test_case_results', 'system_out', 'system_err', 'time']

166 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

167 self.name = name 168 self.num_errors = num_errors 169 self.num_failures = num_failures 170 self.num_tests = num_tests 171 self.test_case_results = [] 172 self.system_out = '' 173 self.system_err = '' 174 self.time = 0.0

175

176 - def accumulate(self, r):

177 """ 178 Add results from r to this result 179 @param r: results to aggregate with this result 180 @type r: Result 181 """ 182 self.num_errors += r.num_errors 183 self.num_failures += r.num_failures 184 self.num_tests += r.num_tests 185 self.time += r.time 186 self.test_case_results.extend(r.test_case_results) 187 if r.system_out: 188 self.system_out += '\n'+r.system_out 189 if r.system_err: 190 self.system_err += '\n'+r.system_err

191

192 - def add_test_case_result(self, r):

193 """ 194 Add results from a testcase to this result container 195 @param r: TestCaseResult 196 @type r: TestCaseResult 197 """ 198 self.test_case_results.append(r)

199

200 - def xml(self):

201 """ 202 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 203 """ 204 return u'<?xml version="1.0" encoding="utf-8"?>'+\ 205 '<testsuite name="%s" tests="%s" errors="%s" failures="%s" time="%s">'%\ 206 (self.name, self.num_tests, self.num_errors, self.num_failures, self.time)+\ 207 '\n'.join([tc.xml() for tc in self.test_case_results])+\ 208 ' <system-out><![CDATA[%s]]></system-out>'%self.system_out+\ 209 ' <system-err><![CDATA[%s]]></system-err>'%self.system_err+\ 210 '</testsuite>'

211

212 -def _text(tag):

213 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

214

215 -def _load_suite_results(test_suite_name, test_suite, result):

216 nodes = [n for n in test_suite.childNodes \ 217 if n.nodeType == DomNode.ELEMENT_NODE] 218 for node in nodes: 219 name = node.tagName 220 if name == 'testsuite': 221 # for now we flatten this hierarchy 222 _load_suite_results(test_suite_name, node, result) 223 elif name == 'system-out': 224 if _text(node): 225 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 226 system_out += '\n'+_text(node) 227 result.system_out += system_out 228 elif name == 'system-err': 229 if _text(node): 230 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 231 system_err += '\n'+_text(node) 232 result.system_err += system_err 233 elif name == 'testcase': 234 name = node.getAttribute('name') or 'unknown' 235 classname = node.getAttribute('classname') or 'unknown' 236 237 # mangle the classname for some sense of uniformity 238 # between rostest/unittest/gtest 239 if '__main__.' in classname: 240 classname = classname[classname.find('__main__.')+9:] 241 if classname == 'rostest.rostest.RosTest': 242 classname = 'rostest' 243 elif not classname.startswith(result.name): 244 classname = "%s.%s"%(result.name,classname) 245 246 time = float(node.getAttribute('time')) or 0.0 247 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 248 tc_result.classname = classname 249 tc_result.time = time 250 result.add_test_case_result(tc_result) 251 for d in [n for n in node.childNodes \ 252 if n.nodeType == DomNode.ELEMENT_NODE]: 253 # convert 'message' attributes to text elements to keep 254 # python unittest and gtest consistent 255 if d.tagName == 'failure': 256 message = d.getAttribute('message') or '' 257 text = _text(d) or message 258 x = TestFailure(d.getAttribute('type') or '', text) 259 tc_result.add_failure(x) 260 elif d.tagName == 'error': 261 message = d.getAttribute('message') or '' 262 text = _text(d) or message 263 x = TestError(d.getAttribute('type') or '', text) 264 tc_result.add_error(x)

265 266 ## #603: unit test suites are not good about screening out illegal 267 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 268 ## screens these out 269 RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ 270 u'|' + \ 271 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ 272 (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 273 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 274 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff)) 275 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 276

277 -def _read_file_safe_xml(test_file):

278 """ 279 read in file, screen out unsafe unicode characters 280 """ 281 f = None 282 try: 283 # this is ugly, but the files in question that are problematic 284 # do not declare unicode type. 285 if not os.path.isfile(test_file): 286 raise Exception("test file does not exist") 287 try: 288 f = codecs.open(test_file, "r", "utf-8" ) 289 x = f.read() 290 except: 291 if f is not None: 292 f.close() 293 f = codecs.open(test_file, "r", "iso8859-1" ) 294 x = f.read() 295 296 for match in _safe_xml_regex.finditer(x): 297 x = x[:match.start()] + "?" + x[match.end():] 298 return x.encode("utf-8") 299 finally: 300 if f is not None: 301 f.close()

302

303 -def read(test_file, test_name):

304 """ 305 Read in the test_result file 306 @param test_file: test file path 307 @type test_file: str 308 @param test_name: name of test 309 @type test_name: str 310 @return: test results 311 @rtype: Result 312 """ 313 try: 314 xml_str = _read_file_safe_xml(test_file) 315 if not xml_str.strip(): 316 print("WARN: test result file is empty [%s]"%(test_file)) 317 return Result(test_name, 0, 0, 0) 318 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 319 except Exception as e: 320 print("WARN: cannot read test result file [%s]: %s"%(test_file, str(e))) 321 return Result(test_name, 0, 0, 0) 322 if not test_suites: 323 print("WARN: test result file [%s] contains no results"%(test_file)) 324 return Result(test_name, 0, 0, 0) 325 326 results = Result(test_name, 0, 0, 0) 327 for index, test_suite in enumerate(test_suites): 328 # skip test suites which are already covered by a parent test suite 329 if index > 0 and test_suite.parentNode in test_suites[0:index]: 330 continue 331 332 #test_suite = test_suite[0] 333 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 334 vals = [v or 0 for v in vals] 335 err, fail, tests = [string.atoi(val) for val in vals] 336 337 result = Result(test_name, err, fail, tests) 338 result.time = 0.0 if not len(test_suite.getAttribute('time')) else float(test_suite.getAttribute('time')) 339 340 # Create a prefix based on the test result filename. The idea is to 341 # disambiguate the case when tests of the same name are provided in 342 # different .xml files. We use the name of the parent directory 343 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 344 fname = os.path.basename(test_file) 345 if fname.startswith('TEST-'): 346 fname = fname[5:] 347 if fname.endswith('.xml'): 348 fname = fname[:-4] 349 test_file_base = "%s.%s"%(test_file_base, fname) 350 _load_suite_results(test_file_base, test_suite, result) 351 results.accumulate(result) 352 return results

353

354 -def read_all(filter_=[]):

355 """ 356 Read in the test_results and aggregate into a single Result object 357 @param filter_: list of packages that should be processed 358 @type filter_: [str] 359 @return: aggregated result 360 @rtype: L{Result} 361 """ 362 dir_ = rospkg.get_test_results_dir() 363 root_result = Result('ros', 0, 0, 0) 364 if not os.path.exists(dir_): 365 return root_result 366 for d in os.listdir(dir_): 367 if filter_ and not d in filter_: 368 continue 369 subdir = os.path.join(dir_, d) 370 if os.path.isdir(subdir): 371 for filename in os.listdir(subdir): 372 if filename.endswith('.xml'): 373 filename = os.path.join(subdir, filename) 374 result = read(filename, os.path.basename(subdir)) 375 root_result.accumulate(result) 376 return root_result

377 378

379 -def test_failure_junit_xml(test_name, message, stdout=None):

380 """ 381 Generate JUnit XML file for a unary test suite where the test failed 382 383 @param test_name: Name of test that failed 384 @type test_name: str 385 @param message: failure message 386 @type message: str 387 @param stdout: stdout data to include in report 388 @type stdout: str 389 """ 390 if not stdout: 391 return """<?xml version="1.0" encoding="UTF-8"?> 392 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 393 <testcase name="test_ran" status="run" time="1" classname="Results"> 394 <failure message="%s" type=""/> 395 </testcase> 396 </testsuite>"""%(test_name, message) 397 else: 398 return """<?xml version="1.0" encoding="UTF-8"?> 399 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 400 <testcase name="test_ran" status="run" time="1" classname="Results"> 401 <failure message="%s" type=""/> 402 </testcase> 403 <system-out><![CDATA[[ 404 %s 405 ]]></system-out> 406 </testsuite>"""%(test_name, message, stdout)

407

408 -def test_success_junit_xml(test_name):

409 """ 410 Generate JUnit XML file for a unary test suite where the test succeeded. 411 412 @param test_name: Name of test that passed 413 @type test_name: str 414 """ 415 return """<?xml version="1.0" encoding="UTF-8"?> 416 <testsuite tests="1" failures="0" time="1" errors="0" name="%s"> 417 <testcase name="test_ran" status="run" time="1" classname="Results"> 418 </testcase> 419 </testsuite>"""%(test_name)

420

421 -def print_summary(junit_results, runner_name='ROSUNIT'):

422 """ 423 Print summary of junitxml results to stdout. 424 """ 425 # we have two separate result objects, which can be a bit 426 # confusing. 'result' counts successful _running_ of tests 427 # (i.e. doesn't check for actual test success). The 'r' result 428 # object contains results of the actual tests. 429 430 buff = cStringIO.StringIO() 431 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 432 for tc_result in junit_results.test_case_results: 433 buff.write(tc_result.description) 434 435 buff.write('\nSUMMARY\n') 436 if (junit_results.num_errors + junit_results.num_failures) == 0: 437 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 438 else: 439 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 440 441 # TODO: still some issues with the numbers adding up if tests fail to launch 442 443 # number of errors from the inner tests, plus add in count for tests 444 # that didn't run properly ('result' object). 445 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 446 num_errors = junit_results.num_errors 447 if num_errors: 448 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 449 else: 450 buff.write(" * ERRORS: 0\n") 451 num_failures = junit_results.num_failures 452 if num_failures: 453 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 454 else: 455 buff.write(" * FAILURES: 0\n") 456 457 print(buff.getvalue())

458

Source Code for Module rosunit.junitxml