rosunit.junitxml

1 #!/usr/bin/env python 2 # Software License Agreement (BSD License) 3 # 4 # Copyright (c) 2008, Willow Garage, Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions 9 # are met: 10 # 11 # * Redistributions of source code must retain the above copyright 12 # notice, this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above 14 # copyright notice, this list of conditions and the following 15 # disclaimer in the documentation and/or other materials provided 16 # with the distribution. 17 # * Neither the name of Willow Garage, Inc. nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 # POSSIBILITY OF SUCH DAMAGE. 33 # 34 # Revision $Id: junitxml.py 14103 2011-07-06 21:39:16Z kwc $ 35 36 """ 37 Library for reading and manipulating Ant JUnit XML result files. 38 """ 39 40 import os 41 import sys 42 import cStringIO 43 import string 44 import codecs 45 import re 46 47 from xml.dom.minidom import parse, parseString 48 from xml.dom import Node as DomNode 49 50 import roslib.rosenv 51

52 -class TestInfo(object):

53 """ 54 Common container for 'error' and 'failure' results 55 """ 56

57 - def __init__(self, type_, text):

58 """ 59 @param type_: type attribute from xml 60 @type type_: str 61 @param text: text property from xml 62 @type text: str 63 """ 64 self.type = type_ 65 self.text = text

66

67 -class TestError(TestInfo):

68 """ 69 'error' result container 70 """

71 - def xml(self):

72 return u'<error type="%s"><![CDATA[%s]]></error>'%(self.type, self.text)

73

74 -class TestFailure(TestInfo):

75 """ 76 'failure' result container 77 """

78 - def xml(self):

79 return u'<failure type="%s"><![CDATA[%s]]></failure>'%(self.type, self.text)

80 81

82 -class TestCaseResult(object):

83 """ 84 'testcase' result container 85 """ 86

87 - def __init__(self, name):

88 """ 89 @param name: name of testcase 90 @type name: str 91 """ 92 self.name = name 93 self.failures = [] 94 self.errors = [] 95 self.time = 0.0 96 self.classname = ''

97

98 - def _passed(self):

99 """ 100 @return: True if test passed 101 @rtype: bool 102 """ 103 return not self.errors and not self.failures

104 ## bool: True if test passed without errors or failures 105 passed = property(_passed) 106

107 - def _failure_description(self):

108 """ 109 @return: description of testcase failure 110 @rtype: str 111 """ 112 if self.failures: 113 tmpl = "[%s][FAILURE]"%self.name 114 tmpl = tmpl + '-'*(80-len(tmpl)) 115 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 116 return '\n'.join(tmpl%x.text for x in self.failures) 117 return ''

118

119 - def _error_description(self):

120 """ 121 @return: description of testcase error 122 @rtype: str 123 """ 124 if self.errors: 125 tmpl = "[%s][ERROR]"%self.name 126 tmpl = tmpl + '-'*(80-len(tmpl)) 127 tmpl = tmpl+"\n%s\n"+'-'*80+"\n\n" 128 return '\n'.join(tmpl%x.text for x in self.errors) 129 return ''

130

131 - def _description(self):

132 """ 133 @return: description of testcase result 134 @rtype: str 135 """ 136 if self.passed: 137 return "[%s][passed]\n"%self.name 138 else: 139 return self._failure_description()+\ 140 self._error_description()

141 ## str: printable description of testcase result 142 description = property(_description)

143 - def add_failure(self, failure):

144 """ 145 @param failure TestFailure 146 """ 147 self.failures.append(failure)

148

149 - def add_error(self, error):

150 """ 151 @param failure TestError 152 """ 153 self.errors.append(error)

154

155 - def xml(self):

156 return u' <testcase classname="%s" name="%s" time="%s">\n'%(self.classname, self.name, self.time)+\ 157 '\n '.join([f.xml() for f in self.failures])+\ 158 '\n '.join([e.xml() for e in self.errors])+\ 159 ' </testcase>'

160

161 -class Result(object):

162 __slots__ = ['name', 'num_errors', 'num_failures', 'num_tests', \ 163 'test_case_results', 'system_out', 'system_err', 'time']

164 - def __init__(self, name, num_errors=0, num_failures=0, num_tests=0):

165 self.name = name 166 self.num_errors = num_errors 167 self.num_failures = num_failures 168 self.num_tests = num_tests 169 self.test_case_results = [] 170 self.system_out = '' 171 self.system_err = '' 172 self.time = 0.0

173

174 - def accumulate(self, r):

175 """ 176 Add results from r to this result 177 @param r: results to aggregate with this result 178 @type r: Result 179 """ 180 self.num_errors += r.num_errors 181 self.num_failures += r.num_failures 182 self.num_tests += r.num_tests 183 self.test_case_results.extend(r.test_case_results) 184 if r.system_out: 185 self.system_out += '\n'+r.system_out 186 if r.system_err: 187 self.system_err += '\n'+r.system_err

188

189 - def add_test_case_result(self, r):

190 """ 191 Add results from a testcase to this result container 192 @param r: TestCaseResult 193 @type r: TestCaseResult 194 """ 195 self.test_case_results.append(r)

196

197 - def xml(self):

198 """ 199 @return: document as unicode (UTF-8 declared) XML according to Ant JUnit spec 200 """ 201 return u'<?xml version="1.0" encoding="utf-8"?>'+\ 202 '<testsuite name="%s" tests="%s" errors="%s" failures="%s" time="%s">'%\ 203 (self.name, self.num_tests, self.num_errors, self.num_failures, self.time)+\ 204 '\n'.join([tc.xml() for tc in self.test_case_results])+\ 205 ' <system-out><![CDATA[%s]]></system-out>'%self.system_out+\ 206 ' <system-err><![CDATA[%s]]></system-err>'%self.system_err+\ 207 '</testsuite>'

208

209 -def _text(tag):

210 return reduce(lambda x, y: x + y, [c.data for c in tag.childNodes if c.nodeType in [DomNode.TEXT_NODE, DomNode.CDATA_SECTION_NODE]], "").strip()

211

212 -def _load_suite_results(test_suite_name, test_suite, result):

213 nodes = [n for n in test_suite.childNodes \ 214 if n.nodeType == DomNode.ELEMENT_NODE] 215 for node in nodes: 216 name = node.tagName 217 if name == 'testsuite': 218 # for now we flatten this hierarchy 219 _load_suite_results(test_suite_name, node, result) 220 elif name == 'system-out': 221 if _text(node): 222 system_out = "[%s] stdout"%test_suite_name + "-"*(71-len(test_suite_name)) 223 system_out += '\n'+_text(node) 224 result.system_out += system_out 225 elif name == 'system-err': 226 if _text(node): 227 system_err = "[%s] stderr"%test_suite_name + "-"*(71-len(test_suite_name)) 228 system_err += '\n'+_text(node) 229 result.system_err += system_err 230 elif name == 'testcase': 231 name = node.getAttribute('name') or 'unknown' 232 classname = node.getAttribute('classname') or 'unknown' 233 234 # mangle the classname for some sense of uniformity 235 # between rostest/unittest/gtest 236 if '__main__.' in classname: 237 classname = classname[classname.find('__main__.')+9:] 238 if classname == 'rostest.rostest.RosTest': 239 classname = 'rostest' 240 elif not classname.startswith(result.name): 241 classname = "%s.%s"%(result.name,classname) 242 243 time = node.getAttribute('time') or 0.0 244 tc_result = TestCaseResult("%s/%s"%(test_suite_name,name)) 245 tc_result.classname = classname 246 tc_result.time = time 247 result.add_test_case_result(tc_result) 248 for d in [n for n in node.childNodes \ 249 if n.nodeType == DomNode.ELEMENT_NODE]: 250 # convert 'message' attributes to text elements to keep 251 # python unittest and gtest consistent 252 if d.tagName == 'failure': 253 message = d.getAttribute('message') or '' 254 text = _text(d) or message 255 x = TestFailure(d.getAttribute('type') or '', text) 256 tc_result.add_failure(x) 257 elif d.tagName == 'error': 258 message = d.getAttribute('message') or '' 259 text = _text(d) or message 260 x = TestError(d.getAttribute('type') or '', text) 261 tc_result.add_error(x)

262 263 ## #603: unit test suites are not good about screening out illegal 264 ## unicode characters. This little recipe I from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML 265 ## screens these out 266 RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ 267 u'|' + \ 268 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ 269 (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 270 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), 271 unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff)) 272 _safe_xml_regex = re.compile(RE_XML_ILLEGAL) 273

274 -def _read_file_safe_xml(test_file):

275 """ 276 read in file, screen out unsafe unicode characters 277 """ 278 f = None 279 try: 280 # this is ugly, but the files in question that are problematic 281 # do not declare unicode type. 282 if not os.path.isfile(test_file): 283 raise Exception("test file does not exist") 284 try: 285 f = codecs.open(test_file, "r", "utf-8" ) 286 x = f.read() 287 except: 288 if f is not None: 289 f.close() 290 f = codecs.open(test_file, "r", "iso8859-1" ) 291 x = f.read() 292 293 for match in _safe_xml_regex.finditer(x): 294 x = x[:match.start()] + "?" + x[match.end():] 295 return x.encode("utf-8") 296 finally: 297 if f is not None: 298 f.close()

299

300 -def read(test_file, test_name):

301 """ 302 Read in the test_result file 303 @param test_file: test file path 304 @type test_file: str 305 @param test_name: name of test 306 @type test_name: str 307 @return: test results 308 @rtype: Result 309 """ 310 try: 311 xml_str = _read_file_safe_xml(test_file) 312 if not xml_str.strip(): 313 print "WARN: test result file is empty [%s]"%(test_file) 314 return Result(test_name, 0, 0, 0) 315 test_suites = parseString(xml_str).getElementsByTagName('testsuite') 316 except Exception as e: 317 print "WARN: cannot read test result file [%s]: %s"%(test_file, str(e)) 318 return Result(test_name, 0, 0, 0) 319 if not test_suites: 320 print "WARN: test result file [%s] contains no results"%(test_file) 321 return Result(test_name, 0, 0, 0) 322 323 results = Result(test_name, 0, 0, 0) 324 for test_suite in test_suites: 325 #test_suite = test_suite[0] 326 vals = [test_suite.getAttribute(attr) for attr in ['errors', 'failures', 'tests']] 327 vals = [v or 0 for v in vals] 328 err, fail, tests = [string.atoi(val) for val in vals] 329 330 result = Result(test_name, err, fail, tests) 331 result.time = test_suite.getAttribute('time') or 0.0 332 333 # Create a prefix based on the test result filename. The idea is to 334 # disambiguate the case when tests of the same name are provided in 335 # different .xml files. We use the name of the parent directory 336 test_file_base = os.path.basename(os.path.dirname(os.path.abspath(test_file))) 337 fname = os.path.basename(test_file) 338 if fname.startswith('TEST-'): 339 fname = fname[5:] 340 if fname.endswith('.xml'): 341 fname = fname[:-4] 342 test_file_base = "%s.%s"%(test_file_base, fname) 343 _load_suite_results(test_file_base, test_suite, result) 344 results.accumulate(result) 345 return results

346

347 -def read_all(filter_=[]):

348 """ 349 Read in the test_results and aggregate into a single Result object 350 @param filter_: list of packages that should be processed 351 @type filter_: [str] 352 @return: aggregated result 353 @rtype: L{Result} 354 """ 355 dir_ = roslib.rosenv.get_test_results_dir() 356 root_result = Result('ros', 0, 0, 0) 357 if not os.path.exists(dir_): 358 return root_result 359 for d in os.listdir(dir_): 360 if filter_ and not d in filter_: 361 continue 362 subdir = os.path.join(dir_, d) 363 if os.path.isdir(subdir): 364 for filename in os.listdir(subdir): 365 if filename.endswith('.xml'): 366 filename = os.path.join(subdir, filename) 367 result = read(filename, os.path.basename(subdir)) 368 root_result.accumulate(result) 369 return root_result

370 371

372 -def test_failure_junit_xml(test_name, message, stdout=None):

373 """ 374 Generate JUnit XML file for a unary test suite where the test failed 375 376 @param test_name: Name of test that failed 377 @type test_name: str 378 @param message: failure message 379 @type message: str 380 @param stdout: stdout data to include in report 381 @type stdout: str 382 """ 383 if not stdout: 384 return """<?xml version="1.0" encoding="UTF-8"?> 385 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 386 <testcase name="test_ran" status="run" time="1" classname="Results"> 387 <failure message="%s" type=""/> 388 </testcase> 389 </testsuite>"""%(test_name, message) 390 else: 391 return """<?xml version="1.0" encoding="UTF-8"?> 392 <testsuite tests="1" failures="1" time="1" errors="0" name="%s"> 393 <testcase name="test_ran" status="run" time="1" classname="Results"> 394 <failure message="%s" type=""/> 395 </testcase> 396 <system-out><![CDATA[[ 397 %s 398 ]]></system-out> 399 </testsuite>"""%(test_name, message, stdout)

400

401 -def test_success_junit_xml(test_name):

402 """ 403 Generate JUnit XML file for a unary test suite where the test succeeded. 404 405 @param test_name: Name of test that passed 406 @type test_name: str 407 """ 408 return """<?xml version="1.0" encoding="UTF-8"?> 409 <testsuite tests="1" failures="0" time="1" errors="0" name="%s"> 410 <testcase name="test_ran" status="run" time="1" classname="Results"> 411 </testcase> 412 </testsuite>"""%(test_name)

413

414 -def print_summary(junit_results, runner_name='ROSUNIT'):

415 """ 416 Print summary of junitxml results to stdout. 417 """ 418 # we have two separate result objects, which can be a bit 419 # confusing. 'result' counts successful _running_ of tests 420 # (i.e. doesn't check for actual test success). The 'r' result 421 # object contains results of the actual tests. 422 423 buff = cStringIO.StringIO() 424 buff.write("[%s]"%runner_name+'-'*71+'\n\n') 425 for tc_result in junit_results.test_case_results: 426 buff.write(tc_result.description) 427 428 buff.write('\nSUMMARY\n') 429 if (junit_results.num_errors + junit_results.num_failures) == 0: 430 buff.write("\033[32m * RESULT: SUCCESS\033[0m\n") 431 else: 432 buff.write("\033[1;31m * RESULT: FAIL\033[0m\n") 433 434 # TODO: still some issues with the numbers adding up if tests fail to launch 435 436 # number of errors from the inner tests, plus add in count for tests 437 # that didn't run properly ('result' object). 438 buff.write(" * TESTS: %s\n"%junit_results.num_tests) 439 num_errors = junit_results.num_errors 440 if num_errors: 441 buff.write("\033[1;31m * ERRORS: %s\033[0m\n"%num_errors) 442 else: 443 buff.write(" * ERRORS: 0\n") 444 num_failures = junit_results.num_failures 445 if num_failures: 446 buff.write("\033[1;31m * FAILURES: %s\033[0m\n"%num_failures) 447 else: 448 buff.write(" * FAILURES: 0\n") 449 450 print buff.getvalue()

451

Source Code for Module rosunit.junitxml