rostest.xmlrunner

1 """ 2 XML Test Runner for PyUnit 3 """ 4 5 # Written by Sebastian Rittau <srittau@jroger.in-berlin.de> and placed in 6 # the Public Domain. With contributions by Paolo Borelli. 7 8 __revision__ = "$Id$" 9 10 import os.path 11 import re 12 import sys 13 import time 14 import traceback 15 import unittest 16 from StringIO import StringIO 17 from xml.sax.saxutils import escape 18 19 from StringIO import StringIO

20 21 22 -class _TestInfo(object):

23 24 """Information about a particular test. 25 26 Used by _XMLTestResult. 27 28 """ 29

30 - def __init__(self, test, time):

31 (self._class, self._method) = test.id().rsplit(".", 1) 32 self._time = time 33 self._error = None 34 self._failure = None

35 36 @staticmethod

37 - def create_success(test, time):

38 """Create a _TestInfo instance for a successful test.""" 39 return _TestInfo(test, time)

40 41 @staticmethod

42 - def create_failure(test, time, failure):

43 """Create a _TestInfo instance for a failed test.""" 44 info = _TestInfo(test, time) 45 info._failure = failure 46 return info

47 48 @staticmethod

49 - def create_error(test, time, error):

50 """Create a _TestInfo instance for an erroneous test.""" 51 info = _TestInfo(test, time) 52 info._error = error 53 return info

54

55 - def print_report(self, stream):

56 """Print information about this test case in XML format to the 57 supplied stream. 58 59 """ 60 stream.write(' <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \ 61 { 62 "class": self._class, 63 "method": self._method, 64 "time": self._time, 65 }) 66 if self._failure != None: 67 self._print_error(stream, 'failure', self._failure) 68 if self._error != None: 69 self._print_error(stream, 'error', self._error) 70 stream.write('</testcase>\n')

71

72 - def print_report_text(self, stream):

73 #stream.write(' <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \ 74 # { 75 # "class": self._class, 76 # "method": self._method, 77 # "time": self._time, 78 # }) 79 stream.write(self._method) 80 if self._failure != None: 81 stream.write(' ... FAILURE!\n') 82 self._print_error_text(stream, 'failure', self._failure) 83 if self._error != None: 84 stream.write(' ... ERROR!\n') 85 self._print_error_text(stream, 'error', self._error) 86 if self._failure == None and self._error == None: 87 stream.write(' ... ok\n')

88

89 - def _print_error(self, stream, tagname, error):

90 """Print information from a failure or error to the supplied stream.""" 91 text = escape(str(error[1])) 92 stream.write('\n') 93 stream.write(' <%s type="%s">%s\n' \ 94 % (tagname, str(error[0].__name__), text)) 95 tb_stream = StringIO() 96 traceback.print_tb(error[2], None, tb_stream) 97 stream.write(escape(tb_stream.getvalue())) 98 stream.write(' </%s>\n' % tagname) 99 stream.write(' ')

100

101 - def _print_error_text(self, stream, tagname, error):

102 """Print information from a failure or error to the supplied stream.""" 103 text = escape(str(error[1])) 104 stream.write('%s: %s\n' \ 105 % (tagname.upper(), text)) 106 tb_stream = StringIO() 107 traceback.print_tb(error[2], None, tb_stream) 108 stream.write(escape(tb_stream.getvalue())) 109 stream.write('-'*80 + '\n')

110

111 -class _XMLTestResult(unittest.TestResult):

112 113 """A test result class that stores result as XML. 114 115 Used by XMLTestRunner. 116 117 """ 118

119 - def __init__(self, classname):

120 unittest.TestResult.__init__(self) 121 self._test_name = classname 122 self._start_time = None 123 self._tests = [] 124 self._error = None 125 self._failure = None

126

127 - def startTest(self, test):

128 unittest.TestResult.startTest(self, test) 129 self._error = None 130 self._failure = None 131 self._start_time = time.time()

132

133 - def stopTest(self, test):

134 time_taken = time.time() - self._start_time 135 unittest.TestResult.stopTest(self, test) 136 if self._error: 137 info = _TestInfo.create_error(test, time_taken, self._error) 138 elif self._failure: 139 info = _TestInfo.create_failure(test, time_taken, self._failure) 140 else: 141 info = _TestInfo.create_success(test, time_taken) 142 self._tests.append(info)

143

144 - def addError(self, test, err):

145 unittest.TestResult.addError(self, test, err) 146 self._error = err

147

148 - def addFailure(self, test, err):

149 unittest.TestResult.addFailure(self, test, err) 150 self._failure = err

151

152 - def print_report(self, stream, time_taken, out, err):

153 """Prints the XML report to the supplied stream. 154 155 The time the tests took to perform as well as the captured standard 156 output and standard error streams must be passed in.a 157 158 """ 159 stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \ 160 { "e": len(self.errors), "f": len(self.failures) }) 161 stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \ 162 { 163 "n": self._test_name, 164 "t": self.testsRun, 165 "time": time_taken, 166 }) 167 for info in self._tests: 168 info.print_report(stream) 169 stream.write(' <system-out><![CDATA[%s]]></system-out>\n' % out) 170 stream.write(' <system-err><![CDATA[%s]]></system-err>\n' % err) 171 stream.write('</testsuite>\n')

172

173 - def print_report_text(self, stream, time_taken, out, err):

174 """Prints the text report to the supplied stream. 175 176 The time the tests took to perform as well as the captured standard 177 output and standard error streams must be passed in.a 178 179 """ 180 #stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \ 181 # { "e": len(self.errors), "f": len(self.failures) }) 182 #stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \ 183 # { 184 # "n": self._test_name, 185 # "t": self.testsRun, 186 # "time": time_taken, 187 # }) 188 for info in self._tests: 189 info.print_report_text(stream)

190

191 192 -class XMLTestRunner(object):

193 194 """A test runner that stores results in XML format compatible with JUnit. 195 196 XMLTestRunner(stream=None) -> XML test runner 197 198 The XML file is written to the supplied stream. If stream is None, the 199 results are stored in a file called TEST-<module>.<class>.xml in the 200 current working directory (if not overridden with the path property), 201 where <module> and <class> are the module and class name of the test class. 202 203 """ 204

205 - def __init__(self, stream=None):

206 self._stream = stream 207 self._path = "."

208

209 - def run(self, test):

210 """Run the given test case or test suite.""" 211 class_ = test.__class__ 212 classname = class_.__module__ + "." + class_.__name__ 213 if self._stream == None: 214 filename = "TEST-%s.xml" % classname 215 stream = file(os.path.join(self._path, filename), "w") 216 stream.write('<?xml version="1.0" encoding="utf-8"?>\n') 217 else: 218 stream = self._stream 219 220 result = _XMLTestResult(classname) 221 start_time = time.time() 222 223 # TODO: Python 2.5: Use the with statement 224 old_stdout = sys.stdout 225 old_stderr = sys.stderr 226 sys.stdout = StringIO() 227 sys.stderr = StringIO() 228 229 try: 230 test(result) 231 try: 232 out_s = sys.stdout.getvalue() 233 except AttributeError: 234 out_s = "" 235 try: 236 err_s = sys.stderr.getvalue() 237 except AttributeError: 238 err_s = "" 239 finally: 240 sys.stdout = old_stdout 241 sys.stderr = old_stderr 242 243 time_taken = time.time() - start_time 244 result.print_report(stream, time_taken, out_s, err_s) 245 246 result.print_report_text(sys.stdout, time_taken, out_s, err_s) 247 248 if self._stream == None: 249 stream.close() 250 251 return result

252

253 - def _set_path(self, path):

254 self._path = path

255 256 path = property(lambda self: self._path, _set_path, None, 257 """The path where the XML files are stored. 258 259 This property is ignored when the XML file is written to a file 260 stream.""")

261

262 263 -class XMLTestRunnerTest(unittest.TestCase):

264 - def setUp(self):

265 self._stream = StringIO()

266

267 - def _try_test_run(self, test_class, expected):

268 269 """Run the test suite against the supplied test class and compare the 270 XML result against the expected XML string. Fail if the expected 271 string doesn't match the actual string. All time attribute in the 272 expected string should have the value "0.000". All error and failure 273 messages are reduced to "Foobar". 274 275 """ 276 277 runner = XMLTestRunner(self._stream) 278 runner.run(unittest.makeSuite(test_class)) 279 280 got = self._stream.getvalue() 281 # Replace all time="X.YYY" attributes by time="0.000" to enable a 282 # simple string comparison. 283 got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got) 284 # Likewise, replace all failure and error messages by a simple "Foobar" 285 # string. 286 got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got) 287 got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got) 288 289 self.assertEqual(expected, got)

290

291 - def test_no_tests(self):

292 """Regression test: Check whether a test run without any tests 293 matches a previous run. 294 295 """ 296 class TestTest(unittest.TestCase): 297 pass

298 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000"> 299 <system-out><![CDATA[]]></system-out> 300 <system-err><![CDATA[]]></system-err> 301 </testsuite> 302 """)

303

304 - def test_success(self):

305 """Regression test: Check whether a test run with a successful test 306 matches a previous run. 307 308 """ 309 class TestTest(unittest.TestCase): 310 def test_foo(self): 311 pass

312 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> 313 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> 314 <system-out><![CDATA[]]></system-out> 315 <system-err><![CDATA[]]></system-err> 316 </testsuite> 317 """) 318

319 - def test_failure(self):

320 """Regression test: Check whether a test run with a failing test 321 matches a previous run. 322 323 """ 324 class TestTest(unittest.TestCase): 325 def test_foo(self): 326 self.assert_(False)

327 self._try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000"> 328 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"> 329 <failure type="exceptions.AssertionError">Foobar</failure> 330 </testcase> 331 <system-out><![CDATA[]]></system-out> 332 <system-err><![CDATA[]]></system-err> 333 </testsuite> 334 """) 335

336 - def test_error(self):

337 """Regression test: Check whether a test run with a erroneous test 338 matches a previous run. 339 340 """ 341 class TestTest(unittest.TestCase): 342 def test_foo(self): 343 raise IndexError()

344 self._try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> 345 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"> 346 <error type="exceptions.IndexError">Foobar</error> 347 </testcase> 348 <system-out><![CDATA[]]></system-out> 349 <system-err><![CDATA[]]></system-err> 350 </testsuite> 351 """) 352

353 - def test_stdout_capture(self):

354 """Regression test: Check whether a test run with output to stdout 355 matches a previous run. 356 357 """ 358 class TestTest(unittest.TestCase): 359 def test_foo(self): 360 print "Test"

361 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> 362 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> 363 <system-out><![CDATA[Test 364 ]]></system-out> 365 <system-err><![CDATA[]]></system-err> 366 </testsuite> 367 """) 368

369 - def test_stderr_capture(self):

370 """Regression test: Check whether a test run with output to stderr 371 matches a previous run. 372 373 """ 374 class TestTest(unittest.TestCase): 375 def test_foo(self): 376 print >>sys.stderr, "Test"

377 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> 378 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> 379 <system-out><![CDATA[]]></system-out> 380 <system-err><![CDATA[Test 381 ]]></system-err> 382 </testsuite> 383 """) 384

385 - class NullStream(object):

386 """A file-like object that discards everything written to it."""

387 - def write(self, buffer):

388 pass

389

390 - def test_unittests_changing_stdout(self):

391 """Check whether the XMLTestRunner recovers gracefully from unit tests 392 that change stdout, but don't change it back properly. 393 394 """ 395 class TestTest(unittest.TestCase): 396 def test_foo(self): 397 sys.stdout = XMLTestRunnerTest.NullStream()

398 399 runner = XMLTestRunner(self._stream) 400 runner.run(unittest.makeSuite(TestTest)) 401

402 - def test_unittests_changing_stderr(self):

403 """Check whether the XMLTestRunner recovers gracefully from unit tests 404 that change stderr, but don't change it back properly. 405 406 """ 407 class TestTest(unittest.TestCase): 408 def test_foo(self): 409 sys.stderr = XMLTestRunnerTest.NullStream()

410 411 runner = XMLTestRunner(self._stream) 412 runner.run(unittest.makeSuite(TestTest)) 413

414 415 -class XMLTestProgram(unittest.TestProgram):

416 - def runTests(self):

417 if self.testRunner is None: 418 self.testRunner = XMLTestRunner() 419 unittest.TestProgram.runTests(self)

420 421 main = XMLTestProgram 422 423 424 if __name__ == "__main__": 425 main(module=None) 426

Source Code for Module rostest.xmlrunner