httpclient_test.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 from __future__ import absolute_import, division, print_function, with_statement
00004 
00005 import base64
00006 import binascii
00007 from contextlib import closing
00008 import functools
00009 import sys
00010 import threading
00011 
00012 from tornado.escape import utf8
00013 from tornado.httpclient import HTTPRequest, HTTPResponse, _RequestProxy, HTTPError, HTTPClient
00014 from tornado.httpserver import HTTPServer
00015 from tornado.ioloop import IOLoop
00016 from tornado.iostream import IOStream
00017 from tornado.log import gen_log
00018 from tornado import netutil
00019 from tornado.stack_context import ExceptionStackContext, NullContext
00020 from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
00021 from tornado.test.util import unittest, skipOnTravis
00022 from tornado.util import u, bytes_type
00023 from tornado.web import Application, RequestHandler, url
00024 
00025 try:
00026     from io import BytesIO  # python 3
00027 except ImportError:
00028     from cStringIO import StringIO as BytesIO
00029 
00030 
00031 class HelloWorldHandler(RequestHandler):
00032     def get(self):
00033         name = self.get_argument("name", "world")
00034         self.set_header("Content-Type", "text/plain")
00035         self.finish("Hello %s!" % name)
00036 
00037 
00038 class PostHandler(RequestHandler):
00039     def post(self):
00040         self.finish("Post arg1: %s, arg2: %s" % (
00041             self.get_argument("arg1"), self.get_argument("arg2")))
00042 
00043 
00044 class ChunkHandler(RequestHandler):
00045     def get(self):
00046         self.write("asdf")
00047         self.flush()
00048         self.write("qwer")
00049 
00050 
00051 class AuthHandler(RequestHandler):
00052     def get(self):
00053         self.finish(self.request.headers["Authorization"])
00054 
00055 
00056 class CountdownHandler(RequestHandler):
00057     def get(self, count):
00058         count = int(count)
00059         if count > 0:
00060             self.redirect(self.reverse_url("countdown", count - 1))
00061         else:
00062             self.write("Zero")
00063 
00064 
00065 class EchoPostHandler(RequestHandler):
00066     def post(self):
00067         self.write(self.request.body)
00068 
00069 
00070 class UserAgentHandler(RequestHandler):
00071     def get(self):
00072         self.write(self.request.headers.get('User-Agent', 'User agent not set'))
00073 
00074 
00075 class ContentLength304Handler(RequestHandler):
00076     def get(self):
00077         self.set_status(304)
00078         self.set_header('Content-Length', 42)
00079 
00080     def _clear_headers_for_304(self):
00081         # Tornado strips content-length from 304 responses, but here we
00082         # want to simulate servers that include the headers anyway.
00083         pass
00084 
00085 
00086 class AllMethodsHandler(RequestHandler):
00087     SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + ('OTHER',)
00088 
00089     def method(self):
00090         self.write(self.request.method)
00091 
00092     get = post = put = delete = options = patch = other = method
00093 
00094 # These tests end up getting run redundantly: once here with the default
00095 # HTTPClient implementation, and then again in each implementation's own
00096 # test suite.
00097 
00098 
00099 class HTTPClientCommonTestCase(AsyncHTTPTestCase):
00100     def get_app(self):
00101         return Application([
00102             url("/hello", HelloWorldHandler),
00103             url("/post", PostHandler),
00104             url("/chunk", ChunkHandler),
00105             url("/auth", AuthHandler),
00106             url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
00107             url("/echopost", EchoPostHandler),
00108             url("/user_agent", UserAgentHandler),
00109             url("/304_with_content_length", ContentLength304Handler),
00110             url("/all_methods", AllMethodsHandler),
00111         ], gzip=True)
00112 
00113     @skipOnTravis
00114     def test_hello_world(self):
00115         response = self.fetch("/hello")
00116         self.assertEqual(response.code, 200)
00117         self.assertEqual(response.headers["Content-Type"], "text/plain")
00118         self.assertEqual(response.body, b"Hello world!")
00119         self.assertEqual(int(response.request_time), 0)
00120 
00121         response = self.fetch("/hello?name=Ben")
00122         self.assertEqual(response.body, b"Hello Ben!")
00123 
00124     def test_streaming_callback(self):
00125         # streaming_callback is also tested in test_chunked
00126         chunks = []
00127         response = self.fetch("/hello",
00128                               streaming_callback=chunks.append)
00129         # with streaming_callback, data goes to the callback and not response.body
00130         self.assertEqual(chunks, [b"Hello world!"])
00131         self.assertFalse(response.body)
00132 
00133     def test_post(self):
00134         response = self.fetch("/post", method="POST",
00135                               body="arg1=foo&arg2=bar")
00136         self.assertEqual(response.code, 200)
00137         self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
00138 
00139     def test_chunked(self):
00140         response = self.fetch("/chunk")
00141         self.assertEqual(response.body, b"asdfqwer")
00142 
00143         chunks = []
00144         response = self.fetch("/chunk",
00145                               streaming_callback=chunks.append)
00146         self.assertEqual(chunks, [b"asdf", b"qwer"])
00147         self.assertFalse(response.body)
00148 
00149     def test_chunked_close(self):
00150         # test case in which chunks spread read-callback processing
00151         # over several ioloop iterations, but the connection is already closed.
00152         sock, port = bind_unused_port()
00153         with closing(sock):
00154             def write_response(stream, request_data):
00155                 stream.write(b"""\
00156 HTTP/1.1 200 OK
00157 Transfer-Encoding: chunked
00158 
00159 1
00160 1
00161 1
00162 2
00163 0
00164 
00165 """.replace(b"\n", b"\r\n"), callback=stream.close)
00166 
00167             def accept_callback(conn, address):
00168                 # fake an HTTP server using chunked encoding where the final chunks
00169                 # and connection close all happen at once
00170                 stream = IOStream(conn, io_loop=self.io_loop)
00171                 stream.read_until(b"\r\n\r\n",
00172                                   functools.partial(write_response, stream))
00173             netutil.add_accept_handler(sock, accept_callback, self.io_loop)
00174             self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
00175             resp = self.wait()
00176             resp.rethrow()
00177             self.assertEqual(resp.body, b"12")
00178             self.io_loop.remove_handler(sock.fileno())
00179 
00180     def test_streaming_stack_context(self):
00181         chunks = []
00182         exc_info = []
00183 
00184         def error_handler(typ, value, tb):
00185             exc_info.append((typ, value, tb))
00186             return True
00187 
00188         def streaming_cb(chunk):
00189             chunks.append(chunk)
00190             if chunk == b'qwer':
00191                 1 / 0
00192 
00193         with ExceptionStackContext(error_handler):
00194             self.fetch('/chunk', streaming_callback=streaming_cb)
00195 
00196         self.assertEqual(chunks, [b'asdf', b'qwer'])
00197         self.assertEqual(1, len(exc_info))
00198         self.assertIs(exc_info[0][0], ZeroDivisionError)
00199 
00200     def test_basic_auth(self):
00201         self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
00202                                     auth_password="open sesame").body,
00203                          b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
00204 
00205     def test_basic_auth_explicit_mode(self):
00206         self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
00207                                     auth_password="open sesame",
00208                                     auth_mode="basic").body,
00209                          b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
00210 
00211     def test_unsupported_auth_mode(self):
00212         # curl and simple clients handle errors a bit differently; the
00213         # important thing is that they don't fall back to basic auth
00214         # on an unknown mode.
00215         with ExpectLog(gen_log, "uncaught exception", required=False):
00216             with self.assertRaises((ValueError, HTTPError)):
00217                 response = self.fetch("/auth", auth_username="Aladdin",
00218                                       auth_password="open sesame",
00219                                       auth_mode="asdf")
00220                 response.rethrow()
00221 
00222     def test_follow_redirect(self):
00223         response = self.fetch("/countdown/2", follow_redirects=False)
00224         self.assertEqual(302, response.code)
00225         self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
00226 
00227         response = self.fetch("/countdown/2")
00228         self.assertEqual(200, response.code)
00229         self.assertTrue(response.effective_url.endswith("/countdown/0"))
00230         self.assertEqual(b"Zero", response.body)
00231 
00232     def test_credentials_in_url(self):
00233         url = self.get_url("/auth").replace("http://", "http://me:secret@")
00234         self.http_client.fetch(url, self.stop)
00235         response = self.wait()
00236         self.assertEqual(b"Basic " + base64.b64encode(b"me:secret"),
00237                          response.body)
00238 
00239     def test_body_encoding(self):
00240         unicode_body = u("\xe9")
00241         byte_body = binascii.a2b_hex(b"e9")
00242 
00243         # unicode string in body gets converted to utf8
00244         response = self.fetch("/echopost", method="POST", body=unicode_body,
00245                               headers={"Content-Type": "application/blah"})
00246         self.assertEqual(response.headers["Content-Length"], "2")
00247         self.assertEqual(response.body, utf8(unicode_body))
00248 
00249         # byte strings pass through directly
00250         response = self.fetch("/echopost", method="POST",
00251                               body=byte_body,
00252                               headers={"Content-Type": "application/blah"})
00253         self.assertEqual(response.headers["Content-Length"], "1")
00254         self.assertEqual(response.body, byte_body)
00255 
00256         # Mixing unicode in headers and byte string bodies shouldn't
00257         # break anything
00258         response = self.fetch("/echopost", method="POST", body=byte_body,
00259                               headers={"Content-Type": "application/blah"},
00260                               user_agent=u("foo"))
00261         self.assertEqual(response.headers["Content-Length"], "1")
00262         self.assertEqual(response.body, byte_body)
00263 
00264     def test_types(self):
00265         response = self.fetch("/hello")
00266         self.assertEqual(type(response.body), bytes_type)
00267         self.assertEqual(type(response.headers["Content-Type"]), str)
00268         self.assertEqual(type(response.code), int)
00269         self.assertEqual(type(response.effective_url), str)
00270 
00271     def test_header_callback(self):
00272         first_line = []
00273         headers = {}
00274         chunks = []
00275 
00276         def header_callback(header_line):
00277             if header_line.startswith('HTTP/'):
00278                 first_line.append(header_line)
00279             elif header_line != '\r\n':
00280                 k, v = header_line.split(':', 1)
00281                 headers[k] = v.strip()
00282 
00283         def streaming_callback(chunk):
00284             # All header callbacks are run before any streaming callbacks,
00285             # so the header data is available to process the data as it
00286             # comes in.
00287             self.assertEqual(headers['Content-Type'], 'text/html; charset=UTF-8')
00288             chunks.append(chunk)
00289 
00290         self.fetch('/chunk', header_callback=header_callback,
00291                    streaming_callback=streaming_callback)
00292         self.assertEqual(len(first_line), 1)
00293         self.assertRegexpMatches(first_line[0], 'HTTP/1.[01] 200 OK\r\n')
00294         self.assertEqual(chunks, [b'asdf', b'qwer'])
00295 
00296     def test_header_callback_stack_context(self):
00297         exc_info = []
00298 
00299         def error_handler(typ, value, tb):
00300             exc_info.append((typ, value, tb))
00301             return True
00302 
00303         def header_callback(header_line):
00304             if header_line.startswith('Content-Type:'):
00305                 1 / 0
00306 
00307         with ExceptionStackContext(error_handler):
00308             self.fetch('/chunk', header_callback=header_callback)
00309         self.assertEqual(len(exc_info), 1)
00310         self.assertIs(exc_info[0][0], ZeroDivisionError)
00311 
00312     def test_configure_defaults(self):
00313         defaults = dict(user_agent='TestDefaultUserAgent', allow_ipv6=False)
00314         # Construct a new instance of the configured client class
00315         client = self.http_client.__class__(self.io_loop, force_instance=True,
00316                                             defaults=defaults)
00317         client.fetch(self.get_url('/user_agent'), callback=self.stop)
00318         response = self.wait()
00319         self.assertEqual(response.body, b'TestDefaultUserAgent')
00320         client.close()
00321 
00322     def test_304_with_content_length(self):
00323         # According to the spec 304 responses SHOULD NOT include
00324         # Content-Length or other entity headers, but some servers do it
00325         # anyway.
00326         # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
00327         response = self.fetch('/304_with_content_length')
00328         self.assertEqual(response.code, 304)
00329         self.assertEqual(response.headers['Content-Length'], '42')
00330 
00331     def test_final_callback_stack_context(self):
00332         # The final callback should be run outside of the httpclient's
00333         # stack_context.  We want to ensure that there is not stack_context
00334         # between the user's callback and the IOLoop, so monkey-patch
00335         # IOLoop.handle_callback_exception and disable the test harness's
00336         # context with a NullContext.
00337         # Note that this does not apply to secondary callbacks (header
00338         # and streaming_callback), as errors there must be seen as errors
00339         # by the http client so it can clean up the connection.
00340         exc_info = []
00341 
00342         def handle_callback_exception(callback):
00343             exc_info.append(sys.exc_info())
00344             self.stop()
00345         self.io_loop.handle_callback_exception = handle_callback_exception
00346         with NullContext():
00347             self.http_client.fetch(self.get_url('/hello'),
00348                                    lambda response: 1 / 0)
00349         self.wait()
00350         self.assertEqual(exc_info[0][0], ZeroDivisionError)
00351 
00352     @gen_test
00353     def test_future_interface(self):
00354         response = yield self.http_client.fetch(self.get_url('/hello'))
00355         self.assertEqual(response.body, b'Hello world!')
00356 
00357     @gen_test
00358     def test_future_http_error(self):
00359         with self.assertRaises(HTTPError) as context:
00360             yield self.http_client.fetch(self.get_url('/notfound'))
00361         self.assertEqual(context.exception.code, 404)
00362         self.assertEqual(context.exception.response.code, 404)
00363 
00364     @gen_test
00365     def test_reuse_request_from_response(self):
00366         # The response.request attribute should be an HTTPRequest, not
00367         # a _RequestProxy.
00368         # This test uses self.http_client.fetch because self.fetch calls
00369         # self.get_url on the input unconditionally.
00370         url = self.get_url('/hello')
00371         response = yield self.http_client.fetch(url)
00372         self.assertEqual(response.request.url, url)
00373         self.assertTrue(isinstance(response.request, HTTPRequest))
00374         response2 = yield self.http_client.fetch(response.request)
00375         self.assertEqual(response2.body, b'Hello world!')
00376 
00377     def test_all_methods(self):
00378         for method in ['GET', 'DELETE', 'OPTIONS']:
00379             response = self.fetch('/all_methods', method=method)
00380             self.assertEqual(response.body, utf8(method))
00381         for method in ['POST', 'PUT', 'PATCH']:
00382             response = self.fetch('/all_methods', method=method, body=b'')
00383             self.assertEqual(response.body, utf8(method))
00384         response = self.fetch('/all_methods', method='HEAD')
00385         self.assertEqual(response.body, b'')
00386         response = self.fetch('/all_methods', method='OTHER',
00387                               allow_nonstandard_methods=True)
00388         self.assertEqual(response.body, b'OTHER')
00389 
00390     @gen_test
00391     def test_body(self):
00392         hello_url = self.get_url('/hello')
00393         with self.assertRaises(AssertionError) as context:
00394             yield self.http_client.fetch(hello_url, body='data')
00395 
00396         self.assertTrue('must be empty' in str(context.exception))
00397 
00398         with self.assertRaises(AssertionError) as context:
00399             yield self.http_client.fetch(hello_url, method='POST')
00400 
00401         self.assertTrue('must not be empty' in str(context.exception))
00402 
00403 
00404 class RequestProxyTest(unittest.TestCase):
00405     def test_request_set(self):
00406         proxy = _RequestProxy(HTTPRequest('http://example.com/',
00407                                           user_agent='foo'),
00408                               dict())
00409         self.assertEqual(proxy.user_agent, 'foo')
00410 
00411     def test_default_set(self):
00412         proxy = _RequestProxy(HTTPRequest('http://example.com/'),
00413                               dict(network_interface='foo'))
00414         self.assertEqual(proxy.network_interface, 'foo')
00415 
00416     def test_both_set(self):
00417         proxy = _RequestProxy(HTTPRequest('http://example.com/',
00418                                           proxy_host='foo'),
00419                               dict(proxy_host='bar'))
00420         self.assertEqual(proxy.proxy_host, 'foo')
00421 
00422     def test_neither_set(self):
00423         proxy = _RequestProxy(HTTPRequest('http://example.com/'),
00424                               dict())
00425         self.assertIs(proxy.auth_username, None)
00426 
00427     def test_bad_attribute(self):
00428         proxy = _RequestProxy(HTTPRequest('http://example.com/'),
00429                               dict())
00430         with self.assertRaises(AttributeError):
00431             proxy.foo
00432 
00433     def test_defaults_none(self):
00434         proxy = _RequestProxy(HTTPRequest('http://example.com/'), None)
00435         self.assertIs(proxy.auth_username, None)
00436 
00437 
00438 class HTTPResponseTestCase(unittest.TestCase):
00439     def test_str(self):
00440         response = HTTPResponse(HTTPRequest('http://example.com'),
00441                                 200, headers={}, buffer=BytesIO())
00442         s = str(response)
00443         self.assertTrue(s.startswith('HTTPResponse('))
00444         self.assertIn('code=200', s)
00445 
00446 
00447 class SyncHTTPClientTest(unittest.TestCase):
00448     def setUp(self):
00449         if IOLoop.configured_class().__name__ in ('TwistedIOLoop',
00450                                                   'AsyncIOMainLoop'):
00451             # TwistedIOLoop only supports the global reactor, so we can't have
00452             # separate IOLoops for client and server threads.
00453             # AsyncIOMainLoop doesn't work with the default policy
00454             # (although it could with some tweaks to this test and a
00455             # policy that created loops for non-main threads).
00456             raise unittest.SkipTest(
00457                 'Sync HTTPClient not compatible with TwistedIOLoop or '
00458                 'AsyncIOMainLoop')
00459         self.server_ioloop = IOLoop()
00460 
00461         sock, self.port = bind_unused_port()
00462         app = Application([('/', HelloWorldHandler)])
00463         self.server = HTTPServer(app, io_loop=self.server_ioloop)
00464         self.server.add_socket(sock)
00465 
00466         self.server_thread = threading.Thread(target=self.server_ioloop.start)
00467         self.server_thread.start()
00468 
00469         self.http_client = HTTPClient()
00470 
00471     def tearDown(self):
00472         def stop_server():
00473             self.server.stop()
00474             self.server_ioloop.stop()
00475         self.server_ioloop.add_callback(stop_server)
00476         self.server_thread.join()
00477         self.http_client.close()
00478         self.server_ioloop.close(all_fds=True)
00479 
00480     def get_url(self, path):
00481         return 'http://localhost:%d%s' % (self.port, path)
00482 
00483     def test_sync_client(self):
00484         response = self.http_client.fetch(self.get_url('/'))
00485         self.assertEqual(b'Hello world!', response.body)
00486 
00487     def test_sync_client_error(self):
00488         # Synchronous HTTPClient raises errors directly; no need for
00489         # response.rethrow()
00490         with self.assertRaises(HTTPError) as assertion:
00491             self.http_client.fetch(self.get_url('/notfound'))
00492         self.assertEqual(assertion.exception.code, 404)
00493 
00494 
00495 class HTTPRequestTestCase(unittest.TestCase):
00496     def test_headers(self):
00497         request = HTTPRequest('http://example.com', headers={'foo': 'bar'})
00498         self.assertEqual(request.headers, {'foo': 'bar'})
00499 
00500     def test_headers_setter(self):
00501         request = HTTPRequest('http://example.com')
00502         request.headers = {'bar': 'baz'}
00503         self.assertEqual(request.headers, {'bar': 'baz'})
00504 
00505     def test_null_headers_setter(self):
00506         request = HTTPRequest('http://example.com')
00507         request.headers = None
00508         self.assertEqual(request.headers, {})
00509 
00510     def test_body(self):
00511         request = HTTPRequest('http://example.com', body='foo')
00512         self.assertEqual(request.body, utf8('foo'))
00513 
00514     def test_body_setter(self):
00515         request = HTTPRequest('http://example.com')
00516         request.body = 'foo'
00517         self.assertEqual(request.body, utf8('foo'))


rosbridge_tools
Author(s): Jonathan Mace
autogenerated on Sat Dec 27 2014 11:25:59