simple_httpclient_test.py
Go to the documentation of this file.
00001 from __future__ import absolute_import, division, print_function, with_statement
00002 
00003 import collections
00004 from contextlib import closing
00005 import errno
00006 import gzip
00007 import logging
00008 import os
00009 import re
00010 import socket
00011 import sys
00012 
00013 from tornado import gen
00014 from tornado.httpclient import AsyncHTTPClient
00015 from tornado.httputil import HTTPHeaders
00016 from tornado.ioloop import IOLoop
00017 from tornado.log import gen_log, app_log
00018 from tornado.netutil import Resolver, bind_sockets
00019 from tornado.simple_httpclient import SimpleAsyncHTTPClient, _default_ca_certs
00020 from tornado.test.httpclient_test import ChunkHandler, CountdownHandler, HelloWorldHandler
00021 from tornado.test import httpclient_test
00022 from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, bind_unused_port, ExpectLog
00023 from tornado.test.util import skipOnTravis, skipIfNoIPv6
00024 from tornado.web import RequestHandler, Application, asynchronous, url, stream_request_body
00025 
00026 
00027 class SimpleHTTPClientCommonTestCase(httpclient_test.HTTPClientCommonTestCase):
00028     def get_http_client(self):
00029         client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
00030                                        force_instance=True)
00031         self.assertTrue(isinstance(client, SimpleAsyncHTTPClient))
00032         return client
00033 
00034 
00035 class TriggerHandler(RequestHandler):
00036     def initialize(self, queue, wake_callback):
00037         self.queue = queue
00038         self.wake_callback = wake_callback
00039 
00040     @asynchronous
00041     def get(self):
00042         logging.debug("queuing trigger")
00043         self.queue.append(self.finish)
00044         if self.get_argument("wake", "true") == "true":
00045             self.wake_callback()
00046 
00047 
00048 class HangHandler(RequestHandler):
00049     @asynchronous
00050     def get(self):
00051         pass
00052 
00053 
00054 class ContentLengthHandler(RequestHandler):
00055     def get(self):
00056         self.set_header("Content-Length", self.get_argument("value"))
00057         self.write("ok")
00058 
00059 
00060 class HeadHandler(RequestHandler):
00061     def head(self):
00062         self.set_header("Content-Length", "7")
00063 
00064 
00065 class OptionsHandler(RequestHandler):
00066     def options(self):
00067         self.set_header("Access-Control-Allow-Origin", "*")
00068         self.write("ok")
00069 
00070 
00071 class NoContentHandler(RequestHandler):
00072     def get(self):
00073         if self.get_argument("error", None):
00074             self.set_header("Content-Length", "5")
00075             self.write("hello")
00076         self.set_status(204)
00077 
00078 
00079 class SeeOtherPostHandler(RequestHandler):
00080     def post(self):
00081         redirect_code = int(self.request.body)
00082         assert redirect_code in (302, 303), "unexpected body %r" % self.request.body
00083         self.set_header("Location", "/see_other_get")
00084         self.set_status(redirect_code)
00085 
00086 
00087 class SeeOtherGetHandler(RequestHandler):
00088     def get(self):
00089         if self.request.body:
00090             raise Exception("unexpected body %r" % self.request.body)
00091         self.write("ok")
00092 
00093 
00094 class HostEchoHandler(RequestHandler):
00095     def get(self):
00096         self.write(self.request.headers["Host"])
00097 
00098 
00099 class NoContentLengthHandler(RequestHandler):
00100     @gen.coroutine
00101     def get(self):
00102         # Emulate the old HTTP/1.0 behavior of returning a body with no
00103         # content-length.  Tornado handles content-length at the framework
00104         # level so we have to go around it.
00105         stream = self.request.connection.stream
00106         yield stream.write(b"HTTP/1.0 200 OK\r\n\r\n"
00107                            b"hello")
00108         stream.close()
00109 
00110 
00111 class EchoPostHandler(RequestHandler):
00112     def post(self):
00113         self.write(self.request.body)
00114 
00115 
00116 @stream_request_body
00117 class RespondInPrepareHandler(RequestHandler):
00118     def prepare(self):
00119         self.set_status(403)
00120         self.finish("forbidden")
00121 
00122 
00123 class SimpleHTTPClientTestMixin(object):
00124     def get_app(self):
00125         # callable objects to finish pending /trigger requests
00126         self.triggers = collections.deque()
00127         return Application([
00128             url("/trigger", TriggerHandler, dict(queue=self.triggers,
00129                                                  wake_callback=self.stop)),
00130             url("/chunk", ChunkHandler),
00131             url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
00132             url("/hang", HangHandler),
00133             url("/hello", HelloWorldHandler),
00134             url("/content_length", ContentLengthHandler),
00135             url("/head", HeadHandler),
00136             url("/options", OptionsHandler),
00137             url("/no_content", NoContentHandler),
00138             url("/see_other_post", SeeOtherPostHandler),
00139             url("/see_other_get", SeeOtherGetHandler),
00140             url("/host_echo", HostEchoHandler),
00141             url("/no_content_length", NoContentLengthHandler),
00142             url("/echo_post", EchoPostHandler),
00143             url("/respond_in_prepare", RespondInPrepareHandler),
00144         ], gzip=True)
00145 
00146     def test_singleton(self):
00147         # Class "constructor" reuses objects on the same IOLoop
00148         self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is
00149                         SimpleAsyncHTTPClient(self.io_loop))
00150         # unless force_instance is used
00151         self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
00152                         SimpleAsyncHTTPClient(self.io_loop,
00153                                               force_instance=True))
00154         # different IOLoops use different objects
00155         with closing(IOLoop()) as io_loop2:
00156             self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
00157                             SimpleAsyncHTTPClient(io_loop2))
00158 
00159     def test_connection_limit(self):
00160         with closing(self.create_client(max_clients=2)) as client:
00161             self.assertEqual(client.max_clients, 2)
00162             seen = []
00163             # Send 4 requests.  Two can be sent immediately, while the others
00164             # will be queued
00165             for i in range(4):
00166                 client.fetch(self.get_url("/trigger"),
00167                              lambda response, i=i: (seen.append(i), self.stop()))
00168             self.wait(condition=lambda: len(self.triggers) == 2)
00169             self.assertEqual(len(client.queue), 2)
00170 
00171             # Finish the first two requests and let the next two through
00172             self.triggers.popleft()()
00173             self.triggers.popleft()()
00174             self.wait(condition=lambda: (len(self.triggers) == 2 and
00175                                          len(seen) == 2))
00176             self.assertEqual(set(seen), set([0, 1]))
00177             self.assertEqual(len(client.queue), 0)
00178 
00179             # Finish all the pending requests
00180             self.triggers.popleft()()
00181             self.triggers.popleft()()
00182             self.wait(condition=lambda: len(seen) == 4)
00183             self.assertEqual(set(seen), set([0, 1, 2, 3]))
00184             self.assertEqual(len(self.triggers), 0)
00185 
00186     def test_redirect_connection_limit(self):
00187         # following redirects should not consume additional connections
00188         with closing(self.create_client(max_clients=1)) as client:
00189             client.fetch(self.get_url('/countdown/3'), self.stop,
00190                          max_redirects=3)
00191             response = self.wait()
00192             response.rethrow()
00193 
00194     def test_default_certificates_exist(self):
00195         open(_default_ca_certs()).close()
00196 
00197     def test_gzip(self):
00198         # All the tests in this file should be using gzip, but this test
00199         # ensures that it is in fact getting compressed.
00200         # Setting Accept-Encoding manually bypasses the client's
00201         # decompression so we can see the raw data.
00202         response = self.fetch("/chunk", use_gzip=False,
00203                               headers={"Accept-Encoding": "gzip"})
00204         self.assertEqual(response.headers["Content-Encoding"], "gzip")
00205         self.assertNotEqual(response.body, b"asdfqwer")
00206         # Our test data gets bigger when gzipped.  Oops.  :)
00207         self.assertEqual(len(response.body), 34)
00208         f = gzip.GzipFile(mode="r", fileobj=response.buffer)
00209         self.assertEqual(f.read(), b"asdfqwer")
00210 
00211     def test_max_redirects(self):
00212         response = self.fetch("/countdown/5", max_redirects=3)
00213         self.assertEqual(302, response.code)
00214         # We requested 5, followed three redirects for 4, 3, 2, then the last
00215         # unfollowed redirect is to 1.
00216         self.assertTrue(response.request.url.endswith("/countdown/5"))
00217         self.assertTrue(response.effective_url.endswith("/countdown/2"))
00218         self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
00219 
00220     def test_header_reuse(self):
00221         # Apps may reuse a headers object if they are only passing in constant
00222         # headers like user-agent.  The header object should not be modified.
00223         headers = HTTPHeaders({'User-Agent': 'Foo'})
00224         self.fetch("/hello", headers=headers)
00225         self.assertEqual(list(headers.get_all()), [('User-Agent', 'Foo')])
00226 
00227     def test_see_other_redirect(self):
00228         for code in (302, 303):
00229             response = self.fetch("/see_other_post", method="POST", body="%d" % code)
00230             self.assertEqual(200, response.code)
00231             self.assertTrue(response.request.url.endswith("/see_other_post"))
00232             self.assertTrue(response.effective_url.endswith("/see_other_get"))
00233             # request is the original request, is a POST still
00234             self.assertEqual("POST", response.request.method)
00235 
00236     @skipOnTravis
00237     def test_request_timeout(self):
00238         response = self.fetch('/trigger?wake=false', request_timeout=0.1)
00239         self.assertEqual(response.code, 599)
00240         self.assertTrue(0.099 < response.request_time < 0.15, response.request_time)
00241         self.assertEqual(str(response.error), "HTTP 599: Timeout")
00242         # trigger the hanging request to let it clean up after itself
00243         self.triggers.popleft()()
00244 
00245     @skipIfNoIPv6
00246     def test_ipv6(self):
00247         try:
00248             [sock] = bind_sockets(None, '::1', family=socket.AF_INET6)
00249             port = sock.getsockname()[1]
00250             self.http_server.add_socket(sock)
00251         except socket.gaierror as e:
00252             if e.args[0] == socket.EAI_ADDRFAMILY:
00253                 # python supports ipv6, but it's not configured on the network
00254                 # interface, so skip this test.
00255                 return
00256             raise
00257         url = '%s://[::1]:%d/hello' % (self.get_protocol(), port)
00258 
00259         # ipv6 is currently enabled by default but can be disabled
00260         self.http_client.fetch(url, self.stop, allow_ipv6=False)
00261         response = self.wait()
00262         self.assertEqual(response.code, 599)
00263 
00264         self.http_client.fetch(url, self.stop)
00265         response = self.wait()
00266         self.assertEqual(response.body, b"Hello world!")
00267 
00268     def xtest_multiple_content_length_accepted(self):
00269         response = self.fetch("/content_length?value=2,2")
00270         self.assertEqual(response.body, b"ok")
00271         response = self.fetch("/content_length?value=2,%202,2")
00272         self.assertEqual(response.body, b"ok")
00273 
00274         response = self.fetch("/content_length?value=2,4")
00275         self.assertEqual(response.code, 599)
00276         response = self.fetch("/content_length?value=2,%202,3")
00277         self.assertEqual(response.code, 599)
00278 
00279     def test_head_request(self):
00280         response = self.fetch("/head", method="HEAD")
00281         self.assertEqual(response.code, 200)
00282         self.assertEqual(response.headers["content-length"], "7")
00283         self.assertFalse(response.body)
00284 
00285     def test_options_request(self):
00286         response = self.fetch("/options", method="OPTIONS")
00287         self.assertEqual(response.code, 200)
00288         self.assertEqual(response.headers["content-length"], "2")
00289         self.assertEqual(response.headers["access-control-allow-origin"], "*")
00290         self.assertEqual(response.body, b"ok")
00291 
00292     def test_no_content(self):
00293         response = self.fetch("/no_content")
00294         self.assertEqual(response.code, 204)
00295         # 204 status doesn't need a content-length, but tornado will
00296         # add a zero content-length anyway.
00297         #
00298         # A test without a content-length header is included below
00299         # in HTTP204NoContentTestCase.
00300         self.assertEqual(response.headers["Content-length"], "0")
00301 
00302         # 204 status with non-zero content length is malformed
00303         with ExpectLog(gen_log, "Malformed HTTP message"):
00304             response = self.fetch("/no_content?error=1")
00305         self.assertEqual(response.code, 599)
00306 
00307     def test_host_header(self):
00308         host_re = re.compile(b"^localhost:[0-9]+$")
00309         response = self.fetch("/host_echo")
00310         self.assertTrue(host_re.match(response.body))
00311 
00312         url = self.get_url("/host_echo").replace("http://", "http://me:secret@")
00313         self.http_client.fetch(url, self.stop)
00314         response = self.wait()
00315         self.assertTrue(host_re.match(response.body), response.body)
00316 
00317     def test_connection_refused(self):
00318         server_socket, port = bind_unused_port()
00319         server_socket.close()
00320         with ExpectLog(gen_log, ".*", required=False):
00321             self.http_client.fetch("http://localhost:%d/" % port, self.stop)
00322             response = self.wait()
00323         self.assertEqual(599, response.code)
00324 
00325         if sys.platform != 'cygwin':
00326             # cygwin returns EPERM instead of ECONNREFUSED here
00327             contains_errno = str(errno.ECONNREFUSED) in str(response.error)
00328             if not contains_errno and hasattr(errno, "WSAECONNREFUSED"):
00329                 contains_errno = str(errno.WSAECONNREFUSED) in str(response.error)
00330             self.assertTrue(contains_errno, response.error)
00331             # This is usually "Connection refused".
00332             # On windows, strerror is broken and returns "Unknown error".
00333             expected_message = os.strerror(errno.ECONNREFUSED)
00334             self.assertTrue(expected_message in str(response.error),
00335                             response.error)
00336 
00337     def test_queue_timeout(self):
00338         with closing(self.create_client(max_clients=1)) as client:
00339             client.fetch(self.get_url('/trigger'), self.stop,
00340                          request_timeout=10)
00341             # Wait for the trigger request to block, not complete.
00342             self.wait()
00343             client.fetch(self.get_url('/hello'), self.stop,
00344                          connect_timeout=0.1)
00345             response = self.wait()
00346 
00347             self.assertEqual(response.code, 599)
00348             self.assertTrue(response.request_time < 1, response.request_time)
00349             self.assertEqual(str(response.error), "HTTP 599: Timeout")
00350             self.triggers.popleft()()
00351             self.wait()
00352 
00353     def test_no_content_length(self):
00354         response = self.fetch("/no_content_length")
00355         self.assertEquals(b"hello", response.body)
00356 
00357     def sync_body_producer(self, write):
00358         write(b'1234')
00359         write(b'5678')
00360 
00361     @gen.coroutine
00362     def async_body_producer(self, write):
00363         yield write(b'1234')
00364         yield gen.Task(IOLoop.current().add_callback)
00365         yield write(b'5678')
00366 
00367     def test_sync_body_producer_chunked(self):
00368         response = self.fetch("/echo_post", method="POST",
00369                               body_producer=self.sync_body_producer)
00370         response.rethrow()
00371         self.assertEqual(response.body, b"12345678")
00372 
00373     def test_sync_body_producer_content_length(self):
00374         response = self.fetch("/echo_post", method="POST",
00375                               body_producer=self.sync_body_producer,
00376                               headers={'Content-Length': '8'})
00377         response.rethrow()
00378         self.assertEqual(response.body, b"12345678")
00379 
00380     def test_async_body_producer_chunked(self):
00381         response = self.fetch("/echo_post", method="POST",
00382                               body_producer=self.async_body_producer)
00383         response.rethrow()
00384         self.assertEqual(response.body, b"12345678")
00385 
00386     def test_async_body_producer_content_length(self):
00387         response = self.fetch("/echo_post", method="POST",
00388                               body_producer=self.async_body_producer,
00389                               headers={'Content-Length': '8'})
00390         response.rethrow()
00391         self.assertEqual(response.body, b"12345678")
00392 
00393     def test_100_continue(self):
00394         response = self.fetch("/echo_post", method="POST",
00395                               body=b"1234",
00396                               expect_100_continue=True)
00397         self.assertEqual(response.body, b"1234")
00398 
00399     def test_100_continue_early_response(self):
00400         def body_producer(write):
00401             raise Exception("should not be called")
00402         response = self.fetch("/respond_in_prepare", method="POST",
00403                               body_producer=body_producer,
00404                               expect_100_continue=True)
00405         self.assertEqual(response.code, 403)
00406 
00407 
00408 class SimpleHTTPClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPTestCase):
00409     def setUp(self):
00410         super(SimpleHTTPClientTestCase, self).setUp()
00411         self.http_client = self.create_client()
00412 
00413     def create_client(self, **kwargs):
00414         return SimpleAsyncHTTPClient(self.io_loop, force_instance=True,
00415                                      **kwargs)
00416 
00417 
00418 class SimpleHTTPSClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPSTestCase):
00419     def setUp(self):
00420         super(SimpleHTTPSClientTestCase, self).setUp()
00421         self.http_client = self.create_client()
00422 
00423     def create_client(self, **kwargs):
00424         return SimpleAsyncHTTPClient(self.io_loop, force_instance=True,
00425                                      defaults=dict(validate_cert=False),
00426                                      **kwargs)
00427 
00428 
00429 class CreateAsyncHTTPClientTestCase(AsyncTestCase):
00430     def setUp(self):
00431         super(CreateAsyncHTTPClientTestCase, self).setUp()
00432         self.saved = AsyncHTTPClient._save_configuration()
00433 
00434     def tearDown(self):
00435         AsyncHTTPClient._restore_configuration(self.saved)
00436         super(CreateAsyncHTTPClientTestCase, self).tearDown()
00437 
00438     def test_max_clients(self):
00439         AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
00440         with closing(AsyncHTTPClient(
00441                 self.io_loop, force_instance=True)) as client:
00442             self.assertEqual(client.max_clients, 10)
00443         with closing(AsyncHTTPClient(
00444                 self.io_loop, max_clients=11, force_instance=True)) as client:
00445             self.assertEqual(client.max_clients, 11)
00446 
00447         # Now configure max_clients statically and try overriding it
00448         # with each way max_clients can be passed
00449         AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12)
00450         with closing(AsyncHTTPClient(
00451                 self.io_loop, force_instance=True)) as client:
00452             self.assertEqual(client.max_clients, 12)
00453         with closing(AsyncHTTPClient(
00454                 self.io_loop, max_clients=13, force_instance=True)) as client:
00455             self.assertEqual(client.max_clients, 13)
00456         with closing(AsyncHTTPClient(
00457                 self.io_loop, max_clients=14, force_instance=True)) as client:
00458             self.assertEqual(client.max_clients, 14)
00459 
00460 
00461 class HTTP100ContinueTestCase(AsyncHTTPTestCase):
00462     def respond_100(self, request):
00463         self.request = request
00464         self.request.connection.stream.write(
00465             b"HTTP/1.1 100 CONTINUE\r\n\r\n",
00466             self.respond_200)
00467 
00468     def respond_200(self):
00469         self.request.connection.stream.write(
00470             b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\nA",
00471             self.request.connection.stream.close)
00472 
00473     def get_app(self):
00474         # Not a full Application, but works as an HTTPServer callback
00475         return self.respond_100
00476 
00477     def test_100_continue(self):
00478         res = self.fetch('/')
00479         self.assertEqual(res.body, b'A')
00480 
00481 
00482 class HTTP204NoContentTestCase(AsyncHTTPTestCase):
00483     def respond_204(self, request):
00484         # A 204 response never has a body, even if doesn't have a content-length
00485         # (which would otherwise mean read-until-close).  Tornado always
00486         # sends a content-length, so we simulate here a server that sends
00487         # no content length and does not close the connection.
00488         #
00489         # Tests of a 204 response with a Content-Length header are included
00490         # in SimpleHTTPClientTestMixin.
00491         request.connection.stream.write(
00492             b"HTTP/1.1 204 No content\r\n\r\n")
00493 
00494     def get_app(self):
00495         return self.respond_204
00496 
00497     def test_204_no_content(self):
00498         resp = self.fetch('/')
00499         self.assertEqual(resp.code, 204)
00500         self.assertEqual(resp.body, b'')
00501 
00502 
00503 class HostnameMappingTestCase(AsyncHTTPTestCase):
00504     def setUp(self):
00505         super(HostnameMappingTestCase, self).setUp()
00506         self.http_client = SimpleAsyncHTTPClient(
00507             self.io_loop,
00508             hostname_mapping={
00509                 'www.example.com': '127.0.0.1',
00510                 ('foo.example.com', 8000): ('127.0.0.1', self.get_http_port()),
00511             })
00512 
00513     def get_app(self):
00514         return Application([url("/hello", HelloWorldHandler), ])
00515 
00516     def test_hostname_mapping(self):
00517         self.http_client.fetch(
00518             'http://www.example.com:%d/hello' % self.get_http_port(), self.stop)
00519         response = self.wait()
00520         response.rethrow()
00521         self.assertEqual(response.body, b'Hello world!')
00522 
00523     def test_port_mapping(self):
00524         self.http_client.fetch('http://foo.example.com:8000/hello', self.stop)
00525         response = self.wait()
00526         response.rethrow()
00527         self.assertEqual(response.body, b'Hello world!')
00528 
00529 
00530 class ResolveTimeoutTestCase(AsyncHTTPTestCase):
00531     def setUp(self):
00532         # Dummy Resolver subclass that never invokes its callback.
00533         class BadResolver(Resolver):
00534             def resolve(self, *args, **kwargs):
00535                 pass
00536 
00537         super(ResolveTimeoutTestCase, self).setUp()
00538         self.http_client = SimpleAsyncHTTPClient(
00539             self.io_loop,
00540             resolver=BadResolver())
00541 
00542     def get_app(self):
00543         return Application([url("/hello", HelloWorldHandler), ])
00544 
00545     def test_resolve_timeout(self):
00546         response = self.fetch('/hello', connect_timeout=0.1)
00547         self.assertEqual(response.code, 599)
00548 
00549 
00550 class MaxHeaderSizeTest(AsyncHTTPTestCase):
00551     def get_app(self):
00552         class SmallHeaders(RequestHandler):
00553             def get(self):
00554                 self.set_header("X-Filler", "a" * 100)
00555                 self.write("ok")
00556 
00557         class LargeHeaders(RequestHandler):
00558             def get(self):
00559                 self.set_header("X-Filler", "a" * 1000)
00560                 self.write("ok")
00561 
00562         return Application([('/small', SmallHeaders),
00563                             ('/large', LargeHeaders)])
00564 
00565     def get_http_client(self):
00566         return SimpleAsyncHTTPClient(io_loop=self.io_loop, max_header_size=1024)
00567 
00568     def test_small_headers(self):
00569         response = self.fetch('/small')
00570         response.rethrow()
00571         self.assertEqual(response.body, b'ok')
00572 
00573     def test_large_headers(self):
00574         with ExpectLog(gen_log, "Unsatisfiable read"):
00575             response = self.fetch('/large')
00576         self.assertEqual(response.code, 599)


rosbridge_server
Author(s): Jonathan Mace
autogenerated on Wed Sep 13 2017 03:18:20