netutil.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 #
00003 # Copyright 2011 Facebook
00004 #
00005 # Licensed under the Apache License, Version 2.0 (the "License"); you may
00006 # not use this file except in compliance with the License. You may obtain
00007 # a copy of the License at
00008 #
00009 #     http://www.apache.org/licenses/LICENSE-2.0
00010 #
00011 # Unless required by applicable law or agreed to in writing, software
00012 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
00013 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
00014 # License for the specific language governing permissions and limitations
00015 # under the License.
00016 
00017 """Miscellaneous network utility code."""
00018 
00019 from __future__ import absolute_import, division, print_function, with_statement
00020 
00021 import errno
00022 import os
00023 import platform
00024 import socket
00025 import stat
00026 
00027 from tornado.concurrent import dummy_executor, run_on_executor
00028 from tornado.ioloop import IOLoop
00029 from tornado.platform.auto import set_close_exec
00030 from tornado.util import u, Configurable, errno_from_exception
00031 
00032 try:
00033     import ssl
00034 except ImportError:
00035     # ssl is not available on Google App Engine
00036     ssl = None
00037 
00038 try:
00039     xrange  # py2
00040 except NameError:
00041     xrange = range  # py3
00042 
00043 if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'):  # python 3.2+
00044     ssl_match_hostname = ssl.match_hostname
00045     SSLCertificateError = ssl.CertificateError
00046 elif ssl is None:
00047     ssl_match_hostname = SSLCertificateError = None
00048 else:
00049     import backports.ssl_match_hostname
00050     ssl_match_hostname = backports.ssl_match_hostname.match_hostname
00051     SSLCertificateError = backports.ssl_match_hostname.CertificateError
00052 
00053 # ThreadedResolver runs getaddrinfo on a thread. If the hostname is unicode,
00054 # getaddrinfo attempts to import encodings.idna. If this is done at
00055 # module-import time, the import lock is already held by the main thread,
00056 # leading to deadlock. Avoid it by caching the idna encoder on the main
00057 # thread now.
00058 u('foo').encode('idna')
00059 
00060 # These errnos indicate that a non-blocking operation must be retried
00061 # at a later time.  On most platforms they're the same value, but on
00062 # some they differ.
00063 _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
00064 
00065 if hasattr(errno, "WSAEWOULDBLOCK"):
00066     _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,)
00067 
00068 # Default backlog used when calling sock.listen()
00069 _DEFAULT_BACKLOG = 128
00070 
00071 def bind_sockets(port, address=None, family=socket.AF_UNSPEC,
00072                  backlog=_DEFAULT_BACKLOG, flags=None):
00073     """Creates listening sockets bound to the given port and address.
00074 
00075     Returns a list of socket objects (multiple sockets are returned if
00076     the given address maps to multiple IP addresses, which is most common
00077     for mixed IPv4 and IPv6 use).
00078 
00079     Address may be either an IP address or hostname.  If it's a hostname,
00080     the server will listen on all IP addresses associated with the
00081     name.  Address may be an empty string or None to listen on all
00082     available interfaces.  Family may be set to either `socket.AF_INET`
00083     or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise
00084     both will be used if available.
00085 
00086     The ``backlog`` argument has the same meaning as for
00087     `socket.listen() <socket.socket.listen>`.
00088 
00089     ``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like
00090     ``socket.AI_PASSIVE | socket.AI_NUMERICHOST``.
00091     """
00092     sockets = []
00093     if address == "":
00094         address = None
00095     if not socket.has_ipv6 and family == socket.AF_UNSPEC:
00096         # Python can be compiled with --disable-ipv6, which causes
00097         # operations on AF_INET6 sockets to fail, but does not
00098         # automatically exclude those results from getaddrinfo
00099         # results.
00100         # http://bugs.python.org/issue16208
00101         family = socket.AF_INET
00102     if flags is None:
00103         flags = socket.AI_PASSIVE
00104     bound_port = None
00105     for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
00106                                       0, flags)):
00107         af, socktype, proto, canonname, sockaddr = res
00108         if (platform.system() == 'Darwin' and address == 'localhost' and
00109                 af == socket.AF_INET6 and sockaddr[3] != 0):
00110             # Mac OS X includes a link-local address fe80::1%lo0 in the
00111             # getaddrinfo results for 'localhost'.  However, the firewall
00112             # doesn't understand that this is a local address and will
00113             # prompt for access (often repeatedly, due to an apparent
00114             # bug in its ability to remember granting access to an
00115             # application). Skip these addresses.
00116             continue
00117         try:
00118             sock = socket.socket(af, socktype, proto)
00119         except socket.error as e:
00120             if errno_from_exception(e) == errno.EAFNOSUPPORT:
00121                 continue
00122             raise
00123         set_close_exec(sock.fileno())
00124         if os.name != 'nt':
00125             sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
00126         if af == socket.AF_INET6:
00127             # On linux, ipv6 sockets accept ipv4 too by default,
00128             # but this makes it impossible to bind to both
00129             # 0.0.0.0 in ipv4 and :: in ipv6.  On other systems,
00130             # separate sockets *must* be used to listen for both ipv4
00131             # and ipv6.  For consistency, always disable ipv4 on our
00132             # ipv6 sockets and use a separate ipv4 socket when needed.
00133             #
00134             # Python 2.x on windows doesn't have IPPROTO_IPV6.
00135             if hasattr(socket, "IPPROTO_IPV6"):
00136                 sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
00137 
00138         # automatic port allocation with port=None
00139         # should bind on the same port on IPv4 and IPv6
00140         host, requested_port = sockaddr[:2]
00141         if requested_port == 0 and bound_port is not None:
00142             sockaddr = tuple([host, bound_port] + list(sockaddr[2:]))
00143 
00144         sock.setblocking(0)
00145         sock.bind(sockaddr)
00146         bound_port = sock.getsockname()[1]
00147         sock.listen(backlog)
00148         sockets.append(sock)
00149     return sockets
00150 
00151 if hasattr(socket, 'AF_UNIX'):
00152     def bind_unix_socket(file, mode=0o600, backlog=_DEFAULT_BACKLOG):
00153         """Creates a listening unix socket.
00154 
00155         If a socket with the given name already exists, it will be deleted.
00156         If any other file with that name exists, an exception will be
00157         raised.
00158 
00159         Returns a socket object (not a list of socket objects like
00160         `bind_sockets`)
00161         """
00162         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
00163         set_close_exec(sock.fileno())
00164         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
00165         sock.setblocking(0)
00166         try:
00167             st = os.stat(file)
00168         except OSError as err:
00169             if errno_from_exception(err) != errno.ENOENT:
00170                 raise
00171         else:
00172             if stat.S_ISSOCK(st.st_mode):
00173                 os.remove(file)
00174             else:
00175                 raise ValueError("File %s exists and is not a socket", file)
00176         sock.bind(file)
00177         os.chmod(file, mode)
00178         sock.listen(backlog)
00179         return sock
00180 
00181 
00182 def add_accept_handler(sock, callback, io_loop=None):
00183     """Adds an `.IOLoop` event handler to accept new connections on ``sock``.
00184 
00185     When a connection is accepted, ``callback(connection, address)`` will
00186     be run (``connection`` is a socket object, and ``address`` is the
00187     address of the other end of the connection).  Note that this signature
00188     is different from the ``callback(fd, events)`` signature used for
00189     `.IOLoop` handlers.
00190     """
00191     if io_loop is None:
00192         io_loop = IOLoop.current()
00193 
00194     def accept_handler(fd, events):
00195         # More connections may come in while we're handling callbacks;
00196         # to prevent starvation of other tasks we must limit the number
00197         # of connections we accept at a time.  Ideally we would accept
00198         # up to the number of connections that were waiting when we
00199         # entered this method, but this information is not available
00200         # (and rearranging this method to call accept() as many times
00201         # as possible before running any callbacks would have adverse
00202         # effects on load balancing in multiprocess configurations).
00203         # Instead, we use the (default) listen backlog as a rough
00204         # heuristic for the number of connections we can reasonably
00205         # accept at once.
00206         for i in xrange(_DEFAULT_BACKLOG):
00207             try:
00208                 connection, address = sock.accept()
00209             except socket.error as e:
00210                 # _ERRNO_WOULDBLOCK indicate we have accepted every
00211                 # connection that is available.
00212                 if errno_from_exception(e) in _ERRNO_WOULDBLOCK:
00213                     return
00214                 # ECONNABORTED indicates that there was a connection
00215                 # but it was closed while still in the accept queue.
00216                 # (observed on FreeBSD).
00217                 if errno_from_exception(e) == errno.ECONNABORTED:
00218                     continue
00219                 raise
00220             callback(connection, address)
00221     io_loop.add_handler(sock, accept_handler, IOLoop.READ)
00222 
00223 
00224 def is_valid_ip(ip):
00225     """Returns true if the given string is a well-formed IP address.
00226 
00227     Supports IPv4 and IPv6.
00228     """
00229     if not ip or '\x00' in ip:
00230         # getaddrinfo resolves empty strings to localhost, and truncates
00231         # on zero bytes.
00232         return False
00233     try:
00234         res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
00235                                  socket.SOCK_STREAM,
00236                                  0, socket.AI_NUMERICHOST)
00237         return bool(res)
00238     except socket.gaierror as e:
00239         if e.args[0] == socket.EAI_NONAME:
00240             return False
00241         raise
00242     return True
00243 
00244 
00245 class Resolver(Configurable):
00246     """Configurable asynchronous DNS resolver interface.
00247 
00248     By default, a blocking implementation is used (which simply calls
00249     `socket.getaddrinfo`).  An alternative implementation can be
00250     chosen with the `Resolver.configure <.Configurable.configure>`
00251     class method::
00252 
00253         Resolver.configure('tornado.netutil.ThreadedResolver')
00254 
00255     The implementations of this interface included with Tornado are
00256 
00257     * `tornado.netutil.BlockingResolver`
00258     * `tornado.netutil.ThreadedResolver`
00259     * `tornado.netutil.OverrideResolver`
00260     * `tornado.platform.twisted.TwistedResolver`
00261     * `tornado.platform.caresresolver.CaresResolver`
00262     """
00263     @classmethod
00264     def configurable_base(cls):
00265         return Resolver
00266 
00267     @classmethod
00268     def configurable_default(cls):
00269         return BlockingResolver
00270 
00271     def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None):
00272         """Resolves an address.
00273 
00274         The ``host`` argument is a string which may be a hostname or a
00275         literal IP address.
00276 
00277         Returns a `.Future` whose result is a list of (family,
00278         address) pairs, where address is a tuple suitable to pass to
00279         `socket.connect <socket.socket.connect>` (i.e. a ``(host,
00280         port)`` pair for IPv4; additional fields may be present for
00281         IPv6). If a ``callback`` is passed, it will be run with the
00282         result as an argument when it is complete.
00283         """
00284         raise NotImplementedError()
00285 
00286     def close(self):
00287         """Closes the `Resolver`, freeing any resources used.
00288 
00289         .. versionadded:: 3.1
00290 
00291         """
00292         pass
00293 
00294 
00295 class ExecutorResolver(Resolver):
00296     """Resolver implementation using a `concurrent.futures.Executor`.
00297 
00298     Use this instead of `ThreadedResolver` when you require additional
00299     control over the executor being used.
00300 
00301     The executor will be shut down when the resolver is closed unless
00302     ``close_resolver=False``; use this if you want to reuse the same
00303     executor elsewhere.
00304     """
00305     def initialize(self, io_loop=None, executor=None, close_executor=True):
00306         self.io_loop = io_loop or IOLoop.current()
00307         if executor is not None:
00308             self.executor = executor
00309             self.close_executor = close_executor
00310         else:
00311             self.executor = dummy_executor
00312             self.close_executor = False
00313 
00314     def close(self):
00315         if self.close_executor:
00316             self.executor.shutdown()
00317         self.executor = None
00318 
00319     @run_on_executor
00320     def resolve(self, host, port, family=socket.AF_UNSPEC):
00321         # On Solaris, getaddrinfo fails if the given port is not found
00322         # in /etc/services and no socket type is given, so we must pass
00323         # one here.  The socket type used here doesn't seem to actually
00324         # matter (we discard the one we get back in the results),
00325         # so the addresses we return should still be usable with SOCK_DGRAM.
00326         addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM)
00327         results = []
00328         for family, socktype, proto, canonname, address in addrinfo:
00329             results.append((family, address))
00330         return results
00331 
00332 
00333 class BlockingResolver(ExecutorResolver):
00334     """Default `Resolver` implementation, using `socket.getaddrinfo`.
00335 
00336     The `.IOLoop` will be blocked during the resolution, although the
00337     callback will not be run until the next `.IOLoop` iteration.
00338     """
00339     def initialize(self, io_loop=None):
00340         super(BlockingResolver, self).initialize(io_loop=io_loop)
00341 
00342 
00343 class ThreadedResolver(ExecutorResolver):
00344     """Multithreaded non-blocking `Resolver` implementation.
00345 
00346     Requires the `concurrent.futures` package to be installed
00347     (available in the standard library since Python 3.2,
00348     installable with ``pip install futures`` in older versions).
00349 
00350     The thread pool size can be configured with::
00351 
00352         Resolver.configure('tornado.netutil.ThreadedResolver',
00353                            num_threads=10)
00354 
00355     .. versionchanged:: 3.1
00356        All ``ThreadedResolvers`` share a single thread pool, whose
00357        size is set by the first one to be created.
00358     """
00359     _threadpool = None
00360     _threadpool_pid = None
00361 
00362     def initialize(self, io_loop=None, num_threads=10):
00363         threadpool = ThreadedResolver._create_threadpool(num_threads)
00364         super(ThreadedResolver, self).initialize(
00365             io_loop=io_loop, executor=threadpool, close_executor=False)
00366 
00367     @classmethod
00368     def _create_threadpool(cls, num_threads):
00369         pid = os.getpid()
00370         if cls._threadpool_pid != pid:
00371             # Threads cannot survive after a fork, so if our pid isn't what it
00372             # was when we created the pool then delete it.
00373             cls._threadpool = None
00374         if cls._threadpool is None:
00375             from concurrent.futures import ThreadPoolExecutor
00376             cls._threadpool = ThreadPoolExecutor(num_threads)
00377             cls._threadpool_pid = pid
00378         return cls._threadpool
00379 
00380 
00381 class OverrideResolver(Resolver):
00382     """Wraps a resolver with a mapping of overrides.
00383 
00384     This can be used to make local DNS changes (e.g. for testing)
00385     without modifying system-wide settings.
00386 
00387     The mapping can contain either host strings or host-port pairs.
00388     """
00389     def initialize(self, resolver, mapping):
00390         self.resolver = resolver
00391         self.mapping = mapping
00392 
00393     def close(self):
00394         self.resolver.close()
00395 
00396     def resolve(self, host, port, *args, **kwargs):
00397         if (host, port) in self.mapping:
00398             host, port = self.mapping[(host, port)]
00399         elif host in self.mapping:
00400             host = self.mapping[host]
00401         return self.resolver.resolve(host, port, *args, **kwargs)
00402 
00403 
00404 # These are the keyword arguments to ssl.wrap_socket that must be translated
00405 # to their SSLContext equivalents (the other arguments are still passed
00406 # to SSLContext.wrap_socket).
00407 _SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile',
00408                                    'cert_reqs', 'ca_certs', 'ciphers'])
00409 
00410 
00411 def ssl_options_to_context(ssl_options):
00412     """Try to convert an ``ssl_options`` dictionary to an
00413     `~ssl.SSLContext` object.
00414 
00415     The ``ssl_options`` dictionary contains keywords to be passed to
00416     `ssl.wrap_socket`.  In Python 3.2+, `ssl.SSLContext` objects can
00417     be used instead.  This function converts the dict form to its
00418     `~ssl.SSLContext` equivalent, and may be used when a component which
00419     accepts both forms needs to upgrade to the `~ssl.SSLContext` version
00420     to use features like SNI or NPN.
00421     """
00422     if isinstance(ssl_options, dict):
00423         assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options
00424     if (not hasattr(ssl, 'SSLContext') or
00425             isinstance(ssl_options, ssl.SSLContext)):
00426         return ssl_options
00427     context = ssl.SSLContext(
00428         ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23))
00429     if 'certfile' in ssl_options:
00430         context.load_cert_chain(ssl_options['certfile'], ssl_options.get('keyfile', None))
00431     if 'cert_reqs' in ssl_options:
00432         context.verify_mode = ssl_options['cert_reqs']
00433     if 'ca_certs' in ssl_options:
00434         context.load_verify_locations(ssl_options['ca_certs'])
00435     if 'ciphers' in ssl_options:
00436         context.set_ciphers(ssl_options['ciphers'])
00437     if hasattr(ssl, 'OP_NO_COMPRESSION'):
00438         # Disable TLS compression to avoid CRIME and related attacks.
00439         # This constant wasn't added until python 3.3.
00440         context.options |= ssl.OP_NO_COMPRESSION
00441     return context
00442 
00443 
00444 def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs):
00445     """Returns an ``ssl.SSLSocket`` wrapping the given socket.
00446 
00447     ``ssl_options`` may be either a dictionary (as accepted by
00448     `ssl_options_to_context`) or an `ssl.SSLContext` object.
00449     Additional keyword arguments are passed to ``wrap_socket``
00450     (either the `~ssl.SSLContext` method or the `ssl` module function
00451     as appropriate).
00452     """
00453     context = ssl_options_to_context(ssl_options)
00454     if hasattr(ssl, 'SSLContext') and isinstance(context, ssl.SSLContext):
00455         if server_hostname is not None and getattr(ssl, 'HAS_SNI'):
00456             # Python doesn't have server-side SNI support so we can't
00457             # really unittest this, but it can be manually tested with
00458             # python3.2 -m tornado.httpclient https://sni.velox.ch
00459             return context.wrap_socket(socket, server_hostname=server_hostname,
00460                                        **kwargs)
00461         else:
00462             return context.wrap_socket(socket, **kwargs)
00463     else:
00464         return ssl.wrap_socket(socket, **dict(context, **kwargs))


rosbridge_tools
Author(s): Jonathan Mace
autogenerated on Sat Dec 27 2014 11:25:59