gronod/headphones
1
0
Fork 1
mirror of https://github.com/rembo10/headphones.git synced 2026-05-16 08:35:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

urllib3

Browse Source 
requests supported version
This commit is contained in:
AdeHub
2019-07-14 21:02:40 +12:00
parent cf85d94a37
commit 3ab31f1da7
49 changed files with 1878 additions and 2373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
lib/urllib3/__init__.py Executable file → Normal file
View File

@@ -4,7 +4,11 @@ urllib3 - Thread-safe connection pooling and re-using.
from __future__ import absolute_import
import warnings
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
from .connectionpool import (
HTTPConnectionPool,
HTTPSConnectionPool,
connection_from_url
)
from . import exceptions
from .filepost import encode_multipart_formdata
@@ -20,25 +24,25 @@ from .util.retry import Retry
import logging
from logging import NullHandler
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
__license__ = "MIT"
__version__ = "1.25.3"
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.25'
__all__ = (
"HTTPConnectionPool",
"HTTPSConnectionPool",
"PoolManager",
"ProxyManager",
"HTTPResponse",
"Retry",
"Timeout",
"add_stderr_logger",
"connection_from_url",
"disable_warnings",
"encode_multipart_formdata",
"get_host",
"make_headers",
"proxy_from_url",
'HTTPConnectionPool',
'HTTPSConnectionPool',
'PoolManager',
'ProxyManager',
'HTTPResponse',
'Retry',
'Timeout',
'add_stderr_logger',
'connection_from_url',
'disable_warnings',
'encode_multipart_formdata',
'get_host',
'make_headers',
'proxy_from_url',
)
logging.getLogger(__name__).addHandler(NullHandler())
@@ -55,10 +59,10 @@ def add_stderr_logger(level=logging.DEBUG):
# even if urllib3 is vendored within another package.
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(handler)
logger.setLevel(level)
logger.debug("Added a stderr logging handler to logger: %s", __name__)
logger.debug('Added a stderr logging handler to logger: %s', __name__)
return handler
@@ -70,17 +74,18 @@ del NullHandler
# shouldn't be: otherwise, it's very hard for users to use most Python
# mechanisms to silence them.
# SecurityWarning's always go off by default.
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host
warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True)
warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
append=True)
# SNIMissingWarnings should go off only once.
warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True)
warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True)
def disable_warnings(category=exceptions.HTTPWarning):
"""
Helper for quickly disabling all urllib3 warnings.
"""
warnings.simplefilter("ignore", category)
warnings.simplefilter('ignore', category)

37
lib/urllib3/_collections.py Executable file → Normal file
View File

@@ -1,5 +1,4 @@
from __future__ import absolute_import
try:
from collections.abc import Mapping, MutableMapping
except ImportError:
@@ -7,7 +6,6 @@ except ImportError:
try:
from threading import RLock
except ImportError: # Platform-specific: No threads available
class RLock:
def __enter__(self):
pass
@@ -21,7 +19,7 @@ from .exceptions import InvalidHeader
from .packages.six import iterkeys, itervalues, PY3
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
_Null = object()
@@ -84,9 +82,7 @@ class RecentlyUsedContainer(MutableMapping):
return len(self._container)
def __iter__(self):
raise NotImplementedError(
"Iteration over this class is unlikely to be threadsafe."
)
raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
def clear(self):
with self.lock:
@@ -154,7 +150,7 @@ class HTTPHeaderDict(MutableMapping):
def __getitem__(self, key):
val = self._container[key.lower()]
return ", ".join(val[1:])
return ', '.join(val[1:])
def __delitem__(self, key):
del self._container[key.lower()]
@@ -163,13 +159,12 @@ class HTTPHeaderDict(MutableMapping):
return key.lower() in self._container
def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, "keys"):
if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
return False
if not isinstance(other, type(self)):
other = type(self)(other)
return dict((k.lower(), v) for k, v in self.itermerged()) == dict(
(k.lower(), v) for k, v in other.itermerged()
)
return (dict((k.lower(), v) for k, v in self.itermerged()) ==
dict((k.lower(), v) for k, v in other.itermerged()))
def __ne__(self, other):
return not self.__eq__(other)
@@ -189,9 +184,9 @@ class HTTPHeaderDict(MutableMapping):
yield vals[0]
def pop(self, key, default=__marker):
"""D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
'''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised.
"""
'''
# Using the MutableMapping function directly fails due to the private marker.
# Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel.
@@ -233,10 +228,8 @@ class HTTPHeaderDict(MutableMapping):
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError(
"extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args))
)
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
@@ -302,7 +295,7 @@ class HTTPHeaderDict(MutableMapping):
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ", ".join(val[1:])
yield val[0], ', '.join(val[1:])
def items(self):
return list(self.iteritems())
@@ -313,7 +306,7 @@ class HTTPHeaderDict(MutableMapping):
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly.
obs_fold_continued_leaders = (" ", "\t")
obs_fold_continued_leaders = (' ', '\t')
headers = []
for line in message.headers:
@@ -323,14 +316,14 @@ class HTTPHeaderDict(MutableMapping):
# in RFC-7230 S3.2.4. This indicates a multiline header, but
# there exists no previous header to which we can attach it.
raise InvalidHeader(
"Header continuation with no previous header: %s" % line
'Header continuation with no previous header: %s' % line
)
else:
key, value = headers[-1]
headers[-1] = (key, value + " " + line.strip())
headers[-1] = (key, value + ' ' + line.strip())
continue
key, value = line.split(":", 1)
key, value = line.split(':', 1)
headers.append((key, value.strip()))
return cls(headers)

195
lib/urllib3/connection.py Executable file → Normal file
View File

@@ -11,7 +11,6 @@ from .packages.six.moves.http_client import HTTPException # noqa: F401
try: # Compiled with SSL?
import ssl
BaseSSLError = ssl.SSLError
except (ImportError, AttributeError): # Platform-specific: No SSL.
ssl = None
@@ -42,7 +41,7 @@ from .util.ssl_ import (
resolve_ssl_version,
assert_fingerprint,
create_urllib3_context,
ssl_wrap_socket,
ssl_wrap_socket
)
@@ -52,16 +51,20 @@ from ._collections import HTTPHeaderDict
log = logging.getLogger(__name__)
port_by_scheme = {"http": 80, "https": 443}
port_by_scheme = {
'http': 80,
'https': 443,
}
# When it comes time to update this value as a part of regular maintenance
# (ie test_recent_date is failing) update it to ~6 months before the current date.
RECENT_DATE = datetime.date(2019, 1, 1)
# When updating RECENT_DATE, move it to within two years of the current date,
# and not less than 6 months ago.
# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or
# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months)
RECENT_DATE = datetime.date(2017, 6, 30)
class DummyConnection(object):
"""Used to detect a failed ConnectionCls import."""
pass
@@ -89,7 +92,7 @@ class HTTPConnection(_HTTPConnection, object):
Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
"""
default_port = port_by_scheme["http"]
default_port = port_by_scheme['http']
#: Disable Nagle's algorithm by default.
#: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
@@ -100,14 +103,14 @@ class HTTPConnection(_HTTPConnection, object):
def __init__(self, *args, **kw):
if six.PY3:
kw.pop("strict", None)
kw.pop('strict', None)
# Pre-set source_address.
self.source_address = kw.get("source_address")
self.source_address = kw.get('source_address')
#: The socket options provided by the user. If no options are
#: provided, we use the default options.
self.socket_options = kw.pop("socket_options", self.default_socket_options)
self.socket_options = kw.pop('socket_options', self.default_socket_options)
_HTTPConnection.__init__(self, *args, **kw)
@@ -128,7 +131,7 @@ class HTTPConnection(_HTTPConnection, object):
those cases where it's appropriate (i.e., when doing DNS lookup to establish the
actual TCP connection across which we're going to send HTTP requests).
"""
return self._dns_host.rstrip(".")
return self._dns_host.rstrip('.')
@host.setter
def host(self, value):
@@ -147,34 +150,30 @@ class HTTPConnection(_HTTPConnection, object):
"""
extra_kw = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
extra_kw['source_address'] = self.source_address
if self.socket_options:
extra_kw["socket_options"] = self.socket_options
extra_kw['socket_options'] = self.socket_options
try:
conn = connection.create_connection(
(self._dns_host, self.port), self.timeout, **extra_kw
)
(self._dns_host, self.port), self.timeout, **extra_kw)
except SocketTimeout:
raise ConnectTimeoutError(
self,
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
self, "Connection to %s timed out. (connect timeout=%s)" %
(self.host, self.timeout))
except SocketError as e:
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e
)
self, "Failed to establish a new connection: %s" % e)
return conn
def _prepare_conn(self, conn):
self.sock = conn
# Google App Engine's httplib does not define _tunnel_host
if getattr(self, "_tunnel_host", None):
if getattr(self, '_tunnel_host', None):
# TODO: Fix tunnel so it doesn't depend on self.sock state.
self._tunnel()
# Mark this connection as not reusable
@@ -190,15 +189,18 @@ class HTTPConnection(_HTTPConnection, object):
body with chunked encoding and not as one block
"""
headers = HTTPHeaderDict(headers if headers is not None else {})
skip_accept_encoding = "accept-encoding" in headers
skip_host = "host" in headers
skip_accept_encoding = 'accept-encoding' in headers
skip_host = 'host' in headers
self.putrequest(
method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
method,
url,
skip_accept_encoding=skip_accept_encoding,
skip_host=skip_host
)
for header, value in headers.items():
self.putheader(header, value)
if "transfer-encoding" not in headers:
self.putheader("Transfer-Encoding", "chunked")
if 'transfer-encoding' not in headers:
self.putheader('Transfer-Encoding', 'chunked')
self.endheaders()
if body is not None:
@@ -209,37 +211,29 @@ class HTTPConnection(_HTTPConnection, object):
if not chunk:
continue
if not isinstance(chunk, bytes):
chunk = chunk.encode("utf8")
chunk = chunk.encode('utf8')
len_str = hex(len(chunk))[2:]
self.send(len_str.encode("utf-8"))
self.send(b"\r\n")
self.send(len_str.encode('utf-8'))
self.send(b'\r\n')
self.send(chunk)
self.send(b"\r\n")
self.send(b'\r\n')
# After the if clause, to always have a closed body
self.send(b"0\r\n\r\n")
self.send(b'0\r\n\r\n')
class HTTPSConnection(HTTPConnection):
default_port = port_by_scheme["https"]
default_port = port_by_scheme['https']
ssl_version = None
def __init__(
self,
host,
port=None,
key_file=None,
cert_file=None,
key_password=None,
strict=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
ssl_context=None,
server_hostname=None,
**kw
):
def __init__(self, host, port=None, key_file=None, cert_file=None,
key_password=None, strict=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
ssl_context=None, server_hostname=None, **kw):
HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw)
HTTPConnection.__init__(self, host, port, strict=strict,
timeout=timeout, **kw)
self.key_file = key_file
self.cert_file = cert_file
@@ -249,40 +243,25 @@ class HTTPSConnection(HTTPConnection):
# Required property for Google AppEngine 1.9.0 which otherwise causes
# HTTPS requests to go out as HTTP. (See Issue #356)
self._protocol = "https"
self._protocol = 'https'
def connect(self):
conn = self._new_conn()
self._prepare_conn(conn)
# Wrap socket using verification with the root certs in
# trusted_root_certs
default_ssl_context = False
if self.ssl_context is None:
default_ssl_context = True
self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(self.cert_reqs),
ssl_version=resolve_ssl_version(None),
cert_reqs=resolve_cert_reqs(None),
)
# Try to load OS default certs if none are given.
# Works well on Windows (requires Python3.4+)
context = self.ssl_context
if (
not self.ca_certs
and not self.ca_cert_dir
and default_ssl_context
and hasattr(context, "load_default_certs")
):
context.load_default_certs()
self.sock = ssl_wrap_socket(
sock=conn,
keyfile=self.key_file,
certfile=self.cert_file,
key_password=self.key_password,
ssl_context=self.ssl_context,
server_hostname=self.server_hostname,
server_hostname=self.server_hostname
)
@@ -291,24 +270,16 @@ class VerifiedHTTPSConnection(HTTPSConnection):
Based on httplib.HTTPSConnection but wraps the socket with
SSL certification.
"""
cert_reqs = None
ca_certs = None
ca_cert_dir = None
ssl_version = None
assert_fingerprint = None
def set_cert(
self,
key_file=None,
cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
):
def set_cert(self, key_file=None, cert_file=None,
cert_reqs=None, key_password=None, ca_certs=None,
assert_hostname=None, assert_fingerprint=None,
ca_cert_dir=None):
"""
This method should only be called once, before the connection is used.
"""
@@ -335,7 +306,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
hostname = self.host
# Google App Engine's httplib does not define _tunnel_host
if getattr(self, "_tunnel_host", None):
if getattr(self, '_tunnel_host', None):
self.sock = conn
# Calls self._set_hostport(), so self.host is
# self._tunnel_host below.
@@ -352,19 +323,15 @@ class VerifiedHTTPSConnection(HTTPSConnection):
is_time_off = datetime.date.today() < RECENT_DATE
if is_time_off:
warnings.warn(
(
"System time is way off (before {0}). This will probably "
"lead to SSL verification errors"
).format(RECENT_DATE),
SystemTimeWarning,
warnings.warn((
'System time is way off (before {0}). This will probably '
'lead to SSL verification errors').format(RECENT_DATE),
SystemTimeWarning
)
# Wrap socket using verification with the root certs in
# trusted_root_certs
default_ssl_context = False
if self.ssl_context is None:
default_ssl_context = True
self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(self.cert_reqs),
@@ -372,17 +339,6 @@ class VerifiedHTTPSConnection(HTTPSConnection):
context = self.ssl_context
context.verify_mode = resolve_cert_reqs(self.cert_reqs)
# Try to load OS default certs if none are given.
# Works well on Windows (requires Python3.4+)
if (
not self.ca_certs
and not self.ca_cert_dir
and default_ssl_context
and hasattr(context, "load_default_certs")
):
context.load_default_certs()
self.sock = ssl_wrap_socket(
sock=conn,
keyfile=self.key_file,
@@ -391,37 +347,31 @@ class VerifiedHTTPSConnection(HTTPSConnection):
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
server_hostname=server_hostname,
ssl_context=context,
)
ssl_context=context)
if self.assert_fingerprint:
assert_fingerprint(
self.sock.getpeercert(binary_form=True), self.assert_fingerprint
)
elif (
context.verify_mode != ssl.CERT_NONE
and not getattr(context, "check_hostname", False)
and self.assert_hostname is not False
):
assert_fingerprint(self.sock.getpeercert(binary_form=True),
self.assert_fingerprint)
elif context.verify_mode != ssl.CERT_NONE \
and not getattr(context, 'check_hostname', False) \
and self.assert_hostname is not False:
# While urllib3 attempts to always turn off hostname matching from
# the TLS library, this cannot always be done. So we check whether
# the TLS Library still thinks it's matching hostnames.
cert = self.sock.getpeercert()
if not cert.get("subjectAltName", ()):
warnings.warn(
(
"Certificate for {0} has no `subjectAltName`, falling back to check for a "
"`commonName` for now. This feature is being removed by major browsers and "
"deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 "
"for details.)".format(hostname)
),
SubjectAltNameWarning,
if not cert.get('subjectAltName', ()):
warnings.warn((
'Certificate for {0} has no `subjectAltName`, falling back to check for a '
'`commonName` for now. This feature is being removed by major browsers and '
'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
'for details.)'.format(hostname)),
SubjectAltNameWarning
)
_match_hostname(cert, self.assert_hostname or server_hostname)
self.is_verified = (
context.verify_mode == ssl.CERT_REQUIRED
or self.assert_fingerprint is not None
context.verify_mode == ssl.CERT_REQUIRED or
self.assert_fingerprint is not None
)
@@ -430,9 +380,8 @@ def _match_hostname(cert, asserted_hostname):
match_hostname(cert, asserted_hostname)
except CertificateError as e:
log.error(
"Certificate did not match expected hostname: %s. " "Certificate: %s",
asserted_hostname,
cert,
'Certificate did not match expected hostname: %s. '
'Certificate: %s', asserted_hostname, cert
)
# Add cert to exception and reraise so client code can inspect
# the cert when catching the exception, if they want to

391
lib/urllib3/connectionpool.py Executable file → Normal file
View File

@@ -30,11 +30,8 @@ from .packages.rfc3986.normalizers import normalize_host
from .connection import (
port_by_scheme,
DummyConnection,
HTTPConnection,
HTTPSConnection,
VerifiedHTTPSConnection,
HTTPException,
BaseSSLError,
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
HTTPException, BaseSSLError,
)
from .request import RequestMethods
from .response import HTTPResponse
@@ -74,7 +71,8 @@ class ConnectionPool(object):
self.port = port
def __str__(self):
return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
return '%s(host=%r, port=%r)' % (type(self).__name__,
self.host, self.port)
def __enter__(self):
return self
@@ -155,24 +153,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
:class:`urllib3.connection.HTTPSConnection` instances.
"""
scheme = "http"
scheme = 'http'
ConnectionCls = HTTPConnection
ResponseCls = HTTPResponse
def __init__(
self,
host,
port=None,
strict=False,
timeout=Timeout.DEFAULT_TIMEOUT,
maxsize=1,
block=False,
headers=None,
retries=None,
_proxy=None,
_proxy_headers=None,
**conn_kw
):
def __init__(self, host, port=None, strict=False,
timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
headers=None, retries=None,
_proxy=None, _proxy_headers=None,
**conn_kw):
ConnectionPool.__init__(self, host, port)
RequestMethods.__init__(self, headers)
@@ -206,27 +195,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
# We cannot know if the user has added default socket options, so we cannot replace the
# list.
self.conn_kw.setdefault("socket_options", [])
self.conn_kw.setdefault('socket_options', [])
def _new_conn(self):
"""
Return a fresh :class:`HTTPConnection`.
"""
self.num_connections += 1
log.debug(
"Starting new HTTP connection (%d): %s:%s",
self.num_connections,
self.host,
self.port or "80",
)
log.debug("Starting new HTTP connection (%d): %s:%s",
self.num_connections, self.host, self.port or "80")
conn = self.ConnectionCls(
host=self.host,
port=self.port,
timeout=self.timeout.connect_timeout,
strict=self.strict,
**self.conn_kw
)
conn = self.ConnectionCls(host=self.host, port=self.port,
timeout=self.timeout.connect_timeout,
strict=self.strict, **self.conn_kw)
return conn
def _get_conn(self, timeout=None):
@@ -250,17 +231,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
except queue.Empty:
if self.block:
raise EmptyPoolError(
self,
"Pool reached maximum size and no more " "connections are allowed.",
)
raise EmptyPoolError(self,
"Pool reached maximum size and no more "
"connections are allowed.")
pass # Oh well, we'll create a new connection then
# If this is a persistent connection, check if it got disconnected
if conn and is_connection_dropped(conn):
log.debug("Resetting dropped connection: %s", self.host)
conn.close()
if getattr(conn, "auto_open", 1) == 0:
if getattr(conn, 'auto_open', 1) == 0:
# This is a proxied connection that has been mutated by
# httplib._tunnel() and cannot be reused (since it would
# attempt to bypass the proxy)
@@ -290,7 +270,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
pass
except queue.Full:
# This should never happen if self.block == True
log.warning("Connection pool is full, discarding connection: %s", self.host)
log.warning(
"Connection pool is full, discarding connection: %s",
self.host)
# Connection never got put back into the pool, close it.
if conn:
@@ -322,30 +304,21 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
if isinstance(err, SocketTimeout):
raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
# See the above comment about EAGAIN in Python 3. In Python 2 we have
# to specifically catch it and throw the timeout error
if hasattr(err, "errno") and err.errno in _blocking_errnos:
raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
if hasattr(err, 'errno') and err.errno in _blocking_errnos:
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
# Catch possible read timeouts thrown as SSL errors. If not the
# case, rethrow the original. We need to do this because of:
# http://bugs.python.org/issue10272
if "timed out" in str(err) or "did not complete (read)" in str(
err
): # Python < 2.7.4
raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
def _make_request(
self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
):
def _make_request(self, conn, method, url, timeout=_Default, chunked=False,
**httplib_request_kw):
"""
Perform a request on a given urllib connection object taken from our
pool.
@@ -385,7 +358,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
read_timeout = timeout_obj.read_timeout
# App Engine doesn't have a sock attr
if getattr(conn, "sock", None):
if getattr(conn, 'sock', None):
# In Python 3 socket.py will catch EAGAIN and return None when you
# try and read into the file pointer created by http.client, which
# instead raises a BadStatusLine exception. Instead of catching
@@ -393,8 +366,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# timeouts, check for a zero timeout before making the request.
if read_timeout == 0:
raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % read_timeout
)
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
if read_timeout is Timeout.DEFAULT_TIMEOUT:
conn.sock.settimeout(socket.getdefaulttimeout())
else: # None or a value
@@ -409,38 +381,26 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Python 3
try:
httplib_response = conn.getresponse()
except BaseException as e:
# Remove the TypeError from the exception chain in
# Python 3 (including for exceptions like SystemExit).
# Otherwise it looks like a bug in the code.
except Exception as e:
# Remove the TypeError from the exception chain in Python 3;
# otherwise it looks like a programming error was the cause.
six.raise_from(e, None)
except (SocketTimeout, BaseSSLError, SocketError) as e:
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
raise
# AppEngine doesn't have a version attr.
http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
log.debug(
'%s://%s:%s "%s %s %s" %s %s',
self.scheme,
self.host,
self.port,
method,
url,
http_version,
httplib_response.status,
httplib_response.length,
)
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
method, url, http_version, httplib_response.status,
httplib_response.length)
try:
assert_header_parsing(httplib_response.msg)
except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3
log.warning(
"Failed to parse headers (url=%s): %s",
self._absolute_url(url),
hpe,
exc_info=True,
)
'Failed to parse headers (url=%s): %s',
self._absolute_url(url), hpe, exc_info=True)
return httplib_response
@@ -470,7 +430,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
Check if the given ``url`` is a member of the same host as this
connection pool.
"""
if url.startswith("/"):
if url.startswith('/'):
return True
# TODO: Add optional support for socket.gethostbyname checking.
@@ -486,22 +446,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return (scheme, host, port) == (self.scheme, self.host, self.port)
def urlopen(
self,
method,
url,
body=None,
headers=None,
retries=None,
redirect=True,
assert_same_host=True,
timeout=_Default,
pool_timeout=None,
release_conn=None,
chunked=False,
body_pos=None,
**response_kw
):
def urlopen(self, method, url, body=None, headers=None, retries=None,
redirect=True, assert_same_host=True, timeout=_Default,
pool_timeout=None, release_conn=None, chunked=False,
body_pos=None, **response_kw):
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
@@ -599,7 +547,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = response_kw.get("preload_content", True)
release_conn = response_kw.get('preload_content', True)
# Check host
if assert_same_host and not self.is_same_host(url):
@@ -621,7 +569,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Merge the proxy headers. Only do this in HTTP. We have to copy the
# headers dict so we can safely change it without those changes being
# reflected in anyone else's copy.
if self.scheme == "http":
if self.scheme == 'http':
headers = headers.copy()
headers.update(self.proxy_headers)
@@ -644,22 +592,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn.timeout = timeout_obj.connect_timeout
is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
)
is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None)
if is_new_proxy_conn:
self._prepare_proxy(conn)
# Make the request on the httplib connection object.
httplib_response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
)
httplib_response = self._make_request(conn, method, url,
timeout=timeout_obj,
body=body, headers=headers,
chunked=chunked)
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
@@ -668,16 +609,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
response_conn = conn if not release_conn else None
# Pass method to Response for length checking
response_kw["request_method"] = method
response_kw['request_method'] = method
# Import httplib's response into our own wrapper object
response = self.ResponseCls.from_httplib(
httplib_response,
pool=self,
connection=response_conn,
retries=retries,
**response_kw
)
response = self.ResponseCls.from_httplib(httplib_response,
pool=self,
connection=response_conn,
retries=retries,
**response_kw)
# Everything went great!
clean_exit = True
@@ -686,28 +625,20 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Timed out by queue.
raise EmptyPoolError(self, "No pool connections are available.")
except (
TimeoutError,
HTTPException,
SocketError,
ProtocolError,
BaseSSLError,
SSLError,
CertificateError,
) as e:
except (TimeoutError, HTTPException, SocketError, ProtocolError,
BaseSSLError, SSLError, CertificateError) as e:
# Discard the connection for these exceptions. It will be
# replaced during the next _get_conn() call.
clean_exit = False
if isinstance(e, (BaseSSLError, CertificateError)):
e = SSLError(e)
elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
e = ProxyError("Cannot connect to proxy.", e)
e = ProxyError('Cannot connect to proxy.', e)
elif isinstance(e, (SocketError, HTTPException)):
e = ProtocolError("Connection aborted.", e)
e = ProtocolError('Connection aborted.', e)
retries = retries.increment(
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
)
retries = retries.increment(method, url, error=e, _pool=self,
_stacktrace=sys.exc_info()[2])
retries.sleep()
# Keep track of the error for the retry warning.
@@ -730,47 +661,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if not conn:
# Try again
log.warning(
"Retrying (%r) after connection " "broken by '%r': %s",
retries,
err,
url,
)
return self.urlopen(
method,
url,
body,
headers,
retries,
redirect,
assert_same_host,
timeout=timeout,
pool_timeout=pool_timeout,
release_conn=release_conn,
body_pos=body_pos,
**response_kw
)
log.warning("Retrying (%r) after connection "
"broken by '%r': %s", retries, err, url)
return self.urlopen(method, url, body, headers, retries,
redirect, assert_same_host,
timeout=timeout, pool_timeout=pool_timeout,
release_conn=release_conn, body_pos=body_pos,
**response_kw)
def drain_and_release_conn(response):
try:
# discard any remaining response body, the connection will be
# released back to the pool once the entire response is read
response.read()
except (
TimeoutError,
HTTPException,
SocketError,
ProtocolError,
BaseSSLError,
SSLError,
):
except (TimeoutError, HTTPException, SocketError, ProtocolError,
BaseSSLError, SSLError):
pass
# Handle redirect?
redirect_location = redirect and response.get_redirect_location()
if redirect_location:
if response.status == 303:
method = "GET"
method = 'GET'
try:
retries = retries.increment(method, url, response=response, _pool=self)
@@ -788,22 +700,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries.sleep_for_retry(response)
log.debug("Redirecting %s -> %s", url, redirect_location)
return self.urlopen(
method,
redirect_location,
body,
headers,
retries=retries,
redirect=redirect,
method, redirect_location, body, headers,
retries=retries, redirect=redirect,
assert_same_host=assert_same_host,
timeout=timeout,
pool_timeout=pool_timeout,
release_conn=release_conn,
body_pos=body_pos,
**response_kw
)
timeout=timeout, pool_timeout=pool_timeout,
release_conn=release_conn, body_pos=body_pos,
**response_kw)
# Check if we should retry the HTTP response.
has_retry_after = bool(response.getheader("Retry-After"))
has_retry_after = bool(response.getheader('Retry-After'))
if retries.is_retry(method, response.status, has_retry_after):
try:
retries = retries.increment(method, url, response=response, _pool=self)
@@ -821,19 +726,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries.sleep(response)
log.debug("Retry: %s", url)
return self.urlopen(
method,
url,
body,
headers,
retries=retries,
redirect=redirect,
method, url, body, headers,
retries=retries, redirect=redirect,
assert_same_host=assert_same_host,
timeout=timeout,
pool_timeout=pool_timeout,
timeout=timeout, pool_timeout=pool_timeout,
release_conn=release_conn,
body_pos=body_pos,
**response_kw
)
body_pos=body_pos, **response_kw)
return response
@@ -856,47 +754,21 @@ class HTTPSConnectionPool(HTTPConnectionPool):
the connection socket into an SSL socket.
"""
scheme = "https"
scheme = 'https'
ConnectionCls = HTTPSConnection
def __init__(
self,
host,
port=None,
strict=False,
timeout=Timeout.DEFAULT_TIMEOUT,
maxsize=1,
block=False,
headers=None,
retries=None,
_proxy=None,
_proxy_headers=None,
key_file=None,
cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
ssl_version=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
**conn_kw
):
def __init__(self, host, port=None,
strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1,
block=False, headers=None, retries=None,
_proxy=None, _proxy_headers=None,
key_file=None, cert_file=None, cert_reqs=None,
key_password=None, ca_certs=None, ssl_version=None,
assert_hostname=None, assert_fingerprint=None,
ca_cert_dir=None, **conn_kw):
HTTPConnectionPool.__init__(
self,
host,
port,
strict,
timeout,
maxsize,
block,
headers,
retries,
_proxy,
_proxy_headers,
**conn_kw
)
HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
block, headers, retries, _proxy, _proxy_headers,
**conn_kw)
self.key_file = key_file
self.cert_file = cert_file
@@ -915,16 +787,14 @@ class HTTPSConnectionPool(HTTPConnectionPool):
"""
if isinstance(conn, VerifiedHTTPSConnection):
conn.set_cert(
key_file=self.key_file,
key_password=self.key_password,
cert_file=self.cert_file,
cert_reqs=self.cert_reqs,
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint,
)
conn.set_cert(key_file=self.key_file,
key_password=self.key_password,
cert_file=self.cert_file,
cert_reqs=self.cert_reqs,
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint)
conn.ssl_version = self.ssl_version
return conn
@@ -941,17 +811,12 @@ class HTTPSConnectionPool(HTTPConnectionPool):
Return a fresh :class:`httplib.HTTPSConnection`.
"""
self.num_connections += 1
log.debug(
"Starting new HTTPS connection (%d): %s:%s",
self.num_connections,
self.host,
self.port or "443",
)
log.debug("Starting new HTTPS connection (%d): %s:%s",
self.num_connections, self.host, self.port or "443")
if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
raise SSLError(
"Can't connect to HTTPS URL because the SSL " "module is not available."
)
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
actual_host = self.host
actual_port = self.port
@@ -959,16 +824,11 @@ class HTTPSConnectionPool(HTTPConnectionPool):
actual_host = self.proxy.host
actual_port = self.proxy.port
conn = self.ConnectionCls(
host=actual_host,
port=actual_port,
timeout=self.timeout.connect_timeout,
strict=self.strict,
cert_file=self.cert_file,
key_file=self.key_file,
key_password=self.key_password,
**self.conn_kw
)
conn = self.ConnectionCls(host=actual_host, port=actual_port,
timeout=self.timeout.connect_timeout,
strict=self.strict, cert_file=self.cert_file,
key_file=self.key_file, key_password=self.key_password,
**self.conn_kw)
return self._prepare_conn(conn)
@@ -979,19 +839,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
super(HTTPSConnectionPool, self)._validate_conn(conn)
# Force connect early to allow us to validate the connection.
if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
if not getattr(conn, 'sock', None): # AppEngine might not have `.sock`
conn.connect()
if not conn.is_verified:
warnings.warn(
(
"Unverified HTTPS request is being made. "
"Adding certificate verification is strongly advised. See: "
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
"#ssl-warnings"
),
InsecureRequestWarning,
)
warnings.warn((
'Unverified HTTPS request is being made. '
'Adding certificate verification is strongly advised. See: '
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
'#ssl-warnings'),
InsecureRequestWarning)
def connection_from_url(url, **kw):
@@ -1016,7 +873,7 @@ def connection_from_url(url, **kw):
"""
scheme, host, port = get_host(url)
port = port or port_by_scheme.get(scheme, 80)
if scheme == "https":
if scheme == 'https':
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
@@ -1033,8 +890,8 @@ def _normalize_host(host, scheme):
# Instead, we need to make sure we never pass ``None`` as the port.
# However, for backward compatibility reasons we can't actually
# *assert* that. See http://bugs.python.org/issue28539
if host.startswith("[") and host.endswith("]"):
host = host.strip("[]")
if host.startswith('[') and host.endswith(']'):
host = host.strip('[]')
if scheme in NORMALIZABLE_SCHEMES:
host = normalize_host(host)
return host

0
lib/urllib3/contrib/__init__.py Executable file → Normal file
View File

20
lib/urllib3/contrib/_appengine_environ.py Executable file → Normal file
View File

@@ -6,7 +6,9 @@ import os
def is_appengine():
return is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms()
return (is_local_appengine() or
is_prod_appengine() or
is_prod_appengine_mvms())
def is_appengine_sandbox():
@@ -14,19 +16,15 @@ def is_appengine_sandbox():
def is_local_appengine():
return (
"APPENGINE_RUNTIME" in os.environ
and "Development/" in os.environ["SERVER_SOFTWARE"]
)
return ('APPENGINE_RUNTIME' in os.environ and
'Development/' in os.environ['SERVER_SOFTWARE'])
def is_prod_appengine():
return (
"APPENGINE_RUNTIME" in os.environ
and "Google App Engine/" in os.environ["SERVER_SOFTWARE"]
and not is_prod_appengine_mvms()
)
return ('APPENGINE_RUNTIME' in os.environ and
'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
not is_prod_appengine_mvms())
def is_prod_appengine_mvms():
return os.environ.get("GAE_VM", False) == "true"
return os.environ.get('GAE_VM', False) == 'true'

0
lib/urllib3/contrib/_securetransport/__init__.py Executable file → Normal file
View File

261
lib/urllib3/contrib/_securetransport/bindings.py Executable file → Normal file
View File

@@ -34,35 +34,29 @@ from __future__ import absolute_import
import platform
from ctypes.util import find_library
from ctypes import (
c_void_p,
c_int32,
c_char_p,
c_size_t,
c_byte,
c_uint32,
c_ulong,
c_long,
c_bool,
c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long,
c_bool
)
from ctypes import CDLL, POINTER, CFUNCTYPE
security_path = find_library("Security")
security_path = find_library('Security')
if not security_path:
raise ImportError("The library Security could not be found")
raise ImportError('The library Security could not be found')
core_foundation_path = find_library("CoreFoundation")
core_foundation_path = find_library('CoreFoundation')
if not core_foundation_path:
raise ImportError("The library CoreFoundation could not be found")
raise ImportError('The library CoreFoundation could not be found')
version = platform.mac_ver()[0]
version_info = tuple(map(int, version.split(".")))
version_info = tuple(map(int, version.split('.')))
if version_info < (10, 8):
raise OSError(
"Only OS X 10.8 and newer are supported, not %s.%s"
% (version_info[0], version_info[1])
'Only OS X 10.8 and newer are supported, not %s.%s' % (
version_info[0], version_info[1]
)
)
Security = CDLL(security_path, use_errno=True)
@@ -135,19 +129,27 @@ try:
Security.SecKeyGetTypeID.argtypes = []
Security.SecKeyGetTypeID.restype = CFTypeID
Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef]
Security.SecCertificateCreateWithData.argtypes = [
CFAllocatorRef,
CFDataRef
]
Security.SecCertificateCreateWithData.restype = SecCertificateRef
Security.SecCertificateCopyData.argtypes = [SecCertificateRef]
Security.SecCertificateCopyData.argtypes = [
SecCertificateRef
]
Security.SecCertificateCopyData.restype = CFDataRef
Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
Security.SecCopyErrorMessageString.argtypes = [
OSStatus,
c_void_p
]
Security.SecCopyErrorMessageString.restype = CFStringRef
Security.SecIdentityCreateWithCertificate.argtypes = [
CFTypeRef,
SecCertificateRef,
POINTER(SecIdentityRef),
POINTER(SecIdentityRef)
]
Security.SecIdentityCreateWithCertificate.restype = OSStatus
@@ -157,126 +159,201 @@ try:
c_void_p,
Boolean,
c_void_p,
POINTER(SecKeychainRef),
POINTER(SecKeychainRef)
]
Security.SecKeychainCreate.restype = OSStatus
Security.SecKeychainDelete.argtypes = [SecKeychainRef]
Security.SecKeychainDelete.argtypes = [
SecKeychainRef
]
Security.SecKeychainDelete.restype = OSStatus
Security.SecPKCS12Import.argtypes = [
CFDataRef,
CFDictionaryRef,
POINTER(CFArrayRef),
POINTER(CFArrayRef)
]
Security.SecPKCS12Import.restype = OSStatus
SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t))
SSLWriteFunc = CFUNCTYPE(
OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)
)
SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t))
Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc]
Security.SSLSetIOFuncs.argtypes = [
SSLContextRef,
SSLReadFunc,
SSLWriteFunc
]
Security.SSLSetIOFuncs.restype = OSStatus
Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t]
Security.SSLSetPeerID.argtypes = [
SSLContextRef,
c_char_p,
c_size_t
]
Security.SSLSetPeerID.restype = OSStatus
Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef]
Security.SSLSetCertificate.argtypes = [
SSLContextRef,
CFArrayRef
]
Security.SSLSetCertificate.restype = OSStatus
Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean]
Security.SSLSetCertificateAuthorities.argtypes = [
SSLContextRef,
CFTypeRef,
Boolean
]
Security.SSLSetCertificateAuthorities.restype = OSStatus
Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef]
Security.SSLSetConnection.argtypes = [
SSLContextRef,
SSLConnectionRef
]
Security.SSLSetConnection.restype = OSStatus
Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t]
Security.SSLSetPeerDomainName.argtypes = [
SSLContextRef,
c_char_p,
c_size_t
]
Security.SSLSetPeerDomainName.restype = OSStatus
Security.SSLHandshake.argtypes = [SSLContextRef]
Security.SSLHandshake.argtypes = [
SSLContextRef
]
Security.SSLHandshake.restype = OSStatus
Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
Security.SSLRead.argtypes = [
SSLContextRef,
c_char_p,
c_size_t,
POINTER(c_size_t)
]
Security.SSLRead.restype = OSStatus
Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
Security.SSLWrite.argtypes = [
SSLContextRef,
c_char_p,
c_size_t,
POINTER(c_size_t)
]
Security.SSLWrite.restype = OSStatus
Security.SSLClose.argtypes = [SSLContextRef]
Security.SSLClose.argtypes = [
SSLContextRef
]
Security.SSLClose.restype = OSStatus
Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)]
Security.SSLGetNumberSupportedCiphers.argtypes = [
SSLContextRef,
POINTER(c_size_t)
]
Security.SSLGetNumberSupportedCiphers.restype = OSStatus
Security.SSLGetSupportedCiphers.argtypes = [
SSLContextRef,
POINTER(SSLCipherSuite),
POINTER(c_size_t),
POINTER(c_size_t)
]
Security.SSLGetSupportedCiphers.restype = OSStatus
Security.SSLSetEnabledCiphers.argtypes = [
SSLContextRef,
POINTER(SSLCipherSuite),
c_size_t,
c_size_t
]
Security.SSLSetEnabledCiphers.restype = OSStatus
Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)]
Security.SSLGetNumberEnabledCiphers.argtype = [
SSLContextRef,
POINTER(c_size_t)
]
Security.SSLGetNumberEnabledCiphers.restype = OSStatus
Security.SSLGetEnabledCiphers.argtypes = [
SSLContextRef,
POINTER(SSLCipherSuite),
POINTER(c_size_t),
POINTER(c_size_t)
]
Security.SSLGetEnabledCiphers.restype = OSStatus
Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)]
Security.SSLGetNegotiatedCipher.argtypes = [
SSLContextRef,
POINTER(SSLCipherSuite)
]
Security.SSLGetNegotiatedCipher.restype = OSStatus
Security.SSLGetNegotiatedProtocolVersion.argtypes = [
SSLContextRef,
POINTER(SSLProtocol),
POINTER(SSLProtocol)
]
Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus
Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)]
Security.SSLCopyPeerTrust.argtypes = [
SSLContextRef,
POINTER(SecTrustRef)
]
Security.SSLCopyPeerTrust.restype = OSStatus
Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef]
Security.SecTrustSetAnchorCertificates.argtypes = [
SecTrustRef,
CFArrayRef
]
Security.SecTrustSetAnchorCertificates.restype = OSStatus
Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean]
Security.SecTrustSetAnchorCertificatesOnly.argstypes = [
SecTrustRef,
Boolean
]
Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus
Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)]
Security.SecTrustEvaluate.argtypes = [
SecTrustRef,
POINTER(SecTrustResultType)
]
Security.SecTrustEvaluate.restype = OSStatus
Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef]
Security.SecTrustGetCertificateCount.argtypes = [
SecTrustRef
]
Security.SecTrustGetCertificateCount.restype = CFIndex
Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex]
Security.SecTrustGetCertificateAtIndex.argtypes = [
SecTrustRef,
CFIndex
]
Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef
Security.SSLCreateContext.argtypes = [
CFAllocatorRef,
SSLProtocolSide,
SSLConnectionType,
SSLConnectionType
]
Security.SSLCreateContext.restype = SSLContextRef
Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean]
Security.SSLSetSessionOption.argtypes = [
SSLContextRef,
SSLSessionOption,
Boolean
]
Security.SSLSetSessionOption.restype = OSStatus
Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol]
Security.SSLSetProtocolVersionMin.argtypes = [
SSLContextRef,
SSLProtocol
]
Security.SSLSetProtocolVersionMin.restype = OSStatus
Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol]
Security.SSLSetProtocolVersionMax.argtypes = [
SSLContextRef,
SSLProtocol
]
Security.SSLSetProtocolVersionMax.restype = OSStatus
Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
Security.SecCopyErrorMessageString.argtypes = [
OSStatus,
c_void_p
]
Security.SecCopyErrorMessageString.restype = CFStringRef
Security.SSLReadFunc = SSLReadFunc
@@ -292,47 +369,64 @@ try:
Security.OSStatus = OSStatus
Security.kSecImportExportPassphrase = CFStringRef.in_dll(
Security, "kSecImportExportPassphrase"
Security, 'kSecImportExportPassphrase'
)
Security.kSecImportItemIdentity = CFStringRef.in_dll(
Security, "kSecImportItemIdentity"
Security, 'kSecImportItemIdentity'
)
# CoreFoundation time!
CoreFoundation.CFRetain.argtypes = [CFTypeRef]
CoreFoundation.CFRetain.argtypes = [
CFTypeRef
]
CoreFoundation.CFRetain.restype = CFTypeRef
CoreFoundation.CFRelease.argtypes = [CFTypeRef]
CoreFoundation.CFRelease.argtypes = [
CFTypeRef
]
CoreFoundation.CFRelease.restype = None
CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef]
CoreFoundation.CFGetTypeID.argtypes = [
CFTypeRef
]
CoreFoundation.CFGetTypeID.restype = CFTypeID
CoreFoundation.CFStringCreateWithCString.argtypes = [
CFAllocatorRef,
c_char_p,
CFStringEncoding,
CFStringEncoding
]
CoreFoundation.CFStringCreateWithCString.restype = CFStringRef
CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding]
CoreFoundation.CFStringGetCStringPtr.argtypes = [
CFStringRef,
CFStringEncoding
]
CoreFoundation.CFStringGetCStringPtr.restype = c_char_p
CoreFoundation.CFStringGetCString.argtypes = [
CFStringRef,
c_char_p,
CFIndex,
CFStringEncoding,
CFStringEncoding
]
CoreFoundation.CFStringGetCString.restype = c_bool
CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex]
CoreFoundation.CFDataCreate.argtypes = [
CFAllocatorRef,
c_char_p,
CFIndex
]
CoreFoundation.CFDataCreate.restype = CFDataRef
CoreFoundation.CFDataGetLength.argtypes = [CFDataRef]
CoreFoundation.CFDataGetLength.argtypes = [
CFDataRef
]
CoreFoundation.CFDataGetLength.restype = CFIndex
CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef]
CoreFoundation.CFDataGetBytePtr.argtypes = [
CFDataRef
]
CoreFoundation.CFDataGetBytePtr.restype = c_void_p
CoreFoundation.CFDictionaryCreate.argtypes = [
@@ -341,11 +435,14 @@ try:
POINTER(CFTypeRef),
CFIndex,
CFDictionaryKeyCallBacks,
CFDictionaryValueCallBacks,
CFDictionaryValueCallBacks
]
CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef
CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef]
CoreFoundation.CFDictionaryGetValue.argtypes = [
CFDictionaryRef,
CFTypeRef
]
CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef
CoreFoundation.CFArrayCreate.argtypes = [
@@ -359,30 +456,36 @@ try:
CoreFoundation.CFArrayCreateMutable.argtypes = [
CFAllocatorRef,
CFIndex,
CFArrayCallBacks,
CFArrayCallBacks
]
CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef
CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p]
CoreFoundation.CFArrayAppendValue.argtypes = [
CFMutableArrayRef,
c_void_p
]
CoreFoundation.CFArrayAppendValue.restype = None
CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef]
CoreFoundation.CFArrayGetCount.argtypes = [
CFArrayRef
]
CoreFoundation.CFArrayGetCount.restype = CFIndex
CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex]
CoreFoundation.CFArrayGetValueAtIndex.argtypes = [
CFArrayRef,
CFIndex
]
CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p
CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll(
CoreFoundation, "kCFAllocatorDefault"
)
CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(
CoreFoundation, "kCFTypeArrayCallBacks"
CoreFoundation, 'kCFAllocatorDefault'
)
CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks')
CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll(
CoreFoundation, "kCFTypeDictionaryKeyCallBacks"
CoreFoundation, 'kCFTypeDictionaryKeyCallBacks'
)
CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll(
CoreFoundation, "kCFTypeDictionaryValueCallBacks"
CoreFoundation, 'kCFTypeDictionaryValueCallBacks'
)
CoreFoundation.CFTypeRef = CFTypeRef
@@ -391,7 +494,7 @@ try:
CoreFoundation.CFDictionaryRef = CFDictionaryRef
except (AttributeError):
raise ImportError("Error initializing ctypes")
raise ImportError('Error initializing ctypes')
class CFConst(object):
@@ -399,7 +502,6 @@ class CFConst(object):
A class object that acts as essentially a namespace for CoreFoundation
constants.
"""
kCFStringEncodingUTF8 = CFStringEncoding(0x08000100)
@@ -407,7 +509,6 @@ class SecurityConst(object):
"""
A class object that acts as essentially a namespace for Security constants.
"""
kSSLSessionOptionBreakOnServerAuth = 0
kSSLProtocol2 = 1

52
lib/urllib3/contrib/_securetransport/low_level.py Executable file → Normal file
View File

@@ -66,18 +66,22 @@ def _cf_string_to_unicode(value):
value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p))
string = CoreFoundation.CFStringGetCStringPtr(
value_as_void_p, CFConst.kCFStringEncodingUTF8
value_as_void_p,
CFConst.kCFStringEncodingUTF8
)
if string is None:
buffer = ctypes.create_string_buffer(1024)
result = CoreFoundation.CFStringGetCString(
value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8
value_as_void_p,
buffer,
1024,
CFConst.kCFStringEncodingUTF8
)
if not result:
raise OSError("Error copying C string from CFStringRef")
raise OSError('Error copying C string from CFStringRef')
string = buffer.value
if string is not None:
string = string.decode("utf-8")
string = string.decode('utf-8')
return string
@@ -93,8 +97,8 @@ def _assert_no_error(error, exception_class=None):
output = _cf_string_to_unicode(cf_error_string)
CoreFoundation.CFRelease(cf_error_string)
if output is None or output == u"":
output = u"OSStatus %s" % error
if output is None or output == u'':
output = u'OSStatus %s' % error
if exception_class is None:
exception_class = ssl.SSLError
@@ -111,7 +115,8 @@ def _cert_array_from_pem(pem_bundle):
pem_bundle = pem_bundle.replace(b"\r\n", b"\n")
der_certs = [
base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle)
base64.b64decode(match.group(1))
for match in _PEM_CERTS_RE.finditer(pem_bundle)
]
if not der_certs:
raise ssl.SSLError("No root certificates specified")
@@ -119,7 +124,7 @@ def _cert_array_from_pem(pem_bundle):
cert_array = CoreFoundation.CFArrayCreateMutable(
CoreFoundation.kCFAllocatorDefault,
0,
ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks)
)
if not cert_array:
raise ssl.SSLError("Unable to allocate memory!")
@@ -181,16 +186,21 @@ def _temporary_keychain():
# some random bytes to password-protect the keychain we're creating, so we
# ask for 40 random bytes.
random_bytes = os.urandom(40)
filename = base64.b16encode(random_bytes[:8]).decode("utf-8")
filename = base64.b16encode(random_bytes[:8]).decode('utf-8')
password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8
tempdirectory = tempfile.mkdtemp()
keychain_path = os.path.join(tempdirectory, filename).encode("utf-8")
keychain_path = os.path.join(tempdirectory, filename).encode('utf-8')
# We now want to create the keychain itself.
keychain = Security.SecKeychainRef()
status = Security.SecKeychainCreate(
keychain_path, len(password), password, False, None, ctypes.byref(keychain)
keychain_path,
len(password),
password,
False,
None,
ctypes.byref(keychain)
)
_assert_no_error(status)
@@ -209,12 +219,14 @@ def _load_items_from_file(keychain, path):
identities = []
result_array = None
with open(path, "rb") as f:
with open(path, 'rb') as f:
raw_filedata = f.read()
try:
filedata = CoreFoundation.CFDataCreate(
CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata)
CoreFoundation.kCFAllocatorDefault,
raw_filedata,
len(raw_filedata)
)
result_array = CoreFoundation.CFArrayRef()
result = Security.SecItemImport(
@@ -225,7 +237,7 @@ def _load_items_from_file(keychain, path):
0, # import flags
None, # key params, can include passphrase in the future
keychain, # The keychain to insert into
ctypes.byref(result_array), # Results
ctypes.byref(result_array) # Results
)
_assert_no_error(result)
@@ -235,7 +247,9 @@ def _load_items_from_file(keychain, path):
# keychain already has them!
result_count = CoreFoundation.CFArrayGetCount(result_array)
for index in range(result_count):
item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index)
item = CoreFoundation.CFArrayGetValueAtIndex(
result_array, index
)
item = ctypes.cast(item, CoreFoundation.CFTypeRef)
if _is_cert(item):
@@ -293,7 +307,9 @@ def _load_client_cert_chain(keychain, *paths):
try:
for file_path in paths:
new_identities, new_certs = _load_items_from_file(keychain, file_path)
new_identities, new_certs = _load_items_from_file(
keychain, file_path
)
identities.extend(new_identities)
certificates.extend(new_certs)
@@ -302,7 +318,9 @@ def _load_client_cert_chain(keychain, *paths):
if not identities:
new_identity = Security.SecIdentityRef()
status = Security.SecIdentityCreateWithCertificate(
keychain, certificates[0], ctypes.byref(new_identity)
keychain,
certificates[0],
ctypes.byref(new_identity)
)
_assert_no_error(status)
identities.append(new_identity)

118
lib/urllib3/contrib/appengine.py Executable file → Normal file
View File

@@ -50,7 +50,7 @@ from ..exceptions import (
MaxRetryError,
ProtocolError,
TimeoutError,
SSLError,
SSLError
)
from ..request import RequestMethods
@@ -96,31 +96,23 @@ class AppEngineManager(RequestMethods):
Beyond those cases, it will raise normal urllib3 errors.
"""
def __init__(
self,
headers=None,
retries=None,
validate_certificate=True,
urlfetch_retries=True,
):
def __init__(self, headers=None, retries=None, validate_certificate=True,
urlfetch_retries=True):
if not urlfetch:
raise AppEnginePlatformError(
"URLFetch is not available in this environment."
)
"URLFetch is not available in this environment.")
if is_prod_appengine_mvms():
raise AppEnginePlatformError(
"Use normal urllib3.PoolManager instead of AppEngineManager"
"on Managed VMs, as using URLFetch is not necessary in "
"this environment."
)
"this environment.")
warnings.warn(
"urllib3 is using URLFetch on Google App Engine sandbox instead "
"of sockets. To use sockets directly instead of URLFetch see "
"https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.",
AppEnginePlatformWarning,
)
AppEnginePlatformWarning)
RequestMethods.__init__(self, headers)
self.validate_certificate = validate_certificate
@@ -135,22 +127,17 @@ class AppEngineManager(RequestMethods):
# Return False to re-raise any potential exceptions
return False
def urlopen(
self,
method,
url,
body=None,
headers=None,
retries=None,
redirect=True,
timeout=Timeout.DEFAULT_TIMEOUT,
**response_kw
):
def urlopen(self, method, url, body=None, headers=None,
retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT,
**response_kw):
retries = self._get_retries(retries, redirect)
try:
follow_redirects = redirect and retries.redirect != 0 and retries.total
follow_redirects = (
redirect and
retries.redirect != 0 and
retries.total)
response = urlfetch.fetch(
url,
payload=body,
@@ -165,52 +152,44 @@ class AppEngineManager(RequestMethods):
raise TimeoutError(self, e)
except urlfetch.InvalidURLError as e:
if "too large" in str(e):
if 'too large' in str(e):
raise AppEnginePlatformError(
"URLFetch request too large, URLFetch only "
"supports requests up to 10mb in size.",
e,
)
"supports requests up to 10mb in size.", e)
raise ProtocolError(e)
except urlfetch.DownloadError as e:
if "Too many redirects" in str(e):
if 'Too many redirects' in str(e):
raise MaxRetryError(self, url, reason=e)
raise ProtocolError(e)
except urlfetch.ResponseTooLargeError as e:
raise AppEnginePlatformError(
"URLFetch response too large, URLFetch only supports"
"responses up to 32mb in size.",
e,
)
"responses up to 32mb in size.", e)
except urlfetch.SSLCertificateError as e:
raise SSLError(e)
except urlfetch.InvalidMethodError as e:
raise AppEnginePlatformError(
"URLFetch does not support method: %s" % method, e
)
"URLFetch does not support method: %s" % method, e)
http_response = self._urlfetch_response_to_http_response(
response, retries=retries, **response_kw
)
response, retries=retries, **response_kw)
# Handle redirect?
redirect_location = redirect and http_response.get_redirect_location()
if redirect_location:
# Check for redirect response
if self.urlfetch_retries and retries.raise_on_redirect:
if (self.urlfetch_retries and retries.raise_on_redirect):
raise MaxRetryError(self, url, "too many redirects")
else:
if http_response.status == 303:
method = "GET"
method = 'GET'
try:
retries = retries.increment(
method, url, response=http_response, _pool=self
)
retries = retries.increment(method, url, response=http_response, _pool=self)
except MaxRetryError:
if retries.raise_on_redirect:
raise MaxRetryError(self, url, "too many redirects")
@@ -220,32 +199,22 @@ class AppEngineManager(RequestMethods):
log.debug("Redirecting %s -> %s", url, redirect_location)
redirect_url = urljoin(url, redirect_location)
return self.urlopen(
method,
redirect_url,
body,
headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw
)
method, redirect_url, body, headers,
retries=retries, redirect=redirect,
timeout=timeout, **response_kw)
# Check if we should retry the HTTP response.
has_retry_after = bool(http_response.getheader("Retry-After"))
has_retry_after = bool(http_response.getheader('Retry-After'))
if retries.is_retry(method, http_response.status, has_retry_after):
retries = retries.increment(method, url, response=http_response, _pool=self)
retries = retries.increment(
method, url, response=http_response, _pool=self)
log.debug("Retry: %s", url)
retries.sleep(http_response)
return self.urlopen(
method,
url,
body=body,
headers=headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw
)
method, url,
body=body, headers=headers,
retries=retries, redirect=redirect,
timeout=timeout, **response_kw)
return http_response
@@ -254,18 +223,18 @@ class AppEngineManager(RequestMethods):
if is_prod_appengine():
# Production GAE handles deflate encoding automatically, but does
# not remove the encoding header.
content_encoding = urlfetch_resp.headers.get("content-encoding")
content_encoding = urlfetch_resp.headers.get('content-encoding')
if content_encoding == "deflate":
del urlfetch_resp.headers["content-encoding"]
if content_encoding == 'deflate':
del urlfetch_resp.headers['content-encoding']
transfer_encoding = urlfetch_resp.headers.get("transfer-encoding")
transfer_encoding = urlfetch_resp.headers.get('transfer-encoding')
# We have a full response's content,
# so let's make sure we don't report ourselves as chunked data.
if transfer_encoding == "chunked":
if transfer_encoding == 'chunked':
encodings = transfer_encoding.split(",")
encodings.remove("chunked")
urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings)
encodings.remove('chunked')
urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings)
original_response = HTTPResponse(
# In order for decoding to work, we must present the content as
@@ -293,21 +262,20 @@ class AppEngineManager(RequestMethods):
warnings.warn(
"URLFetch does not support granular timeout settings, "
"reverting to total or default URLFetch timeout.",
AppEnginePlatformWarning,
)
AppEnginePlatformWarning)
return timeout.total
return timeout
def _get_retries(self, retries, redirect):
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
retries = Retry.from_int(
retries, redirect=redirect, default=self.retries)
if retries.connect or retries.read or retries.redirect:
warnings.warn(
"URLFetch only supports total retries and does not "
"recognize connect, read, or redirect retry parameters.",
AppEnginePlatformWarning,
)
AppEnginePlatformWarning)
return retries

98
lib/urllib3/contrib/ntlmpool.py Executable file → Normal file
View File

@@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
Implements an NTLM authentication version of an urllib3 connection pool
"""
scheme = "https"
scheme = 'https'
def __init__(self, user, pw, authurl, *args, **kwargs):
"""
@@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
super(NTLMConnectionPool, self).__init__(*args, **kwargs)
self.authurl = authurl
self.rawuser = user
user_parts = user.split("\\", 1)
user_parts = user.split('\\', 1)
self.domain = user_parts[0].upper()
self.user = user_parts[1]
self.pw = pw
@@ -40,84 +40,72 @@ class NTLMConnectionPool(HTTPSConnectionPool):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug(
"Starting NTLM HTTPS connection no. %d: https://%s%s",
self.num_connections,
self.host,
self.authurl,
)
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
self.num_connections, self.host, self.authurl)
headers = {"Connection": "Keep-Alive"}
req_header = "Authorization"
resp_header = "www-authenticate"
headers = {'Connection': 'Keep-Alive'}
req_header = 'Authorization'
resp_header = 'www-authenticate'
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE(
self.rawuser
)
log.debug("Request headers: %s", headers)
conn.request("GET", self.authurl, None, headers)
headers[req_header] = (
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug("Response status: %s %s", res.status, res.reason)
log.debug("Response headers: %s", reshdr)
log.debug("Response data: %s [...]", res.read(100))
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', reshdr)
log.debug('Response data: %s [...]', res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(", ")
auth_header_values = reshdr[resp_header].split(', ')
auth_header_value = None
for s in auth_header_values:
if s[:5] == "NTLM ":
if s[:5] == 'NTLM ':
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception(
"Unexpected %s response header: %s" % (resp_header, reshdr[resp_header])
)
raise Exception('Unexpected %s response header: %s' %
(resp_header, reshdr[resp_header]))
# Send authentication message
ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
auth_header_value
)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags
)
headers[req_header] = "NTLM %s" % auth_msg
log.debug("Request headers: %s", headers)
conn.request("GET", self.authurl, None, headers)
ServerChallenge, NegotiateFlags = \
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge,
self.user,
self.domain,
self.pw,
NegotiateFlags)
headers[req_header] = 'NTLM %s' % auth_msg
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
log.debug("Response status: %s %s", res.status, res.reason)
log.debug("Response headers: %s", dict(res.getheaders()))
log.debug("Response data: %s [...]", res.read()[:100])
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception(
"Server rejected request: wrong " "username or password"
)
raise Exception("Wrong server response: %s %s" % (res.status, res.reason))
raise Exception('Server rejected request: wrong '
'username or password')
raise Exception('Wrong server response: %s %s' %
(res.status, res.reason))
res.fp = None
log.debug("Connection established")
log.debug('Connection established')
return conn
def urlopen(
self,
method,
url,
body=None,
headers=None,
retries=3,
redirect=True,
assert_same_host=True,
):
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True):
if headers is None:
headers = {}
headers["Connection"] = "Keep-Alive"
return super(NTLMConnectionPool, self).urlopen(
method, url, body, headers, retries, redirect, assert_same_host
)
headers['Connection'] = 'Keep-Alive'
return super(NTLMConnectionPool, self).urlopen(method, url, body,
headers, retries,
redirect,
assert_same_host)

129
lib/urllib3/contrib/pyopenssl.py Executable file → Normal file
View File

@@ -47,7 +47,6 @@ import OpenSSL.SSL
from cryptography import x509
from cryptography.hazmat.backends.openssl import backend as openssl_backend
from cryptography.hazmat.backends.openssl.x509 import _Certificate
try:
from cryptography.x509 import UnsupportedExtension
except ImportError:
@@ -55,7 +54,6 @@ except ImportError:
class UnsupportedExtension(Exception):
pass
from socket import timeout, error as SocketError
from io import BytesIO
@@ -73,7 +71,7 @@ import sys
from .. import util
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
# SNI always works.
HAS_SNI = True
@@ -84,23 +82,25 @@ _openssl_versions = {
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}
if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"):
if hasattr(ssl, 'PROTOCOL_SSLv3') and hasattr(OpenSSL.SSL, 'SSLv3_METHOD'):
_openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
_openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
_stdlib_to_openssl_verify = {
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
ssl.CERT_REQUIRED:
OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items())
_openssl_to_stdlib_verify = dict(
(v, k) for k, v in _stdlib_to_openssl_verify.items()
)
# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384
@@ -113,7 +113,7 @@ log = logging.getLogger(__name__)
def inject_into_urllib3():
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
_validate_dependencies_met()
@@ -126,7 +126,7 @@ def inject_into_urllib3():
def extract_from_urllib3():
"Undo monkey-patching by :func:`inject_into_urllib3`."
'Undo monkey-patching by :func:`inject_into_urllib3`.'
util.SSLContext = orig_util_SSLContext
util.ssl_.SSLContext = orig_util_SSLContext
@@ -143,23 +143,17 @@ def _validate_dependencies_met():
"""
# Method added in `cryptography==1.1`; not available in older versions
from cryptography.x509.extensions import Extensions
if getattr(Extensions, "get_extension_for_class", None) is None:
raise ImportError(
"'cryptography' module missing required functionality. "
"Try upgrading to v1.3.4 or newer."
)
raise ImportError("'cryptography' module missing required functionality. "
"Try upgrading to v1.3.4 or newer.")
# pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
# attribute is only present on those versions.
from OpenSSL.crypto import X509
x509 = X509()
if getattr(x509, "_x509", None) is None:
raise ImportError(
"'pyOpenSSL' module missing required functionality. "
"Try upgrading to v0.14 or newer."
)
raise ImportError("'pyOpenSSL' module missing required functionality. "
"Try upgrading to v0.14 or newer.")
def _dnsname_to_stdlib(name):
@@ -175,7 +169,6 @@ def _dnsname_to_stdlib(name):
If the name cannot be idna-encoded then we return None signalling that
the name given should be skipped.
"""
def idna_encode(name):
"""
Borrowed wholesale from the Python Cryptography Project. It turns out
@@ -185,23 +178,23 @@ def _dnsname_to_stdlib(name):
import idna
try:
for prefix in [u"*.", u"."]:
for prefix in [u'*.', u'.']:
if name.startswith(prefix):
name = name[len(prefix) :]
return prefix.encode("ascii") + idna.encode(name)
name = name[len(prefix):]
return prefix.encode('ascii') + idna.encode(name)
return idna.encode(name)
except idna.core.IDNAError:
return None
# Don't send IPv6 addresses through the IDNA encoder.
if ":" in name:
if ':' in name:
return name
name = idna_encode(name)
if name is None:
return None
elif sys.version_info >= (3, 0):
name = name.decode("utf-8")
name = name.decode('utf-8')
return name
@@ -220,16 +213,14 @@ def get_subj_alt_name(peer_cert):
# We want to find the SAN extension. Ask Cryptography to locate it (it's
# faster than looping in Python)
try:
ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
ext = cert.extensions.get_extension_for_class(
x509.SubjectAlternativeName
).value
except x509.ExtensionNotFound:
# No such extension, return the empty list.
return []
except (
x509.DuplicateExtension,
UnsupportedExtension,
x509.UnsupportedGeneralNameType,
UnicodeError,
) as e:
except (x509.DuplicateExtension, UnsupportedExtension,
x509.UnsupportedGeneralNameType, UnicodeError) as e:
# A problem has been found with the quality of the certificate. Assume
# no SAN field is present.
log.warning(
@@ -248,23 +239,23 @@ def get_subj_alt_name(peer_cert):
# does with certificates, and so we need to attempt to do the same.
# We also want to skip over names which cannot be idna encoded.
names = [
("DNS", name)
for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
if name is not None
]
names.extend(
("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
('IP Address', str(name))
for name in ext.get_values_for_type(x509.IPAddress)
)
return names
class WrappedSocket(object):
"""API-compatibility wrapper for Python OpenSSL's Connection-class.
'''API-compatibility wrapper for Python OpenSSL's Connection-class.
Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
collector of pypy.
"""
'''
def __init__(self, connection, socket, suppress_ragged_eofs=True):
self.connection = connection
@@ -287,18 +278,18 @@ class WrappedSocket(object):
try:
data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b""
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
return b''
else:
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b""
return b''
else:
raise
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise timeout("The read operation timed out")
raise timeout('The read operation timed out')
else:
return self.recv(*args, **kwargs)
@@ -312,7 +303,7 @@ class WrappedSocket(object):
try:
return self.connection.recv_into(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
return 0
else:
raise SocketError(str(e))
@@ -323,7 +314,7 @@ class WrappedSocket(object):
raise
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise timeout("The read operation timed out")
raise timeout('The read operation timed out')
else:
return self.recv_into(*args, **kwargs)
@@ -348,9 +339,7 @@ class WrappedSocket(object):
def sendall(self, data):
total_sent = 0
while total_sent < len(data):
sent = self._send_until_done(
data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
)
sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
total_sent += sent
def shutdown(self):
@@ -374,11 +363,15 @@ class WrappedSocket(object):
return x509
if binary_form:
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509)
return OpenSSL.crypto.dump_certificate(
OpenSSL.crypto.FILETYPE_ASN1,
x509)
return {
"subject": ((("commonName", x509.get_subject().CN),),),
"subjectAltName": get_subj_alt_name(x509),
'subject': (
(('commonName', x509.get_subject().CN),),
),
'subjectAltName': get_subj_alt_name(x509)
}
def version(self):
@@ -395,12 +388,9 @@ class WrappedSocket(object):
if _fileobject: # Platform-specific: Python 2
def makefile(self, mode, bufsize=-1):
self._makefile_refs += 1
return _fileobject(self, mode, bufsize, close=True)
else: # Platform-specific: Python 3
makefile = backport_makefile
@@ -413,7 +403,6 @@ class PyOpenSSLContext(object):
for translating the interface of the standard library ``SSLContext`` object
to calls into PyOpenSSL.
"""
def __init__(self, protocol):
self.protocol = _openssl_versions[protocol]
self._ctx = OpenSSL.SSL.Context(self.protocol)
@@ -435,21 +424,24 @@ class PyOpenSSLContext(object):
@verify_mode.setter
def verify_mode(self, value):
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
self._ctx.set_verify(
_stdlib_to_openssl_verify[value],
_verify_callback
)
def set_default_verify_paths(self):
self._ctx.set_default_verify_paths()
def set_ciphers(self, ciphers):
if isinstance(ciphers, six.text_type):
ciphers = ciphers.encode("utf-8")
ciphers = ciphers.encode('utf-8')
self._ctx.set_cipher_list(ciphers)
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
if cafile is not None:
cafile = cafile.encode("utf-8")
cafile = cafile.encode('utf-8')
if capath is not None:
capath = capath.encode("utf-8")
capath = capath.encode('utf-8')
self._ctx.load_verify_locations(cafile, capath)
if cadata is not None:
self._ctx.load_verify_locations(BytesIO(cadata))
@@ -458,22 +450,17 @@ class PyOpenSSLContext(object):
self._ctx.use_certificate_chain_file(certfile)
if password is not None:
if not isinstance(password, six.binary_type):
password = password.encode("utf-8")
password = password.encode('utf-8')
self._ctx.set_passwd_cb(lambda *_: password)
self._ctx.use_privatekey_file(keyfile or certfile)
def wrap_socket(
self,
sock,
server_side=False,
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
server_hostname=None,
):
def wrap_socket(self, sock, server_side=False,
do_handshake_on_connect=True, suppress_ragged_eofs=True,
server_hostname=None):
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
server_hostname = server_hostname.encode("utf-8")
server_hostname = server_hostname.encode('utf-8')
if server_hostname is not None:
cnx.set_tlsext_host_name(server_hostname)
@@ -485,10 +472,10 @@ class PyOpenSSLContext(object):
cnx.do_handshake()
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(sock, sock.gettimeout()):
raise timeout("select timed out")
raise timeout('select timed out')
continue
except OpenSSL.SSL.Error as e:
raise ssl.SSLError("bad handshake: %r" % e)
raise ssl.SSLError('bad handshake: %r' % e)
break
return WrappedSocket(cnx, sock)

157
lib/urllib3/contrib/securetransport.py Executable file → Normal file
View File

@@ -62,12 +62,12 @@ import threading
import weakref
from .. import util
from ._securetransport.bindings import Security, SecurityConst, CoreFoundation
from ._securetransport.bindings import (
Security, SecurityConst, CoreFoundation
)
from ._securetransport.low_level import (
_assert_no_error,
_cert_array_from_pem,
_temporary_keychain,
_load_client_cert_chain,
_assert_no_error, _cert_array_from_pem, _temporary_keychain,
_load_client_cert_chain
)
try: # Platform-specific: Python 2
@@ -76,7 +76,7 @@ except ImportError: # Platform-specific: Python 3
_fileobject = None
from ..packages.backports.makefile import backport_makefile
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
# SNI always works
HAS_SNI = True
@@ -147,36 +147,28 @@ CIPHER_SUITES = [
# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version.
# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+
_protocol_to_min_max = {
util.PROTOCOL_TLS: (
SecurityConst.kTLSProtocol1,
SecurityConst.kTLSProtocolMaxSupported,
)
util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocolMaxSupported),
}
if hasattr(ssl, "PROTOCOL_SSLv2"):
_protocol_to_min_max[ssl.PROTOCOL_SSLv2] = (
SecurityConst.kSSLProtocol2,
SecurityConst.kSSLProtocol2,
SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2
)
if hasattr(ssl, "PROTOCOL_SSLv3"):
_protocol_to_min_max[ssl.PROTOCOL_SSLv3] = (
SecurityConst.kSSLProtocol3,
SecurityConst.kSSLProtocol3,
SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3
)
if hasattr(ssl, "PROTOCOL_TLSv1"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1] = (
SecurityConst.kTLSProtocol1,
SecurityConst.kTLSProtocol1,
SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1
)
if hasattr(ssl, "PROTOCOL_TLSv1_1"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = (
SecurityConst.kTLSProtocol11,
SecurityConst.kTLSProtocol11,
SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11
)
if hasattr(ssl, "PROTOCOL_TLSv1_2"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = (
SecurityConst.kTLSProtocol12,
SecurityConst.kTLSProtocol12,
SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12
)
@@ -226,7 +218,7 @@ def _read_callback(connection_id, data_buffer, data_length_pointer):
while read_count < requested_length:
if timeout is None or timeout >= 0:
if not util.wait_for_read(base_socket, timeout):
raise socket.error(errno.EAGAIN, "timed out")
raise socket.error(errno.EAGAIN, 'timed out')
remaining = requested_length - read_count
buffer = (ctypes.c_char * remaining).from_address(
@@ -282,7 +274,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer):
while sent < bytes_to_write:
if timeout is None or timeout >= 0:
if not util.wait_for_write(base_socket, timeout):
raise socket.error(errno.EAGAIN, "timed out")
raise socket.error(errno.EAGAIN, 'timed out')
chunk_sent = base_socket.send(data)
sent += chunk_sent
@@ -324,7 +316,6 @@ class WrappedSocket(object):
Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage
collector of PyPy.
"""
def __init__(self, socket):
self.socket = socket
self.context = None
@@ -389,7 +380,7 @@ class WrappedSocket(object):
# We want data in memory, so load it up.
if os.path.isfile(trust_bundle):
with open(trust_bundle, "rb") as f:
with open(trust_bundle, 'rb') as f:
trust_bundle = f.read()
cert_array = None
@@ -403,7 +394,9 @@ class WrappedSocket(object):
# created for this connection, shove our CAs into it, tell ST to
# ignore everything else it knows, and then ask if it can build a
# chain. This is a buuuunch of code.
result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
result = Security.SSLCopyPeerTrust(
self.context, ctypes.byref(trust)
)
_assert_no_error(result)
if not trust:
raise ssl.SSLError("Failed to copy trust reference")
@@ -415,7 +408,9 @@ class WrappedSocket(object):
_assert_no_error(result)
trust_result = Security.SecTrustResultType()
result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result))
result = Security.SecTrustEvaluate(
trust, ctypes.byref(trust_result)
)
_assert_no_error(result)
finally:
if trust:
@@ -427,24 +422,23 @@ class WrappedSocket(object):
# Ok, now we can look at what the result was.
successes = (
SecurityConst.kSecTrustResultUnspecified,
SecurityConst.kSecTrustResultProceed,
SecurityConst.kSecTrustResultProceed
)
if trust_result.value not in successes:
raise ssl.SSLError(
"certificate verify failed, error code: %d" % trust_result.value
"certificate verify failed, error code: %d" %
trust_result.value
)
def handshake(
self,
server_hostname,
verify,
trust_bundle,
min_version,
max_version,
client_cert,
client_key,
client_key_passphrase,
):
def handshake(self,
server_hostname,
verify,
trust_bundle,
min_version,
max_version,
client_cert,
client_key,
client_key_passphrase):
"""
Actually performs the TLS handshake. This is run automatically by
wrapped socket, and shouldn't be needed in user code.
@@ -474,7 +468,7 @@ class WrappedSocket(object):
# If we have a server hostname, we should set that too.
if server_hostname:
if not isinstance(server_hostname, bytes):
server_hostname = server_hostname.encode("utf-8")
server_hostname = server_hostname.encode('utf-8')
result = Security.SSLSetPeerDomainName(
self.context, server_hostname, len(server_hostname)
@@ -494,9 +488,7 @@ class WrappedSocket(object):
# was added in macOS 10.13 along with kTLSProtocol13.
result = Security.SSLSetProtocolVersionMax(self.context, max_version)
if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported:
result = Security.SSLSetProtocolVersionMax(
self.context, SecurityConst.kTLSProtocol12
)
result = Security.SSLSetProtocolVersionMax(self.context, SecurityConst.kTLSProtocol12)
_assert_no_error(result)
# If there's a trust DB, we need to use it. We do that by telling
@@ -505,7 +497,9 @@ class WrappedSocket(object):
# authing in that case.
if not verify or trust_bundle is not None:
result = Security.SSLSetSessionOption(
self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True
self.context,
SecurityConst.kSSLSessionOptionBreakOnServerAuth,
True
)
_assert_no_error(result)
@@ -515,7 +509,9 @@ class WrappedSocket(object):
self._client_cert_chain = _load_client_cert_chain(
self._keychain, client_cert, client_key
)
result = Security.SSLSetCertificate(self.context, self._client_cert_chain)
result = Security.SSLSetCertificate(
self.context, self._client_cert_chain
)
_assert_no_error(result)
while True:
@@ -566,7 +562,7 @@ class WrappedSocket(object):
# There are some result codes that we want to treat as "not always
# errors". Specifically, those are errSSLWouldBlock,
# errSSLClosedGraceful, and errSSLClosedNoNotify.
if result == SecurityConst.errSSLWouldBlock:
if (result == SecurityConst.errSSLWouldBlock):
# If we didn't process any bytes, then this was just a time out.
# However, we can get errSSLWouldBlock in situations when we *did*
# read some data, and in those cases we should just read "short"
@@ -574,10 +570,7 @@ class WrappedSocket(object):
if processed_bytes.value == 0:
# Timed out, no data read.
raise socket.timeout("recv timed out")
elif result in (
SecurityConst.errSSLClosedGraceful,
SecurityConst.errSSLClosedNoNotify,
):
elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify):
# The remote peer has closed this connection. We should do so as
# well. Note that we don't actually return here because in
# principle this could actually be fired along with return data.
@@ -616,7 +609,7 @@ class WrappedSocket(object):
def sendall(self, data):
total_sent = 0
while total_sent < len(data):
sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE])
sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
total_sent += sent
def shutdown(self):
@@ -663,14 +656,18 @@ class WrappedSocket(object):
# instead to just flag to urllib3 that it shouldn't do its own hostname
# validation when using SecureTransport.
if not binary_form:
raise ValueError("SecureTransport only supports dumping binary certs")
raise ValueError(
"SecureTransport only supports dumping binary certs"
)
trust = Security.SecTrustRef()
certdata = None
der_bytes = None
try:
# Grab the trust store.
result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
result = Security.SSLCopyPeerTrust(
self.context, ctypes.byref(trust)
)
_assert_no_error(result)
if not trust:
# Probably we haven't done the handshake yet. No biggie.
@@ -702,24 +699,22 @@ class WrappedSocket(object):
def version(self):
protocol = Security.SSLProtocol()
result = Security.SSLGetNegotiatedProtocolVersion(
self.context, ctypes.byref(protocol)
)
result = Security.SSLGetNegotiatedProtocolVersion(self.context, ctypes.byref(protocol))
_assert_no_error(result)
if protocol.value == SecurityConst.kTLSProtocol13:
return "TLSv1.3"
return 'TLSv1.3'
elif protocol.value == SecurityConst.kTLSProtocol12:
return "TLSv1.2"
return 'TLSv1.2'
elif protocol.value == SecurityConst.kTLSProtocol11:
return "TLSv1.1"
return 'TLSv1.1'
elif protocol.value == SecurityConst.kTLSProtocol1:
return "TLSv1"
return 'TLSv1'
elif protocol.value == SecurityConst.kSSLProtocol3:
return "SSLv3"
return 'SSLv3'
elif protocol.value == SecurityConst.kSSLProtocol2:
return "SSLv2"
return 'SSLv2'
else:
raise ssl.SSLError("Unknown TLS version: %r" % protocol)
raise ssl.SSLError('Unknown TLS version: %r' % protocol)
def _reuse(self):
self._makefile_refs += 1
@@ -732,21 +727,16 @@ class WrappedSocket(object):
if _fileobject: # Platform-specific: Python 2
def makefile(self, mode, bufsize=-1):
self._makefile_refs += 1
return _fileobject(self, mode, bufsize, close=True)
else: # Platform-specific: Python 3
def makefile(self, mode="r", buffering=None, *args, **kwargs):
# We disable buffering with SecureTransport because it conflicts with
# the buffering that ST does internally (see issue #1153 for more).
buffering = 0
return backport_makefile(self, mode, buffering, *args, **kwargs)
WrappedSocket.makefile = makefile
@@ -756,7 +746,6 @@ class SecureTransportContext(object):
interface of the standard library ``SSLContext`` object to calls into
SecureTransport.
"""
def __init__(self, protocol):
self._min_version, self._max_version = _protocol_to_min_max[protocol]
self._options = 0
@@ -823,12 +812,16 @@ class SecureTransportContext(object):
def set_ciphers(self, ciphers):
# For now, we just require the default cipher string.
if ciphers != util.ssl_.DEFAULT_CIPHERS:
raise ValueError("SecureTransport doesn't support custom cipher strings")
raise ValueError(
"SecureTransport doesn't support custom cipher strings"
)
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
# OK, we only really support cadata and cafile.
if capath is not None:
raise ValueError("SecureTransport does not support cert directories")
raise ValueError(
"SecureTransport does not support cert directories"
)
self._trust_bundle = cafile or cadata
@@ -837,14 +830,9 @@ class SecureTransportContext(object):
self._client_key = keyfile
self._client_cert_passphrase = password
def wrap_socket(
self,
sock,
server_side=False,
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
server_hostname=None,
):
def wrap_socket(self, sock, server_side=False,
do_handshake_on_connect=True, suppress_ragged_eofs=True,
server_hostname=None):
# So, what do we do here? Firstly, we assert some properties. This is a
# stripped down shim, so there is some functionality we don't support.
# See PEP 543 for the real deal.
@@ -858,13 +846,8 @@ class SecureTransportContext(object):
# Now we can handshake
wrapped_socket.handshake(
server_hostname,
self._verify,
self._trust_bundle,
self._min_version,
self._max_version,
self._client_cert,
self._client_key,
self._client_key_passphrase,
server_hostname, self._verify, self._trust_bundle,
self._min_version, self._max_version, self._client_cert,
self._client_key, self._client_key_passphrase
)
return wrapped_socket

101
lib/urllib3/contrib/socks.py Executable file → Normal file
View File

@@ -42,20 +42,23 @@ except ImportError:
import warnings
from ..exceptions import DependencyWarning
warnings.warn(
(
"SOCKS support in urllib3 requires the installation of optional "
"dependencies: specifically, PySocks. For more information, see "
"https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies"
warnings.warn((
'SOCKS support in urllib3 requires the installation of optional '
'dependencies: specifically, PySocks. For more information, see '
'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies'
),
DependencyWarning,
DependencyWarning
)
raise
from socket import error as SocketError, timeout as SocketTimeout
from ..connection import HTTPConnection, HTTPSConnection
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from ..connection import (
HTTPConnection, HTTPSConnection
)
from ..connectionpool import (
HTTPConnectionPool, HTTPSConnectionPool
)
from ..exceptions import ConnectTimeoutError, NewConnectionError
from ..poolmanager import PoolManager
from ..util.url import parse_url
@@ -70,9 +73,8 @@ class SOCKSConnection(HTTPConnection):
"""
A plain-text HTTP connection that connects via a SOCKS proxy.
"""
def __init__(self, *args, **kwargs):
self._socks_options = kwargs.pop("_socks_options")
self._socks_options = kwargs.pop('_socks_options')
super(SOCKSConnection, self).__init__(*args, **kwargs)
def _new_conn(self):
@@ -81,30 +83,28 @@ class SOCKSConnection(HTTPConnection):
"""
extra_kw = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
extra_kw['source_address'] = self.source_address
if self.socket_options:
extra_kw["socket_options"] = self.socket_options
extra_kw['socket_options'] = self.socket_options
try:
conn = socks.create_connection(
(self.host, self.port),
proxy_type=self._socks_options["socks_version"],
proxy_addr=self._socks_options["proxy_host"],
proxy_port=self._socks_options["proxy_port"],
proxy_username=self._socks_options["username"],
proxy_password=self._socks_options["password"],
proxy_rdns=self._socks_options["rdns"],
proxy_type=self._socks_options['socks_version'],
proxy_addr=self._socks_options['proxy_host'],
proxy_port=self._socks_options['proxy_port'],
proxy_username=self._socks_options['username'],
proxy_password=self._socks_options['password'],
proxy_rdns=self._socks_options['rdns'],
timeout=self.timeout,
**extra_kw
)
except SocketTimeout:
raise ConnectTimeoutError(
self,
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
self, "Connection to %s timed out. (connect timeout=%s)" %
(self.host, self.timeout))
except socks.ProxyError as e:
# This is fragile as hell, but it seems to be the only way to raise
@@ -114,22 +114,23 @@ class SOCKSConnection(HTTPConnection):
if isinstance(error, SocketTimeout):
raise ConnectTimeoutError(
self,
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
"Connection to %s timed out. (connect timeout=%s)" %
(self.host, self.timeout)
)
else:
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % error
self,
"Failed to establish a new connection: %s" % error
)
else:
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e
self,
"Failed to establish a new connection: %s" % e
)
except SocketError as e: # Defensive: PySocks should catch all these.
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e
)
self, "Failed to establish a new connection: %s" % e)
return conn
@@ -155,53 +156,47 @@ class SOCKSProxyManager(PoolManager):
A version of the urllib3 ProxyManager that routes connections via the
defined SOCKS proxy.
"""
pool_classes_by_scheme = {
"http": SOCKSHTTPConnectionPool,
"https": SOCKSHTTPSConnectionPool,
'http': SOCKSHTTPConnectionPool,
'https': SOCKSHTTPSConnectionPool,
}
def __init__(
self,
proxy_url,
username=None,
password=None,
num_pools=10,
headers=None,
**connection_pool_kw
):
def __init__(self, proxy_url, username=None, password=None,
num_pools=10, headers=None, **connection_pool_kw):
parsed = parse_url(proxy_url)
if username is None and password is None and parsed.auth is not None:
split = parsed.auth.split(":")
split = parsed.auth.split(':')
if len(split) == 2:
username, password = split
if parsed.scheme == "socks5":
if parsed.scheme == 'socks5':
socks_version = socks.PROXY_TYPE_SOCKS5
rdns = False
elif parsed.scheme == "socks5h":
elif parsed.scheme == 'socks5h':
socks_version = socks.PROXY_TYPE_SOCKS5
rdns = True
elif parsed.scheme == "socks4":
elif parsed.scheme == 'socks4':
socks_version = socks.PROXY_TYPE_SOCKS4
rdns = False
elif parsed.scheme == "socks4a":
elif parsed.scheme == 'socks4a':
socks_version = socks.PROXY_TYPE_SOCKS4
rdns = True
else:
raise ValueError("Unable to determine SOCKS version from %s" % proxy_url)
raise ValueError(
"Unable to determine SOCKS version from %s" % proxy_url
)
self.proxy_url = proxy_url
socks_options = {
"socks_version": socks_version,
"proxy_host": parsed.host,
"proxy_port": parsed.port,
"username": username,
"password": password,
"rdns": rdns,
'socks_version': socks_version,
'proxy_host': parsed.host,
'proxy_port': parsed.port,
'username': username,
'password': password,
'rdns': rdns
}
connection_pool_kw["_socks_options"] = socks_options
connection_pool_kw['_socks_options'] = socks_options
super(SOCKSProxyManager, self).__init__(
num_pools, headers, **connection_pool_kw

29
lib/urllib3/exceptions.py Executable file → Normal file
View File

@@ -1,6 +1,7 @@
from __future__ import absolute_import
from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
from .packages.six.moves.http_client import (
IncompleteRead as httplib_IncompleteRead
)
# Base Exceptions
@@ -16,7 +17,6 @@ class HTTPWarning(Warning):
class PoolError(HTTPError):
"Base exception for errors caused within a pool."
def __init__(self, pool, message):
self.pool = pool
HTTPError.__init__(self, "%s: %s" % (pool, message))
@@ -28,7 +28,6 @@ class PoolError(HTTPError):
class RequestError(PoolError):
"Base exception for PoolErrors that have associated URLs."
def __init__(self, pool, url, message):
self.url = url
PoolError.__init__(self, pool, message)
@@ -64,7 +63,6 @@ ConnectionError = ProtocolError
# Leaf Exceptions
class MaxRetryError(RequestError):
"""Raised when the maximum number of retries is exceeded.
@@ -78,7 +76,8 @@ class MaxRetryError(RequestError):
def __init__(self, pool, url, reason=None):
self.reason = reason
message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason)
message = "Max retries exceeded with url: %s (Caused by %r)" % (
url, reason)
RequestError.__init__(self, pool, url, message)
@@ -94,7 +93,6 @@ class HostChangedError(RequestError):
class TimeoutStateError(HTTPError):
""" Raised when passing an invalid state to a timeout """
pass
@@ -104,7 +102,6 @@ class TimeoutError(HTTPError):
Catching this error will catch both :exc:`ReadTimeoutErrors
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
"""
pass
@@ -152,8 +149,8 @@ class LocationParseError(LocationValueError):
class ResponseError(HTTPError):
"Used as a container for an error reason supplied in a MaxRetryError."
GENERIC_ERROR = "too many error responses"
SPECIFIC_ERROR = "too many {status_code} error responses"
GENERIC_ERROR = 'too many error responses'
SPECIFIC_ERROR = 'too many {status_code} error responses'
class SecurityWarning(HTTPWarning):
@@ -191,7 +188,6 @@ class DependencyWarning(HTTPWarning):
Warned when an attempt is made to import a module with missing optional
dependencies.
"""
pass
@@ -205,7 +201,6 @@ class BodyNotHttplibCompatible(HTTPError):
Body should be httplib.HTTPResponse like (have an fp attribute which
returns raw chunks) for read_chunked().
"""
pass
@@ -217,15 +212,12 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
for `partial` to avoid creating large objects on streamed
reads.
"""
def __init__(self, partial, expected):
super(IncompleteRead, self).__init__(partial, expected)
def __repr__(self):
return "IncompleteRead(%i bytes read, " "%i more expected)" % (
self.partial,
self.expected,
)
return ('IncompleteRead(%i bytes read, '
'%i more expected)' % (self.partial, self.expected))
class InvalidHeader(HTTPError):
@@ -244,9 +236,8 @@ class ProxySchemeUnknown(AssertionError, ValueError):
class HeaderParsingError(HTTPError):
"Raised by assert_header_parsing, but we convert it to a log.warning statement."
def __init__(self, defects, unparsed_data):
message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data)
message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
super(HeaderParsingError, self).__init__(message)

89
lib/urllib3/fields.py Executable file → Normal file
View File

@@ -6,7 +6,7 @@ import re
from .packages import six
def guess_content_type(filename, default="application/octet-stream"):
def guess_content_type(filename, default='application/octet-stream'):
"""
Guess the "Content-Type" of a file.
@@ -41,22 +41,22 @@ def format_header_param_rfc2231(name, value):
if not any(ch in value for ch in '"\\\r\n'):
result = u'%s="%s"' % (name, value)
try:
result.encode("ascii")
result.encode('ascii')
except (UnicodeEncodeError, UnicodeDecodeError):
pass
else:
return result
if not six.PY3: # Python 2:
value = value.encode("utf-8")
value = value.encode('utf-8')
# encode_rfc2231 accepts an encoded string and returns an ascii-encoded
# string in Python 2 but accepts and returns unicode strings in Python 3
value = email.utils.encode_rfc2231(value, "utf-8")
value = "%s*=%s" % (name, value)
value = email.utils.encode_rfc2231(value, 'utf-8')
value = '%s*=%s' % (name, value)
if not six.PY3: # Python 2:
value = value.decode("utf-8")
value = value.decode('utf-8')
return value
@@ -69,21 +69,23 @@ _HTML5_REPLACEMENTS = {
}
# All control characters from 0x00 to 0x1F *except* 0x1B.
_HTML5_REPLACEMENTS.update(
{
six.unichr(cc): u"%{:02X}".format(cc)
for cc in range(0x00, 0x1F + 1)
if cc not in (0x1B,)
}
)
_HTML5_REPLACEMENTS.update({
six.unichr(cc): u"%{:02X}".format(cc)
for cc
in range(0x00, 0x1F+1)
if cc not in (0x1B,)
})
def _replace_multiple(value, needles_and_replacements):
def replacer(match):
return needles_and_replacements[match.group(0)]
pattern = re.compile(
r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()])
r"|".join([
re.escape(needle) for needle in needles_and_replacements.keys()
])
)
result = pattern.sub(replacer, value)
@@ -138,15 +140,13 @@ class RequestField(object):
An optional callable that is used to encode and format the headers. By
default, this is :func:`format_header_param_html5`.
"""
def __init__(
self,
name,
data,
filename=None,
headers=None,
header_formatter=format_header_param_html5,
):
self,
name,
data,
filename=None,
headers=None,
header_formatter=format_header_param_html5):
self._name = name
self._filename = filename
self.data = data
@@ -156,7 +156,11 @@ class RequestField(object):
self.header_formatter = header_formatter
@classmethod
def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5):
def from_tuples(
cls,
fieldname,
value,
header_formatter=format_header_param_html5):
"""
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
@@ -185,8 +189,7 @@ class RequestField(object):
data = value
request_param = cls(
fieldname, data, filename=filename, header_formatter=header_formatter
)
fieldname, data, filename=filename, header_formatter=header_formatter)
request_param.make_multipart(content_type=content_type)
return request_param
@@ -224,7 +227,7 @@ class RequestField(object):
if value is not None:
parts.append(self._render_part(name, value))
return u"; ".join(parts)
return u'; '.join(parts)
def render_headers(self):
"""
@@ -232,22 +235,21 @@ class RequestField(object):
"""
lines = []
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location']
for sort_key in sort_keys:
if self.headers.get(sort_key, False):
lines.append(u"%s: %s" % (sort_key, self.headers[sort_key]))
lines.append(u'%s: %s' % (sort_key, self.headers[sort_key]))
for header_name, header_value in self.headers.items():
if header_name not in sort_keys:
if header_value:
lines.append(u"%s: %s" % (header_name, header_value))
lines.append(u'%s: %s' % (header_name, header_value))
lines.append(u"\r\n")
return u"\r\n".join(lines)
lines.append(u'\r\n')
return u'\r\n'.join(lines)
def make_multipart(
self, content_disposition=None, content_type=None, content_location=None
):
def make_multipart(self, content_disposition=None, content_type=None,
content_location=None):
"""
Makes this request field into a multipart request field.
@@ -260,14 +262,11 @@ class RequestField(object):
The 'Content-Location' of the request body.
"""
self.headers["Content-Disposition"] = content_disposition or u"form-data"
self.headers["Content-Disposition"] += u"; ".join(
[
u"",
self._render_parts(
((u"name", self._name), (u"filename", self._filename))
),
]
)
self.headers["Content-Type"] = content_type
self.headers["Content-Location"] = content_location
self.headers['Content-Disposition'] = content_disposition or u'form-data'
self.headers['Content-Disposition'] += u'; '.join([
u'', self._render_parts(
((u'name', self._name), (u'filename', self._filename))
)
])
self.headers['Content-Type'] = content_type
self.headers['Content-Location'] = content_location

12
lib/urllib3/filepost.py Executable file → Normal file
View File

@@ -9,7 +9,7 @@ from .packages import six
from .packages.six import b
from .fields import RequestField
writer = codecs.lookup("utf-8")[3]
writer = codecs.lookup('utf-8')[3]
def choose_boundary():
@@ -18,7 +18,7 @@ def choose_boundary():
"""
boundary = binascii.hexlify(os.urandom(16))
if six.PY3:
boundary = boundary.decode("ascii")
boundary = boundary.decode('ascii')
return boundary
@@ -76,7 +76,7 @@ def encode_multipart_formdata(fields, boundary=None):
boundary = choose_boundary()
for field in iter_field_objects(fields):
body.write(b("--%s\r\n" % (boundary)))
body.write(b('--%s\r\n' % (boundary)))
writer(body).write(field.render_headers())
data = field.data
@@ -89,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None):
else:
body.write(data)
body.write(b"\r\n")
body.write(b'\r\n')
body.write(b("--%s--\r\n" % (boundary)))
body.write(b('--%s--\r\n' % (boundary)))
content_type = str("multipart/form-data; boundary=%s" % boundary)
content_type = str('multipart/form-data; boundary=%s' % boundary)
return body.getvalue(), content_type

2
lib/urllib3/packages/__init__.py Executable file → Normal file
View File

@@ -2,4 +2,4 @@ from __future__ import absolute_import
from . import ssl_match_hostname
__all__ = ("ssl_match_hostname",)
__all__ = ('ssl_match_hostname', )

0
lib/urllib3/packages/backports/__init__.py Executable file → Normal file
View File

9
lib/urllib3/packages/backports/makefile.py Executable file → Normal file
View File

@@ -11,14 +11,15 @@ import io
from socket import SocketIO
def backport_makefile(
self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
def backport_makefile(self, mode="r", buffering=None, encoding=None,
errors=None, newline=None):
"""
Backport of ``socket.makefile`` from Python 3.5.
"""
if not set(mode) <= {"r", "w", "b"}:
raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
raise ValueError(
"invalid mode %r (only r, w, b allowed)" % (mode,)
)
writing = "w" in mode
reading = "r" in mode or not writing
assert reading or writing

40
lib/urllib3/packages/rfc3986/__init__.py Executable file → Normal file
View File

@@ -31,26 +31,26 @@ from .api import URIReference
from .api import urlparse
from .parseresult import ParseResult
__title__ = "rfc3986"
__author__ = "Ian Stapleton Cordasco"
__author_email__ = "graffatcolmingov@gmail.com"
__license__ = "Apache v2.0"
__copyright__ = "Copyright 2014 Rackspace"
__version__ = "1.3.2"
__title__ = 'rfc3986'
__author__ = 'Ian Stapleton Cordasco'
__author_email__ = 'graffatcolmingov@gmail.com'
__license__ = 'Apache v2.0'
__copyright__ = 'Copyright 2014 Rackspace'
__version__ = '1.3.0'
__all__ = (
"ParseResult",
"URIReference",
"IRIReference",
"is_valid_uri",
"normalize_uri",
"uri_reference",
"iri_reference",
"urlparse",
"__title__",
"__author__",
"__author_email__",
"__license__",
"__copyright__",
"__version__",
'ParseResult',
'URIReference',
'IRIReference',
'is_valid_uri',
'normalize_uri',
'uri_reference',
'iri_reference',
'urlparse',
'__title__',
'__author__',
'__author_email__',
'__license__',
'__copyright__',
'__version__',
)

120
lib/urllib3/packages/rfc3986/_mixin.py Executable file → Normal file
View File

@@ -26,7 +26,7 @@ class URIMixin(object):
If the authority is not ``None`` and can not be parsed.
"""
if not self.authority:
return {"userinfo": None, "host": None, "port": None}
return {'userinfo': None, 'host': None, 'port': None}
match = self._match_subauthority()
@@ -40,13 +40,10 @@ class URIMixin(object):
# We had a match, now let's ensure that it is actually a valid host
# address if it is IPv4
matches = match.groupdict()
host = matches.get("host")
host = matches.get('host')
if (
host
and misc.IPv4_MATCHER.match(host)
and not validators.valid_ipv4_host_address(host)
):
if (host and misc.IPv4_MATCHER.match(host) and not
validators.valid_ipv4_host_address(host)):
# If we have a host, it appears to be IPv4 and it does not have
# valid bytes, it is an InvalidAuthority.
raise exc.InvalidAuthority(self.authority.encode(self.encoding))
@@ -63,7 +60,7 @@ class URIMixin(object):
authority = self.authority_info()
except exc.InvalidAuthority:
return None
return authority["host"]
return authority['host']
@property
def port(self):
@@ -72,7 +69,7 @@ class URIMixin(object):
authority = self.authority_info()
except exc.InvalidAuthority:
return None
return authority["port"]
return authority['port']
@property
def userinfo(self):
@@ -81,7 +78,7 @@ class URIMixin(object):
authority = self.authority_info()
except exc.InvalidAuthority:
return None
return authority["userinfo"]
return authority['userinfo']
def is_absolute(self):
"""Determine if this URI Reference is an absolute URI.
@@ -113,18 +110,16 @@ class URIMixin(object):
:returns: ``True`` if the URI is valid. ``False`` otherwise.
:rtype: bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
validators = [
(self.scheme_is_valid, kwargs.get("require_scheme", False)),
(self.authority_is_valid, kwargs.get("require_authority", False)),
(self.path_is_valid, kwargs.get("require_path", False)),
(self.query_is_valid, kwargs.get("require_query", False)),
(self.fragment_is_valid, kwargs.get("require_fragment", False)),
]
(self.scheme_is_valid, kwargs.get('require_scheme', False)),
(self.authority_is_valid, kwargs.get('require_authority', False)),
(self.path_is_valid, kwargs.get('require_path', False)),
(self.query_is_valid, kwargs.get('require_query', False)),
(self.fragment_is_valid, kwargs.get('require_fragment', False)),
]
return all(v(r) for v, r in validators)
def authority_is_valid(self, require=False):
@@ -141,18 +136,18 @@ class URIMixin(object):
:rtype:
bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
try:
self.authority_info()
except exc.InvalidAuthority:
return False
return validators.authority_is_valid(
self.authority, host=self.host, require=require
self.authority,
host=self.host,
require=require,
)
def scheme_is_valid(self, require=False):
@@ -167,11 +162,9 @@ class URIMixin(object):
:returns: ``True`` if the scheme is valid. ``False`` otherwise.
:rtype: bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
return validators.scheme_is_valid(self.scheme, require)
def path_is_valid(self, require=False):
@@ -186,11 +179,9 @@ class URIMixin(object):
:returns: ``True`` if the path is valid. ``False`` otherwise.
:rtype: bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
return validators.path_is_valid(self.path, require)
def query_is_valid(self, require=False):
@@ -205,11 +196,9 @@ class URIMixin(object):
:returns: ``True`` if the query is valid. ``False`` otherwise.
:rtype: bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
return validators.query_is_valid(self.query, require)
def fragment_is_valid(self, require=False):
@@ -224,11 +213,9 @@ class URIMixin(object):
:returns: ``True`` if the fragment is valid. ``False`` otherwise.
:rtype: bool
"""
warnings.warn(
"Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning,
)
warnings.warn("Please use rfc3986.validators.Validator instead. "
"This method will be eventually removed.",
DeprecationWarning)
return validators.fragment_is_valid(self.fragment, require)
def normalized_equality(self, other_ref):
@@ -282,7 +269,7 @@ class URIMixin(object):
if resolving.authority is not None:
target = resolving.copy_with(
scheme=base_uri.scheme,
path=normalizers.normalize_path(resolving.path),
path=normalizers.normalize_path(resolving.path)
)
else:
if resolving.path is None:
@@ -294,10 +281,10 @@ class URIMixin(object):
scheme=base_uri.scheme,
authority=base_uri.authority,
path=base_uri.path,
query=query,
query=query
)
else:
if resolving.path.startswith("/"):
if resolving.path.startswith('/'):
path = normalizers.normalize_path(resolving.path)
else:
path = normalizers.normalize_path(
@@ -307,7 +294,7 @@ class URIMixin(object):
scheme=base_uri.scheme,
authority=base_uri.authority,
path=path,
query=resolving.query,
query=resolving.query
)
return target
@@ -320,25 +307,20 @@ class URIMixin(object):
# See http://tools.ietf.org/html/rfc3986#section-5.3
result_list = []
if self.scheme:
result_list.extend([self.scheme, ":"])
result_list.extend([self.scheme, ':'])
if self.authority:
result_list.extend(["//", self.authority])
result_list.extend(['//', self.authority])
if self.path:
result_list.append(self.path)
if self.query is not None:
result_list.extend(["?", self.query])
result_list.extend(['?', self.query])
if self.fragment is not None:
result_list.extend(["#", self.fragment])
return "".join(result_list)
result_list.extend(['#', self.fragment])
return ''.join(result_list)
def copy_with(
self,
scheme=misc.UseExisting,
authority=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
):
def copy_with(self, scheme=misc.UseExisting, authority=misc.UseExisting,
path=misc.UseExisting, query=misc.UseExisting,
fragment=misc.UseExisting):
"""Create a copy of this reference with the new components.
:param str scheme:
@@ -357,11 +339,11 @@ class URIMixin(object):
URIReference
"""
attributes = {
"scheme": scheme,
"authority": authority,
"path": path,
"query": query,
"fragment": fragment,
'scheme': scheme,
'authority': authority,
'path': path,
'query': query,
'fragment': fragment,
}
for key, value in list(attributes.items()):
if value is misc.UseExisting:

219
lib/urllib3/packages/rfc3986/abnf_regexp.py Executable file → Normal file
View File

@@ -24,35 +24,35 @@ SUB_DELIMITERS_SET = set(SUB_DELIMITERS)
# Escape the '*' for use in regular expressions
SUB_DELIMITERS_RE = r"!$&'()\*+,;="
RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET)
ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
DIGIT = "0123456789"
ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
DIGIT = '0123456789'
# https://tools.ietf.org/html/rfc3986#section-2.3
UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r"._!-"
UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r'._!-'
UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS)
NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET)
# We need to escape the '-' in this case:
UNRESERVED_RE = r"A-Za-z0-9._~\-"
UNRESERVED_RE = r'A-Za-z0-9._~\-'
# Percent encoded character values
PERCENT_ENCODED = PCT_ENCODED = "%[A-Fa-f0-9]{2}"
PCHAR = "([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED
PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}'
PCHAR = '([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':@]|%s)' % PCT_ENCODED
# NOTE(sigmavirus24): We're going to use more strict regular expressions
# than appear in Appendix B for scheme. This will prevent over-eager
# consuming of items that aren't schemes.
SCHEME_RE = "[a-zA-Z][a-zA-Z0-9+.-]*"
_AUTHORITY_RE = "[^/?#]*"
_PATH_RE = "[^?#]*"
_QUERY_RE = "[^#]*"
_FRAGMENT_RE = ".*"
SCHEME_RE = '[a-zA-Z][a-zA-Z0-9+.-]*'
_AUTHORITY_RE = '[^/?#]*'
_PATH_RE = '[^?#]*'
_QUERY_RE = '[^#]*'
_FRAGMENT_RE = '.*'
# Extracted from http://tools.ietf.org/html/rfc3986#appendix-B
COMPONENT_PATTERN_DICT = {
"scheme": SCHEME_RE,
"authority": _AUTHORITY_RE,
"path": _PATH_RE,
"query": _QUERY_RE,
"fragment": _FRAGMENT_RE,
'scheme': SCHEME_RE,
'authority': _AUTHORITY_RE,
'path': _PATH_RE,
'query': _QUERY_RE,
'fragment': _FRAGMENT_RE,
}
# See http://tools.ietf.org/html/rfc3986#appendix-B
@@ -61,9 +61,9 @@ COMPONENT_PATTERN_DICT = {
# modified to ignore other matches that are not important to the parsing of
# the reference so we can also simply use SRE_Match#groups.
URL_PARSING_RE = (
r"(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?"
r"(?P<path>{path})(?:\?(?P<query>{query}))?"
r"(?:#(?P<fragment>{fragment}))?"
r'(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?'
r'(?P<path>{path})(?:\?(?P<query>{query}))?'
r'(?:#(?P<fragment>{fragment}))?'
).format(**COMPONENT_PATTERN_DICT)
@@ -73,58 +73,71 @@ URL_PARSING_RE = (
# Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2
# The pattern for a regular name, e.g., www.google.com, api.github.com
REGULAR_NAME_RE = REG_NAME = "((?:{0}|[{1}])*)".format(
"%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + UNRESERVED_RE
REGULAR_NAME_RE = REG_NAME = '((?:{0}|[{1}])*)'.format(
'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + UNRESERVED_RE
)
# The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1,
IPv4_RE = r"([0-9]{1,3}\.){3}[0-9]{1,3}"
IPv4_RE = r'([0-9]{1,3}\.){3}[0-9]{1,3}'
# Hexadecimal characters used in each piece of an IPv6 address
HEXDIG_RE = "[0-9A-Fa-f]{1,4}"
HEXDIG_RE = '[0-9A-Fa-f]{1,4}'
# Least-significant 32 bits of an IPv6 address
LS32_RE = "({hex}:{hex}|{ipv4})".format(hex=HEXDIG_RE, ipv4=IPv4_RE)
LS32_RE = '({hex}:{hex}|{ipv4})'.format(hex=HEXDIG_RE, ipv4=IPv4_RE)
# Substitutions into the following patterns for IPv6 patterns defined
# http://tools.ietf.org/html/rfc3986#page-20
_subs = {"hex": HEXDIG_RE, "ls32": LS32_RE}
_subs = {'hex': HEXDIG_RE, 'ls32': LS32_RE}
# Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details
# about ABNF (Augmented Backus-Naur Form) use in the comments
variations = [
# 6( h16 ":" ) ls32
"(%(hex)s:){6}%(ls32)s" % _subs,
'(%(hex)s:){6}%(ls32)s' % _subs,
# "::" 5( h16 ":" ) ls32
"::(%(hex)s:){5}%(ls32)s" % _subs,
'::(%(hex)s:){5}%(ls32)s' % _subs,
# [ h16 ] "::" 4( h16 ":" ) ls32
"(%(hex)s)?::(%(hex)s:){4}%(ls32)s" % _subs,
'(%(hex)s)?::(%(hex)s:){4}%(ls32)s' % _subs,
# [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
"((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s" % _subs,
'((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s' % _subs,
# [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
"((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s" % _subs,
'((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s' % _subs,
# [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
"((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s" % _subs,
'((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s' % _subs,
# [ *4( h16 ":" ) h16 ] "::" ls32
"((%(hex)s:){0,4}%(hex)s)?::%(ls32)s" % _subs,
'((%(hex)s:){0,4}%(hex)s)?::%(ls32)s' % _subs,
# [ *5( h16 ":" ) h16 ] "::" h16
"((%(hex)s:){0,5}%(hex)s)?::%(hex)s" % _subs,
'((%(hex)s:){0,5}%(hex)s)?::%(hex)s' % _subs,
# [ *6( h16 ":" ) h16 ] "::"
"((%(hex)s:){0,6}%(hex)s)?::" % _subs,
'((%(hex)s:){0,6}%(hex)s)?::' % _subs,
]
IPv6_RE = "(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))".format(*variations)
IPv6_RE = '(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))'.format(
*variations
)
IPv_FUTURE_RE = r"v[0-9A-Fa-f]+\.[%s]+" % (UNRESERVED_RE + SUB_DELIMITERS_RE + ":")
IPv_FUTURE_RE = r'v[0-9A-Fa-f]+\.[%s]+' % (
UNRESERVED_RE + SUB_DELIMITERS_RE + ':'
)
# RFC 6874 Zone ID ABNF
ZONE_ID = "(?:[" + UNRESERVED_RE + "]|" + PCT_ENCODED + ")+"
ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+'
IPv6_ADDRZ_RFC4007_RE = IPv6_RE + "(?:(?:%25|%)" + ZONE_ID + ")?"
IPv6_ADDRZ_RE = IPv6_RE + "(?:%25" + ZONE_ID + ")?"
IPv6_ADDRZ_RFC4007_RE = IPv6_RE + '(?:(?:%25|%)' + ZONE_ID + ')?'
IPv6_ADDRZ_RE = IPv6_RE + '(?:%25' + ZONE_ID + ')?'
IP_LITERAL_RE = r"\[({0}|{1})\]".format(IPv6_ADDRZ_RFC4007_RE, IPv_FUTURE_RE)
IP_LITERAL_RE = r'\[({0}|{1})\]'.format(
IPv6_ADDRZ_RFC4007_RE,
IPv_FUTURE_RE,
)
# Pattern for matching the host piece of the authority
HOST_RE = HOST_PATTERN = "({0}|{1}|{2})".format(REG_NAME, IPv4_RE, IP_LITERAL_RE)
USERINFO_RE = "^([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":]|%s)+" % (PCT_ENCODED)
PORT_RE = "[0-9]{1,5}"
HOST_RE = HOST_PATTERN = '({0}|{1}|{2})'.format(
REG_NAME,
IPv4_RE,
IP_LITERAL_RE,
)
USERINFO_RE = '^([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':]|%s)+' % (
PCT_ENCODED
)
PORT_RE = '[0-9]{1,5}'
# ####################
# Path Matcher Section
@@ -133,29 +146,25 @@ PORT_RE = "[0-9]{1,5}"
# See http://tools.ietf.org/html/rfc3986#section-3.3 for more information
# about the path patterns defined below.
segments = {
"segment": PCHAR + "*",
'segment': PCHAR + '*',
# Non-zero length segment
"segment-nz": PCHAR + "+",
'segment-nz': PCHAR + '+',
# Non-zero length segment without ":"
"segment-nz-nc": PCHAR.replace(":", "") + "+",
'segment-nz-nc': PCHAR.replace(':', '') + '+'
}
# Path types taken from Section 3.3 (linked above)
PATH_EMPTY = "^$"
PATH_ROOTLESS = "%(segment-nz)s(/%(segment)s)*" % segments
PATH_NOSCHEME = "%(segment-nz-nc)s(/%(segment)s)*" % segments
PATH_ABSOLUTE = "/(%s)?" % PATH_ROOTLESS
PATH_ABEMPTY = "(/%(segment)s)*" % segments
PATH_RE = "^(%s|%s|%s|%s|%s)$" % (
PATH_ABEMPTY,
PATH_ABSOLUTE,
PATH_NOSCHEME,
PATH_ROOTLESS,
PATH_EMPTY,
PATH_EMPTY = '^$'
PATH_ROOTLESS = '%(segment-nz)s(/%(segment)s)*' % segments
PATH_NOSCHEME = '%(segment-nz-nc)s(/%(segment)s)*' % segments
PATH_ABSOLUTE = '/(%s)?' % PATH_ROOTLESS
PATH_ABEMPTY = '(/%(segment)s)*' % segments
PATH_RE = '^(%s|%s|%s|%s|%s)$' % (
PATH_ABEMPTY, PATH_ABSOLUTE, PATH_NOSCHEME, PATH_ROOTLESS, PATH_EMPTY
)
FRAGMENT_RE = QUERY_RE = (
"^([/?:@" + UNRESERVED_RE + SUB_DELIMITERS_RE + "]|%s)*$" % PCT_ENCODED
'^([/?:@' + UNRESERVED_RE + SUB_DELIMITERS_RE + ']|%s)*$' % PCT_ENCODED
)
# ##########################
@@ -163,8 +172,8 @@ FRAGMENT_RE = QUERY_RE = (
# ##########################
# See http://tools.ietf.org/html/rfc3986#section-4.2 for details
RELATIVE_PART_RE = "(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT["authority"],
RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % (
COMPONENT_PATTERN_DICT['authority'],
PATH_ABEMPTY,
PATH_ABSOLUTE,
PATH_NOSCHEME,
@@ -172,8 +181,8 @@ RELATIVE_PART_RE = "(//%s%s|%s|%s|%s)" % (
)
# See http://tools.ietf.org/html/rfc3986#section-3 for definition
HIER_PART_RE = "(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT["authority"],
HIER_PART_RE = '(//%s%s|%s|%s|%s)' % (
COMPONENT_PATTERN_DICT['authority'],
PATH_ABEMPTY,
PATH_ABSOLUTE,
PATH_ROOTLESS,
@@ -186,75 +195,71 @@ HIER_PART_RE = "(//%s%s|%s|%s|%s)" % (
# Only wide-unicode gets the high-ranges of UCSCHAR
if sys.maxunicode > 0xFFFF: # pragma: no cover
IPRIVATE = u"\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD"
IPRIVATE = u'\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD'
UCSCHAR_RE = (
u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
u"\U00010000-\U0001FFFD\U00020000-\U0002FFFD"
u"\U00030000-\U0003FFFD\U00040000-\U0004FFFD"
u"\U00050000-\U0005FFFD\U00060000-\U0006FFFD"
u"\U00070000-\U0007FFFD\U00080000-\U0008FFFD"
u"\U00090000-\U0009FFFD\U000A0000-\U000AFFFD"
u"\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD"
u"\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD"
u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF'
u'\U00010000-\U0001FFFD\U00020000-\U0002FFFD'
u'\U00030000-\U0003FFFD\U00040000-\U0004FFFD'
u'\U00050000-\U0005FFFD\U00060000-\U0006FFFD'
u'\U00070000-\U0007FFFD\U00080000-\U0008FFFD'
u'\U00090000-\U0009FFFD\U000A0000-\U000AFFFD'
u'\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD'
u'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD'
)
else: # pragma: no cover
IPRIVATE = u"\uE000-\uF8FF"
UCSCHAR_RE = u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
IPRIVATE = u'\uE000-\uF8FF'
UCSCHAR_RE = (
u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF'
)
IUNRESERVED_RE = u"A-Za-z0-9\\._~\\-" + UCSCHAR_RE
IPCHAR = u"([" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":@]|%s)" % PCT_ENCODED
IUNRESERVED_RE = u'A-Za-z0-9\\._~\\-' + UCSCHAR_RE
IPCHAR = u'([' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':@]|%s)' % PCT_ENCODED
isegments = {
"isegment": IPCHAR + u"*",
'isegment': IPCHAR + u'*',
# Non-zero length segment
"isegment-nz": IPCHAR + u"+",
'isegment-nz': IPCHAR + u'+',
# Non-zero length segment without ":"
"isegment-nz-nc": IPCHAR.replace(":", "") + u"+",
'isegment-nz-nc': IPCHAR.replace(':', '') + u'+'
}
IPATH_ROOTLESS = u"%(isegment-nz)s(/%(isegment)s)*" % isegments
IPATH_NOSCHEME = u"%(isegment-nz-nc)s(/%(isegment)s)*" % isegments
IPATH_ABSOLUTE = u"/(?:%s)?" % IPATH_ROOTLESS
IPATH_ABEMPTY = u"(?:/%(isegment)s)*" % isegments
IPATH_RE = u"^(?:%s|%s|%s|%s|%s)$" % (
IPATH_ABEMPTY,
IPATH_ABSOLUTE,
IPATH_NOSCHEME,
IPATH_ROOTLESS,
PATH_EMPTY,
IPATH_ROOTLESS = u'%(isegment-nz)s(/%(isegment)s)*' % isegments
IPATH_NOSCHEME = u'%(isegment-nz-nc)s(/%(isegment)s)*' % isegments
IPATH_ABSOLUTE = u'/(?:%s)?' % IPATH_ROOTLESS
IPATH_ABEMPTY = u'(?:/%(isegment)s)*' % isegments
IPATH_RE = u'^(?:%s|%s|%s|%s|%s)$' % (
IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_NOSCHEME, IPATH_ROOTLESS, PATH_EMPTY
)
IREGULAR_NAME_RE = IREG_NAME = u"(?:{0}|[{1}])*".format(
u"%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + IUNRESERVED_RE
IREGULAR_NAME_RE = IREG_NAME = u'(?:{0}|[{1}])*'.format(
u'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + IUNRESERVED_RE
)
IHOST_RE = IHOST_PATTERN = u"({0}|{1}|{2})".format(IREG_NAME, IPv4_RE, IP_LITERAL_RE)
IUSERINFO_RE = (
u"^(?:[" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":]|%s)+" % (PCT_ENCODED)
IHOST_RE = IHOST_PATTERN = u'({0}|{1}|{2})'.format(
IREG_NAME,
IPv4_RE,
IP_LITERAL_RE,
)
IFRAGMENT_RE = (
u"^(?:[/?:@" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u"]|%s)*$" % PCT_ENCODED
)
IQUERY_RE = (
u"^(?:[/?:@"
+ IUNRESERVED_RE
+ SUB_DELIMITERS_RE
+ IPRIVATE
+ u"]|%s)*$" % PCT_ENCODED
IUSERINFO_RE = u'^(?:[' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':]|%s)+' % (
PCT_ENCODED
)
IRELATIVE_PART_RE = u"(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT["authority"],
IFRAGMENT_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE
+ u']|%s)*$' % PCT_ENCODED)
IQUERY_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE
+ IPRIVATE + u']|%s)*$' % PCT_ENCODED)
IRELATIVE_PART_RE = u'(//%s%s|%s|%s|%s)' % (
COMPONENT_PATTERN_DICT['authority'],
IPATH_ABEMPTY,
IPATH_ABSOLUTE,
IPATH_NOSCHEME,
PATH_EMPTY,
)
IHIER_PART_RE = u"(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT["authority"],
IHIER_PART_RE = u'(//%s%s|%s|%s|%s)' % (
COMPONENT_PATTERN_DICT['authority'],
IPATH_ABEMPTY,
IPATH_ABSOLUTE,
IPATH_ROOTLESS,

10
lib/urllib3/packages/rfc3986/api.py Executable file → Normal file
View File

@@ -24,7 +24,7 @@ from .parseresult import ParseResult
from .uri import URIReference
def uri_reference(uri, encoding="utf-8"):
def uri_reference(uri, encoding='utf-8'):
"""Parse a URI string into a URIReference.
This is a convenience function. You could achieve the same end by using
@@ -38,7 +38,7 @@ def uri_reference(uri, encoding="utf-8"):
return URIReference.from_string(uri, encoding)
def iri_reference(iri, encoding="utf-8"):
def iri_reference(iri, encoding='utf-8'):
"""Parse a IRI string into an IRIReference.
This is a convenience function. You could achieve the same end by using
@@ -52,7 +52,7 @@ def iri_reference(iri, encoding="utf-8"):
return IRIReference.from_string(iri, encoding)
def is_valid_uri(uri, encoding="utf-8", **kwargs):
def is_valid_uri(uri, encoding='utf-8', **kwargs):
"""Determine if the URI given is valid.
This is a convenience function. You could use either
@@ -77,7 +77,7 @@ def is_valid_uri(uri, encoding="utf-8", **kwargs):
return URIReference.from_string(uri, encoding).is_valid(**kwargs)
def normalize_uri(uri, encoding="utf-8"):
def normalize_uri(uri, encoding='utf-8'):
"""Normalize the given URI.
This is a convenience function. You could use either
@@ -93,7 +93,7 @@ def normalize_uri(uri, encoding="utf-8"):
return normalized_reference.unsplit()
def urlparse(uri, encoding="utf-8"):
def urlparse(uri, encoding='utf-8'):
"""Parse a given URI and return a ParseResult.
This is a partial replacement of the standard library's urlparse function.

47
lib/urllib3/packages/rfc3986/builder.py Executable file → Normal file
View File

@@ -29,16 +29,8 @@ class URIBuilder(object):
"""
def __init__(
self,
scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
):
def __init__(self, scheme=None, userinfo=None, host=None, port=None,
path=None, query=None, fragment=None):
"""Initialize our URI builder.
:param str scheme:
@@ -66,11 +58,9 @@ class URIBuilder(object):
def __repr__(self):
"""Provide a convenient view of our builder object."""
formatstr = (
"URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, "
"host={b.host}, port={b.port}, path={b.path}, "
"query={b.query}, fragment={b.fragment})"
)
formatstr = ('URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, '
'host={b.host}, port={b.port}, path={b.path}, '
'query={b.query}, fragment={b.fragment})')
return formatstr.format(b=self)
def add_scheme(self, scheme):
@@ -111,12 +101,13 @@ class URIBuilder(object):
port=None, path=None, query=None, fragment=None)
"""
if username is None:
raise ValueError("Username cannot be None")
raise ValueError('Username cannot be None')
userinfo = normalizers.normalize_username(username)
if password is not None:
userinfo = "{}:{}".format(
userinfo, normalizers.normalize_password(password)
userinfo = '{}:{}'.format(
userinfo,
normalizers.normalize_password(password),
)
return URIBuilder(
@@ -166,19 +157,23 @@ class URIBuilder(object):
port_int = int(port)
if port_int < 0:
raise ValueError(
"ports are not allowed to be negative. You provided {}".format(port_int)
'ports are not allowed to be negative. You provided {}'.format(
port_int,
)
)
if port_int > 65535:
raise ValueError(
"ports are not allowed to be larger than 65535. "
"You provided {}".format(port_int)
'ports are not allowed to be larger than 65535. '
'You provided {}'.format(
port_int,
)
)
return URIBuilder(
scheme=self.scheme,
userinfo=self.userinfo,
host=self.host,
port="{}".format(port_int),
port='{}'.format(port_int),
path=self.path,
query=self.query,
fragment=self.fragment,
@@ -198,8 +193,8 @@ class URIBuilder(object):
path='/checkout.php', query=None, fragment=None)
"""
if not path.startswith("/"):
path = "/{}".format(path)
if not path.startswith('/'):
path = '/{}'.format(path)
return URIBuilder(
scheme=self.scheme,
@@ -294,7 +289,9 @@ class URIBuilder(object):
"""
return uri.URIReference(
self.scheme,
normalizers.normalize_authority((self.userinfo, self.host, self.port)),
normalizers.normalize_authority(
(self.userinfo, self.host, self.port)
),
self.path,
self.query,
self.fragment,

15
lib/urllib3/packages/rfc3986/compat.py Executable file → Normal file
View File

@@ -25,7 +25,12 @@ try:
except ImportError: # Python 2.x
from urllib import urlencode
__all__ = ("to_bytes", "to_str", "urlquote", "urlencode")
__all__ = (
'to_bytes',
'to_str',
'urlquote',
'urlencode',
)
PY3 = (3, 0) <= sys.version_info < (4, 0)
PY2 = (2, 6) <= sys.version_info < (2, 8)
@@ -35,15 +40,15 @@ if PY3:
unicode = str # Python 3.x
def to_str(b, encoding="utf-8"):
def to_str(b, encoding='utf-8'):
"""Ensure that b is text in the specified encoding."""
if hasattr(b, "decode") and not isinstance(b, unicode):
if hasattr(b, 'decode') and not isinstance(b, unicode):
b = b.decode(encoding)
return b
def to_bytes(s, encoding="utf-8"):
def to_bytes(s, encoding='utf-8'):
"""Ensure that s is converted to bytes from the encoding."""
if hasattr(s, "encode") and not isinstance(s, bytes):
if hasattr(s, 'encode') and not isinstance(s, bytes):
s = s.encode(encoding)
return s

38
lib/urllib3/packages/rfc3986/exceptions.py Executable file → Normal file
View File

@@ -16,8 +16,8 @@ class InvalidAuthority(RFC3986Exception):
def __init__(self, authority):
"""Initialize the exception with the invalid authority."""
super(InvalidAuthority, self).__init__(
u"The authority ({0}) is not valid.".format(compat.to_str(authority))
)
u"The authority ({0}) is not valid.".format(
compat.to_str(authority)))
class InvalidPort(RFC3986Exception):
@@ -25,7 +25,8 @@ class InvalidPort(RFC3986Exception):
def __init__(self, port):
"""Initialize the exception with the invalid port."""
super(InvalidPort, self).__init__('The port ("{0}") is not valid.'.format(port))
super(InvalidPort, self).__init__(
'The port ("{0}") is not valid.'.format(port))
class ResolutionError(RFC3986Exception):
@@ -34,8 +35,7 @@ class ResolutionError(RFC3986Exception):
def __init__(self, uri):
"""Initialize the error with the failed URI."""
super(ResolutionError, self).__init__(
"{0} is not an absolute URI.".format(uri.unsplit())
)
"{0} is not an absolute URI.".format(uri.unsplit()))
class ValidationError(RFC3986Exception):
@@ -49,15 +49,17 @@ class MissingComponentError(ValidationError):
def __init__(self, uri, *component_names):
"""Initialize the error with the missing component name."""
verb = "was"
verb = 'was'
if len(component_names) > 1:
verb = "were"
verb = 'were'
self.uri = uri
self.components = sorted(component_names)
components = ", ".join(self.components)
components = ', '.join(self.components)
super(MissingComponentError, self).__init__(
"{} {} required but missing".format(components, verb), uri, self.components
"{} {} required but missing".format(components, verb),
uri,
self.components,
)
@@ -68,7 +70,7 @@ class UnpermittedComponentError(ValidationError):
"""Initialize the error with the unpermitted component."""
super(UnpermittedComponentError, self).__init__(
"{} was required to be one of {!r} but was {!r}".format(
component_name, list(sorted(allowed_values)), component_value
component_name, list(sorted(allowed_values)), component_value,
),
component_name,
component_value,
@@ -84,9 +86,11 @@ class PasswordForbidden(ValidationError):
def __init__(self, uri):
"""Initialize the error with the URI that failed validation."""
unsplit = getattr(uri, "unsplit", lambda: uri)
unsplit = getattr(uri, 'unsplit', lambda: uri)
super(PasswordForbidden, self).__init__(
'"{}" contained a password when validation forbade it'.format(unsplit())
'"{}" contained a password when validation forbade it'.format(
unsplit()
)
)
self.uri = uri
@@ -96,15 +100,17 @@ class InvalidComponentsError(ValidationError):
def __init__(self, uri, *component_names):
"""Initialize the error with the invalid component name(s)."""
verb = "was"
verb = 'was'
if len(component_names) > 1:
verb = "were"
verb = 'were'
self.uri = uri
self.components = sorted(component_names)
components = ", ".join(self.components)
components = ', '.join(self.components)
super(InvalidComponentsError, self).__init__(
"{} {} found to be invalid".format(components, verb), uri, self.components
"{} {} found to be invalid".format(components, verb),
uri,
self.components,
)

69
lib/urllib3/packages/rfc3986/iri.py Executable file → Normal file
View File

@@ -29,7 +29,8 @@ except ImportError: # pragma: no cover
idna = None
class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin):
class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
uri.URIMixin):
"""Immutable object representing a parsed IRI Reference.
Can be encoded into an URIReference object via the procedure
@@ -42,11 +43,16 @@ class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
slots = ()
def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"):
def __new__(cls, scheme, authority, path, query, fragment,
encoding='utf-8'):
"""Create a new IRIReference."""
ref = super(IRIReference, cls).__new__(
cls, scheme or None, authority or None, path or None, query, fragment
)
cls,
scheme or None,
authority or None,
path or None,
query,
fragment)
ref.encoding = encoding
return ref
@@ -60,10 +66,8 @@ class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
other_ref = self.__class__.from_string(other)
except TypeError:
raise TypeError(
"Unable to compare {0}() to {1}()".format(
type(self).__name__, type(other).__name__
)
)
'Unable to compare {0}() to {1}()'.format(
type(self).__name__, type(other).__name__))
# See http://tools.ietf.org/html/rfc3986#section-6.2
return tuple(self) == tuple(other_ref)
@@ -72,7 +76,7 @@ class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
return misc.ISUBAUTHORITY_MATCHER.match(self.authority)
@classmethod
def from_string(cls, iri_string, encoding="utf-8"):
def from_string(cls, iri_string, encoding='utf-8'):
"""Parse a IRI reference from the given unicode IRI string.
:param str iri_string: Unicode IRI to be parsed into a reference.
@@ -83,15 +87,14 @@ class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
split_iri = misc.IRI_MATCHER.match(iri_string).groupdict()
return cls(
split_iri["scheme"],
split_iri["authority"],
normalizers.encode_component(split_iri["path"], encoding),
normalizers.encode_component(split_iri["query"], encoding),
normalizers.encode_component(split_iri["fragment"], encoding),
split_iri['scheme'], split_iri['authority'],
normalizers.encode_component(split_iri['path'], encoding),
normalizers.encode_component(split_iri['query'], encoding),
normalizers.encode_component(split_iri['fragment'], encoding),
encoding,
)
def encode(self, idna_encoder=None): # noqa: C901
def encode(self, idna_encoder=None):
"""Encode an IRIReference into a URIReference instance.
If the ``idna`` module is installed or the ``rfc3986[idna]``
@@ -113,38 +116,28 @@ class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
"Could not import the 'idna' module "
"and the IRI hostname requires encoding"
)
def idna_encoder(name):
if any(ord(c) > 128 for c in name):
else:
def idna_encoder(x):
try:
return idna.encode(
name.lower(), strict=True, std3_rules=True
)
return idna.encode(x, strict=True, std3_rules=True).lower()
except idna.IDNAError:
raise exceptions.InvalidAuthority(self.authority)
return name
authority = ""
if self.host:
authority = ".".join(
[compat.to_str(idna_encoder(part)) for part in self.host.split(".")]
)
authority = ".".join([compat.to_str(idna_encoder(part))
for part in self.host.split(".")])
if self.userinfo is not None:
authority = (
normalizers.encode_component(self.userinfo, self.encoding)
+ "@"
+ authority
)
authority = (normalizers.encode_component(
self.userinfo, self.encoding) + '@' + authority)
if self.port is not None:
authority += ":" + str(self.port)
return uri.URIReference(
self.scheme,
authority,
path=self.path,
query=self.query,
fragment=self.fragment,
encoding=self.encoding,
)
return uri.URIReference(self.scheme,
authority,
path=self.path,
query=self.query,
fragment=self.fragment,
encoding=self.encoding)

105
lib/urllib3/packages/rfc3986/misc.py Executable file → Normal file
View File

@@ -25,16 +25,16 @@ from . import abnf_regexp
# These are enumerated for the named tuple used as a superclass of
# URIReference
URI_COMPONENTS = ["scheme", "authority", "path", "query", "fragment"]
URI_COMPONENTS = ['scheme', 'authority', 'path', 'query', 'fragment']
important_characters = {
"generic_delimiters": abnf_regexp.GENERIC_DELIMITERS,
"sub_delimiters": abnf_regexp.SUB_DELIMITERS,
'generic_delimiters': abnf_regexp.GENERIC_DELIMITERS,
'sub_delimiters': abnf_regexp.SUB_DELIMITERS,
# We need to escape the '*' in this case
"re_sub_delimiters": abnf_regexp.SUB_DELIMITERS_RE,
"unreserved_chars": abnf_regexp.UNRESERVED_CHARS,
're_sub_delimiters': abnf_regexp.SUB_DELIMITERS_RE,
'unreserved_chars': abnf_regexp.UNRESERVED_CHARS,
# We need to escape the '-' in this case:
"re_unreserved": abnf_regexp.UNRESERVED_RE,
're_unreserved': abnf_regexp.UNRESERVED_RE,
}
# For details about delimiters and reserved characters, see:
@@ -49,21 +49,23 @@ NON_PCT_ENCODED = abnf_regexp.NON_PCT_ENCODED_SET
URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE)
SUBAUTHORITY_MATCHER = re.compile(
(
"^(?:(?P<userinfo>{0})@)?" # userinfo
"(?P<host>{1})" # host
":?(?P<port>{2})?$" # port
).format(abnf_regexp.USERINFO_RE, abnf_regexp.HOST_PATTERN, abnf_regexp.PORT_RE)
)
SUBAUTHORITY_MATCHER = re.compile((
'^(?:(?P<userinfo>{0})@)?' # userinfo
'(?P<host>{1})' # host
':?(?P<port>{2})?$' # port
).format(abnf_regexp.USERINFO_RE,
abnf_regexp.HOST_PATTERN,
abnf_regexp.PORT_RE))
HOST_MATCHER = re.compile("^" + abnf_regexp.HOST_RE + "$")
IPv4_MATCHER = re.compile("^" + abnf_regexp.IPv4_RE + "$")
IPv6_MATCHER = re.compile(r"^\[" + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r"\]$")
HOST_MATCHER = re.compile('^' + abnf_regexp.HOST_RE + '$')
IPv4_MATCHER = re.compile('^' + abnf_regexp.IPv4_RE + '$')
IPv6_MATCHER = re.compile(r'^\[' + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r'\]$')
# Used by host validator
IPv6_NO_RFC4007_MATCHER = re.compile(r"^\[%s\]$" % (abnf_regexp.IPv6_ADDRZ_RE))
IPv6_NO_RFC4007_MATCHER = re.compile(r'^\[%s\]$' % (
abnf_regexp.IPv6_ADDRZ_RE
))
# Matcher used to validate path components
PATH_MATCHER = re.compile(abnf_regexp.PATH_RE)
@@ -78,22 +80,20 @@ QUERY_MATCHER = re.compile(abnf_regexp.QUERY_RE)
FRAGMENT_MATCHER = QUERY_MATCHER
# Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1
SCHEME_MATCHER = re.compile("^{0}$".format(abnf_regexp.SCHEME_RE))
SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE))
RELATIVE_REF_MATCHER = re.compile(
r"^%s(\?%s)?(#%s)?$"
% (abnf_regexp.RELATIVE_PART_RE, abnf_regexp.QUERY_RE, abnf_regexp.FRAGMENT_RE)
)
RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % (
abnf_regexp.RELATIVE_PART_RE,
abnf_regexp.QUERY_RE,
abnf_regexp.FRAGMENT_RE,
))
# See http://tools.ietf.org/html/rfc3986#section-4.3
ABSOLUTE_URI_MATCHER = re.compile(
r"^%s:%s(\?%s)?$"
% (
abnf_regexp.COMPONENT_PATTERN_DICT["scheme"],
abnf_regexp.HIER_PART_RE,
abnf_regexp.QUERY_RE[1:-1],
)
)
ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % (
abnf_regexp.COMPONENT_PATTERN_DICT['scheme'],
abnf_regexp.HIER_PART_RE,
abnf_regexp.QUERY_RE[1:-1],
))
# ###############
# IRIs / RFC 3987
@@ -101,25 +101,46 @@ ABSOLUTE_URI_MATCHER = re.compile(
IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE)
ISUBAUTHORITY_MATCHER = re.compile(
(
u"^(?:(?P<userinfo>{0})@)?" # iuserinfo
u"(?P<host>{1})" # ihost
u":?(?P<port>{2})?$" # port
).format(abnf_regexp.IUSERINFO_RE, abnf_regexp.IHOST_RE, abnf_regexp.PORT_RE),
re.UNICODE,
)
ISUBAUTHORITY_MATCHER = re.compile((
u'^(?:(?P<userinfo>{0})@)?' # iuserinfo
u'(?P<host>{1})' # ihost
u':?(?P<port>{2})?$' # port
).format(abnf_regexp.IUSERINFO_RE,
abnf_regexp.IHOST_RE,
abnf_regexp.PORT_RE), re.UNICODE)
IHOST_MATCHER = re.compile('^' + abnf_regexp.IHOST_RE + '$', re.UNICODE)
IPATH_MATCHER = re.compile(abnf_regexp.IPATH_RE, re.UNICODE)
IQUERY_MATCHER = re.compile(abnf_regexp.IQUERY_RE, re.UNICODE)
IFRAGMENT_MATCHER = re.compile(abnf_regexp.IFRAGMENT_RE, re.UNICODE)
RELATIVE_IRI_MATCHER = re.compile(u'^%s(?:\\?%s)?(?:%s)?$' % (
abnf_regexp.IRELATIVE_PART_RE,
abnf_regexp.IQUERY_RE,
abnf_regexp.IFRAGMENT_RE
), re.UNICODE)
ABSOLUTE_IRI_MATCHER = re.compile(u'^%s:%s(?:\\?%s)?$' % (
abnf_regexp.COMPONENT_PATTERN_DICT['scheme'],
abnf_regexp.IHIER_PART_RE,
abnf_regexp.IQUERY_RE[1:-1]
), re.UNICODE)
# Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3
def merge_paths(base_uri, relative_path):
"""Merge a base URI's path with a relative URI's path."""
if base_uri.path is None and base_uri.authority is not None:
return "/" + relative_path
return '/' + relative_path
else:
path = base_uri.path or ""
index = path.rfind("/")
return path[:index] + "/" + relative_path
path = base_uri.path or ''
index = path.rfind('/')
return path[:index] + '/' + relative_path
UseExisting = object()

53
lib/urllib3/packages/rfc3986/normalizers.py Executable file → Normal file
View File

@@ -27,13 +27,13 @@ def normalize_scheme(scheme):
def normalize_authority(authority):
"""Normalize an authority tuple to a string."""
userinfo, host, port = authority
result = ""
result = ''
if userinfo:
result += normalize_percent_characters(userinfo) + "@"
result += normalize_percent_characters(userinfo) + '@'
if host:
result += normalize_host(host)
if port:
result += ":" + port
result += ':' + port
return result
@@ -50,19 +50,16 @@ def normalize_password(password):
def normalize_host(host):
"""Normalize a host string."""
if misc.IPv6_MATCHER.match(host):
percent = host.find("%")
percent = host.find('%')
if percent != -1:
percent_25 = host.find("%25")
percent_25 = host.find('%25')
# Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25'
# from RFC 6874. If the host is '[<IPv6 addr>%25]' then we
# assume RFC 4007 and normalize to '[<IPV6 addr>%2525]'
if (
percent_25 == -1
or percent < percent_25
or (percent == percent_25 and percent_25 == len(host) - 4)
):
host = host.replace("%", "%25", 1)
if percent_25 == -1 or percent < percent_25 or \
(percent == percent_25 and percent_25 == len(host) - 4):
host = host.replace('%', '%25', 1)
# Don't normalize the casing of the Zone ID
return host[:percent].lower() + host[percent:]
@@ -93,7 +90,7 @@ def normalize_fragment(fragment):
return normalize_percent_characters(fragment)
PERCENT_MATCHER = re.compile("%[A-Fa-f0-9]{2}")
PERCENT_MATCHER = re.compile('%[A-Fa-f0-9]{2}')
def normalize_percent_characters(s):
@@ -114,15 +111,15 @@ def remove_dot_segments(s):
See also Section 5.2.4 of :rfc:`3986`.
"""
# See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
segments = s.split("/") # Turn the path into a list of segments
segments = s.split('/') # Turn the path into a list of segments
output = [] # Initialize the variable to use to store output
for segment in segments:
# '.' is the current directory, so ignore it, it is superfluous
if segment == ".":
if segment == '.':
continue
# Anything other than '..', should be appended to the output
elif segment != "..":
elif segment != '..':
output.append(segment)
# In this case segment == '..', if we can, we should pop the last
# element
@@ -131,15 +128,15 @@ def remove_dot_segments(s):
# If the path starts with '/' and the output is empty or the first string
# is non-empty
if s.startswith("/") and (not output or output[0]):
output.insert(0, "")
if s.startswith('/') and (not output or output[0]):
output.insert(0, '')
# If the path starts with '/.' or '/..' ensure we add one more empty
# string to add a trailing '/'
if s.endswith(("/.", "/..")):
output.append("")
if s.endswith(('/.', '/..')):
output.append('')
return "/".join(output)
return '/'.join(output)
def encode_component(uri_component, encoding):
@@ -149,24 +146,22 @@ def encode_component(uri_component, encoding):
# Try to see if the component we're encoding is already percent-encoded
# so we can skip all '%' characters but still encode all others.
percent_encodings = len(
PERCENT_MATCHER.findall(compat.to_str(uri_component, encoding))
)
percent_encodings = len(PERCENT_MATCHER.findall(
compat.to_str(uri_component, encoding)))
uri_bytes = compat.to_bytes(uri_component, encoding)
is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
is_percent_encoded = percent_encodings == uri_bytes.count(b'%')
encoded_uri = bytearray()
for i in range(0, len(uri_bytes)):
# Will return a single character bytestring on both Python 2 & 3
byte = uri_bytes[i : i + 1]
byte = uri_bytes[i:i+1]
byte_ord = ord(byte)
if (is_percent_encoded and byte == b"%") or (
byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED
):
if ((is_percent_encoded and byte == b'%')
or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)):
encoded_uri.extend(byte)
continue
encoded_uri.extend("%{0:02x}".format(byte_ord).encode().upper())
encoded_uri.extend('%{0:02x}'.format(byte_ord).encode().upper())
return encoded_uri.decode(encoding)

326
lib/urllib3/packages/rfc3986/parseresult.py Executable file → Normal file
View File

@@ -21,25 +21,27 @@ from . import misc
from . import normalizers
from . import uri
__all__ = ("ParseResult", "ParseResultBytes")
__all__ = ('ParseResult', 'ParseResultBytes')
PARSED_COMPONENTS = ("scheme", "userinfo", "host", "port", "path", "query", "fragment")
PARSED_COMPONENTS = ('scheme', 'userinfo', 'host', 'port', 'path', 'query',
'fragment')
class ParseResultMixin(object):
def _generate_authority(self, attributes):
# I swear I did not align the comparisons below. That's just how they
# happened to align based on pep8 and attribute lengths.
userinfo, host, port = (attributes[p] for p in ("userinfo", "host", "port"))
if self.userinfo != userinfo or self.host != host or self.port != port:
userinfo, host, port = (attributes[p]
for p in ('userinfo', 'host', 'port'))
if (self.userinfo != userinfo or
self.host != host or
self.port != port):
if port:
port = "{0}".format(port)
port = '{0}'.format(port)
return normalizers.normalize_authority(
(
compat.to_str(userinfo, self.encoding),
compat.to_str(host, self.encoding),
port,
)
(compat.to_str(userinfo, self.encoding),
compat.to_str(host, self.encoding),
port)
)
return self.authority
@@ -63,7 +65,8 @@ class ParseResultMixin(object):
return self.query
class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin):
class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS),
ParseResultMixin):
"""Implementation of urlparse compatibility class.
This uses the URIReference logic to handle compatibility with the
@@ -72,18 +75,8 @@ class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin
slots = ()
def __new__(
cls,
scheme,
userinfo,
host,
port,
path,
query,
fragment,
uri_ref,
encoding="utf-8",
):
def __new__(cls, scheme, userinfo, host, port, path, query, fragment,
uri_ref, encoding='utf-8'):
"""Create a new ParseResult."""
parse_result = super(ParseResult, cls).__new__(
cls,
@@ -93,57 +86,42 @@ class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin
port or None,
path or None,
query,
fragment,
)
fragment)
parse_result.encoding = encoding
parse_result.reference = uri_ref
return parse_result
@classmethod
def from_parts(
cls,
scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
encoding="utf-8",
):
def from_parts(cls, scheme=None, userinfo=None, host=None, port=None,
path=None, query=None, fragment=None, encoding='utf-8'):
"""Create a ParseResult instance from its parts."""
authority = ""
authority = ''
if userinfo is not None:
authority += userinfo + "@"
authority += userinfo + '@'
if host is not None:
authority += host
if port is not None:
authority += ":{0}".format(port)
uri_ref = uri.URIReference(
scheme=scheme,
authority=authority,
path=path,
query=query,
fragment=fragment,
encoding=encoding,
).normalize()
authority += ':{0}'.format(port)
uri_ref = uri.URIReference(scheme=scheme,
authority=authority,
path=path,
query=query,
fragment=fragment,
encoding=encoding).normalize()
userinfo, host, port = authority_from(uri_ref, strict=True)
return cls(
scheme=uri_ref.scheme,
userinfo=userinfo,
host=host,
port=port,
path=uri_ref.path,
query=uri_ref.query,
fragment=uri_ref.fragment,
uri_ref=uri_ref,
encoding=encoding,
)
return cls(scheme=uri_ref.scheme,
userinfo=userinfo,
host=host,
port=port,
path=uri_ref.path,
query=uri_ref.query,
fragment=uri_ref.fragment,
uri_ref=uri_ref,
encoding=encoding)
@classmethod
def from_string(
cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
):
def from_string(cls, uri_string, encoding='utf-8', strict=True,
lazy_normalize=True):
"""Parse a URI from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference.
@@ -158,65 +136,53 @@ class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin
reference = reference.normalize()
userinfo, host, port = authority_from(reference, strict)
return cls(
scheme=reference.scheme,
userinfo=userinfo,
host=host,
port=port,
path=reference.path,
query=reference.query,
fragment=reference.fragment,
uri_ref=reference,
encoding=encoding,
)
return cls(scheme=reference.scheme,
userinfo=userinfo,
host=host,
port=port,
path=reference.path,
query=reference.query,
fragment=reference.fragment,
uri_ref=reference,
encoding=encoding)
@property
def authority(self):
"""Return the normalized authority."""
return self.reference.authority
def copy_with(
self,
scheme=misc.UseExisting,
userinfo=misc.UseExisting,
host=misc.UseExisting,
port=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
):
def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting,
host=misc.UseExisting, port=misc.UseExisting,
path=misc.UseExisting, query=misc.UseExisting,
fragment=misc.UseExisting):
"""Create a copy of this instance replacing with specified parts."""
attributes = zip(
PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment)
)
attributes = zip(PARSED_COMPONENTS,
(scheme, userinfo, host, port, path, query, fragment))
attrs_dict = {}
for name, value in attributes:
if value is misc.UseExisting:
value = getattr(self, name)
attrs_dict[name] = value
authority = self._generate_authority(attrs_dict)
ref = self.reference.copy_with(
scheme=attrs_dict["scheme"],
authority=authority,
path=attrs_dict["path"],
query=attrs_dict["query"],
fragment=attrs_dict["fragment"],
)
ref = self.reference.copy_with(scheme=attrs_dict['scheme'],
authority=authority,
path=attrs_dict['path'],
query=attrs_dict['query'],
fragment=attrs_dict['fragment'])
return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict)
def encode(self, encoding=None):
"""Convert to an instance of ParseResultBytes."""
encoding = encoding or self.encoding
attrs = dict(
zip(
PARSED_COMPONENTS,
(
attr.encode(encoding) if hasattr(attr, "encode") else attr
for attr in self
),
)
zip(PARSED_COMPONENTS,
(attr.encode(encoding) if hasattr(attr, 'encode') else attr
for attr in self)))
return ParseResultBytes(
uri_ref=self.reference,
encoding=encoding,
**attrs
)
return ParseResultBytes(uri_ref=self.reference, encoding=encoding, **attrs)
def unsplit(self, use_idna=False):
"""Create a URI string from the components.
@@ -226,30 +192,18 @@ class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin
"""
parse_result = self
if use_idna and self.host:
hostbytes = self.host.encode("idna")
hostbytes = self.host.encode('idna')
host = hostbytes.decode(self.encoding)
parse_result = self.copy_with(host=host)
return parse_result.reference.unsplit()
class ParseResultBytes(
namedtuple("ParseResultBytes", PARSED_COMPONENTS), ParseResultMixin
):
class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS),
ParseResultMixin):
"""Compatibility shim for the urlparse.ParseResultBytes object."""
def __new__(
cls,
scheme,
userinfo,
host,
port,
path,
query,
fragment,
uri_ref,
encoding="utf-8",
lazy_normalize=True,
):
def __new__(cls, scheme, userinfo, host, port, path, query, fragment,
uri_ref, encoding='utf-8', lazy_normalize=True):
"""Create a new ParseResultBytes instance."""
parse_result = super(ParseResultBytes, cls).__new__(
cls,
@@ -259,63 +213,48 @@ class ParseResultBytes(
port or None,
path or None,
query or None,
fragment or None,
)
fragment or None)
parse_result.encoding = encoding
parse_result.reference = uri_ref
parse_result.lazy_normalize = lazy_normalize
return parse_result
@classmethod
def from_parts(
cls,
scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
encoding="utf-8",
lazy_normalize=True,
):
def from_parts(cls, scheme=None, userinfo=None, host=None, port=None,
path=None, query=None, fragment=None, encoding='utf-8',
lazy_normalize=True):
"""Create a ParseResult instance from its parts."""
authority = ""
authority = ''
if userinfo is not None:
authority += userinfo + "@"
authority += userinfo + '@'
if host is not None:
authority += host
if port is not None:
authority += ":{0}".format(int(port))
uri_ref = uri.URIReference(
scheme=scheme,
authority=authority,
path=path,
query=query,
fragment=fragment,
encoding=encoding,
)
authority += ':{0}'.format(int(port))
uri_ref = uri.URIReference(scheme=scheme,
authority=authority,
path=path,
query=query,
fragment=fragment,
encoding=encoding)
if not lazy_normalize:
uri_ref = uri_ref.normalize()
to_bytes = compat.to_bytes
userinfo, host, port = authority_from(uri_ref, strict=True)
return cls(
scheme=to_bytes(scheme, encoding),
userinfo=to_bytes(userinfo, encoding),
host=to_bytes(host, encoding),
port=port,
path=to_bytes(path, encoding),
query=to_bytes(query, encoding),
fragment=to_bytes(fragment, encoding),
uri_ref=uri_ref,
encoding=encoding,
lazy_normalize=lazy_normalize,
)
return cls(scheme=to_bytes(scheme, encoding),
userinfo=to_bytes(userinfo, encoding),
host=to_bytes(host, encoding),
port=port,
path=to_bytes(path, encoding),
query=to_bytes(query, encoding),
fragment=to_bytes(fragment, encoding),
uri_ref=uri_ref,
encoding=encoding,
lazy_normalize=lazy_normalize)
@classmethod
def from_string(
cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
):
def from_string(cls, uri_string, encoding='utf-8', strict=True,
lazy_normalize=True):
"""Parse a URI from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference.
@@ -331,54 +270,44 @@ class ParseResultBytes(
userinfo, host, port = authority_from(reference, strict)
to_bytes = compat.to_bytes
return cls(
scheme=to_bytes(reference.scheme, encoding),
userinfo=to_bytes(userinfo, encoding),
host=to_bytes(host, encoding),
port=port,
path=to_bytes(reference.path, encoding),
query=to_bytes(reference.query, encoding),
fragment=to_bytes(reference.fragment, encoding),
uri_ref=reference,
encoding=encoding,
lazy_normalize=lazy_normalize,
)
return cls(scheme=to_bytes(reference.scheme, encoding),
userinfo=to_bytes(userinfo, encoding),
host=to_bytes(host, encoding),
port=port,
path=to_bytes(reference.path, encoding),
query=to_bytes(reference.query, encoding),
fragment=to_bytes(reference.fragment, encoding),
uri_ref=reference,
encoding=encoding,
lazy_normalize=lazy_normalize)
@property
def authority(self):
"""Return the normalized authority."""
return self.reference.authority.encode(self.encoding)
def copy_with(
self,
scheme=misc.UseExisting,
userinfo=misc.UseExisting,
host=misc.UseExisting,
port=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
lazy_normalize=True,
):
def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting,
host=misc.UseExisting, port=misc.UseExisting,
path=misc.UseExisting, query=misc.UseExisting,
fragment=misc.UseExisting, lazy_normalize=True):
"""Create a copy of this instance replacing with specified parts."""
attributes = zip(
PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment)
)
attributes = zip(PARSED_COMPONENTS,
(scheme, userinfo, host, port, path, query, fragment))
attrs_dict = {}
for name, value in attributes:
if value is misc.UseExisting:
value = getattr(self, name)
if not isinstance(value, bytes) and hasattr(value, "encode"):
if not isinstance(value, bytes) and hasattr(value, 'encode'):
value = value.encode(self.encoding)
attrs_dict[name] = value
authority = self._generate_authority(attrs_dict)
to_str = compat.to_str
ref = self.reference.copy_with(
scheme=to_str(attrs_dict["scheme"], self.encoding),
scheme=to_str(attrs_dict['scheme'], self.encoding),
authority=to_str(authority, self.encoding),
path=to_str(attrs_dict["path"], self.encoding),
query=to_str(attrs_dict["query"], self.encoding),
fragment=to_str(attrs_dict["fragment"], self.encoding),
path=to_str(attrs_dict['path'], self.encoding),
query=to_str(attrs_dict['query'], self.encoding),
fragment=to_str(attrs_dict['fragment'], self.encoding)
)
if not lazy_normalize:
ref = ref.normalize()
@@ -400,7 +329,7 @@ class ParseResultBytes(
# self.host is bytes, to encode to idna, we need to decode it
# first
host = self.host.decode(self.encoding)
hostbytes = host.encode("idna")
hostbytes = host.encode('idna')
parse_result = self.copy_with(host=hostbytes)
if self.lazy_normalize:
parse_result = parse_result.copy_with(lazy_normalize=False)
@@ -416,16 +345,16 @@ def split_authority(authority):
# Set-up rest in case there is no userinfo portion
rest = authority
if "@" in authority:
userinfo, rest = authority.rsplit("@", 1)
if '@' in authority:
userinfo, rest = authority.rsplit('@', 1)
# Handle IPv6 host addresses
if rest.startswith("["):
host, rest = rest.split("]", 1)
host += "]"
if rest.startswith('['):
host, rest = rest.split(']', 1)
host += ']'
if ":" in rest:
extra_host, port = rest.split(":", 1)
if ':' in rest:
extra_host, port = rest.split(':', 1)
elif not host and rest:
host = rest
@@ -445,9 +374,8 @@ def authority_from(reference, strict):
else:
# Thanks to Richard Barrell for this idea:
# https://twitter.com/0x2ba22e11/status/617338811975139328
userinfo, host, port = (
subauthority.get(p) for p in ("userinfo", "host", "port")
)
userinfo, host, port = (subauthority.get(p)
for p in ('userinfo', 'host', 'port'))
if port:
try:

45
lib/urllib3/packages/rfc3986/uri.py Executable file → Normal file
View File

@@ -22,7 +22,7 @@ from . import normalizers
from ._mixin import URIMixin
class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin):
"""Immutable object representing a parsed URI Reference.
.. note::
@@ -82,11 +82,16 @@ class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
slots = ()
def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"):
def __new__(cls, scheme, authority, path, query, fragment,
encoding='utf-8'):
"""Create a new URIReference."""
ref = super(URIReference, cls).__new__(
cls, scheme or None, authority or None, path or None, query, fragment
)
cls,
scheme or None,
authority or None,
path or None,
query,
fragment)
ref.encoding = encoding
return ref
@@ -102,10 +107,8 @@ class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
other_ref = URIReference.from_string(other)
except TypeError:
raise TypeError(
"Unable to compare URIReference() to {0}()".format(
type(other).__name__
)
)
'Unable to compare URIReference() to {0}()'.format(
type(other).__name__))
# See http://tools.ietf.org/html/rfc3986#section-6.2
naive_equality = tuple(self) == tuple(other_ref)
@@ -122,17 +125,16 @@ class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
"""
# See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in
# this method.
return URIReference(
normalizers.normalize_scheme(self.scheme or ""),
normalizers.normalize_authority((self.userinfo, self.host, self.port)),
normalizers.normalize_path(self.path or ""),
normalizers.normalize_query(self.query),
normalizers.normalize_fragment(self.fragment),
self.encoding,
)
return URIReference(normalizers.normalize_scheme(self.scheme or ''),
normalizers.normalize_authority(
(self.userinfo, self.host, self.port)),
normalizers.normalize_path(self.path or ''),
normalizers.normalize_query(self.query),
normalizers.normalize_fragment(self.fragment),
self.encoding)
@classmethod
def from_string(cls, uri_string, encoding="utf-8"):
def from_string(cls, uri_string, encoding='utf-8'):
"""Parse a URI reference from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference.
@@ -143,10 +145,9 @@ class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
split_uri = misc.URI_MATCHER.match(uri_string).groupdict()
return cls(
split_uri["scheme"],
split_uri["authority"],
normalizers.encode_component(split_uri["path"], encoding),
normalizers.encode_component(split_uri["query"], encoding),
normalizers.encode_component(split_uri["fragment"], encoding),
split_uri['scheme'], split_uri['authority'],
normalizers.encode_component(split_uri['path'], encoding),
normalizers.encode_component(split_uri['query'], encoding),
normalizers.encode_component(split_uri['fragment'], encoding),
encoding,
)

91
lib/urllib3/packages/rfc3986/validators.py Executable file → Normal file
View File

@@ -45,9 +45,15 @@ class Validator(object):
"""
COMPONENT_NAMES = frozenset(
["scheme", "userinfo", "host", "port", "path", "query", "fragment"]
)
COMPONENT_NAMES = frozenset([
'scheme',
'userinfo',
'host',
'port',
'path',
'query',
'fragment',
])
def __init__(self):
"""Initialize our default validations."""
@@ -56,13 +62,13 @@ class Validator(object):
self.allowed_ports = set()
self.allow_password = True
self.required_components = {
"scheme": False,
"userinfo": False,
"host": False,
"port": False,
"path": False,
"query": False,
"fragment": False,
'scheme': False,
'userinfo': False,
'host': False,
'port': False,
'path': False,
'query': False,
'fragment': False,
}
self.validated_components = self.required_components.copy()
@@ -159,8 +165,12 @@ class Validator(object):
components = [c.lower() for c in components]
for component in components:
if component not in self.COMPONENT_NAMES:
raise ValueError('"{}" is not a valid component'.format(component))
self.validated_components.update({component: True for component in components})
raise ValueError(
'"{}" is not a valid component'.format(component)
)
self.validated_components.update({
component: True for component in components
})
return self
def require_presence_of(self, *components):
@@ -180,8 +190,12 @@ class Validator(object):
components = [c.lower() for c in components]
for component in components:
if component not in self.COMPONENT_NAMES:
raise ValueError('"{}" is not a valid component'.format(component))
self.required_components.update({component: True for component in components})
raise ValueError(
'"{}" is not a valid component'.format(component)
)
self.required_components.update({
component: True for component in components
})
return self
def validate(self, uri):
@@ -221,9 +235,9 @@ class Validator(object):
if validated_components:
ensure_components_are_valid(uri, validated_components)
ensure_one_of(self.allowed_schemes, uri, "scheme")
ensure_one_of(self.allowed_hosts, uri, "host")
ensure_one_of(self.allowed_ports, uri, "port")
ensure_one_of(self.allowed_schemes, uri, 'scheme')
ensure_one_of(self.allowed_hosts, uri, 'host')
ensure_one_of(self.allowed_ports, uri, 'port')
def check_password(uri):
@@ -231,7 +245,7 @@ def check_password(uri):
userinfo = uri.userinfo
if not userinfo:
return
credentials = userinfo.split(":", 1)
credentials = userinfo.split(':', 1)
if len(credentials) <= 1:
return
raise exceptions.PasswordForbidden(uri)
@@ -241,18 +255,18 @@ def ensure_one_of(allowed_values, uri, attribute):
"""Assert that the uri's attribute is one of the allowed values."""
value = getattr(uri, attribute)
if value is not None and allowed_values and value not in allowed_values:
raise exceptions.UnpermittedComponentError(attribute, value, allowed_values)
raise exceptions.UnpermittedComponentError(
attribute, value, allowed_values,
)
def ensure_required_components_exist(uri, required_components):
"""Assert that all required components are present in the URI."""
missing_components = sorted(
[
component
for component in required_components
if getattr(uri, component) is None
]
)
missing_components = sorted([
component
for component in required_components
if getattr(uri, component) is None
])
if missing_components:
raise exceptions.MissingComponentError(uri, *missing_components)
@@ -268,7 +282,8 @@ def is_valid(value, matcher, require):
Whether or not the value is required.
"""
if require:
return value is not None and matcher.match(value)
return (value is not None
and matcher.match(value))
# require is False and value is not None
return value is None or matcher.match(value)
@@ -378,17 +393,17 @@ def valid_ipv4_host_address(host):
"""Determine if the given host is a valid IPv4 address."""
# If the host exists, and it might be IPv4, check each byte in the
# address.
return all([0 <= int(byte, base=10) <= 255 for byte in host.split(".")])
return all([0 <= int(byte, base=10) <= 255 for byte in host.split('.')])
_COMPONENT_VALIDATORS = {
"scheme": scheme_is_valid,
"path": path_is_valid,
"query": query_is_valid,
"fragment": fragment_is_valid,
'scheme': scheme_is_valid,
'path': path_is_valid,
'query': query_is_valid,
'fragment': fragment_is_valid,
}
_SUBAUTHORITY_VALIDATORS = set(["userinfo", "host", "port"])
_SUBAUTHORITY_VALIDATORS = set(['userinfo', 'host', 'port'])
def subauthority_component_is_valid(uri, component):
@@ -400,19 +415,19 @@ def subauthority_component_is_valid(uri, component):
# If we can parse the authority into sub-components and we're not
# validating the port, we can assume it's valid.
if component == "host":
return host_is_valid(subauthority_dict["host"])
elif component != "port":
if component == 'host':
return host_is_valid(subauthority_dict['host'])
elif component != 'port':
return True
try:
port = int(subauthority_dict["port"])
port = int(subauthority_dict['port'])
except TypeError:
# If the port wasn't provided it'll be None and int(None) raises a
# TypeError
return True
return 0 <= port <= 65535
return (0 <= port <= 65535)
def ensure_components_are_valid(uri, validated_components):

203
lib/urllib3/packages/six.py Executable file → Normal file
View File

@@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)
if PY3:
string_types = (str,)
integer_types = (int,)
class_types = (type,)
string_types = str,
integer_types = int,
class_types = type,
text_type = str
binary_type = bytes
MAXSIZE = sys.maxsize
else:
string_types = (basestring,)
string_types = basestring,
integer_types = (int, long)
class_types = (type, types.ClassType)
text_type = unicode
@@ -58,9 +58,9 @@ else:
else:
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
class X(object):
def __len__(self):
return 1 << 31
try:
len(X())
except OverflowError:
@@ -84,6 +84,7 @@ def _import_module(name):
class _LazyDescr(object):
def __init__(self, name):
self.name = name
@@ -100,6 +101,7 @@ class _LazyDescr(object):
class MovedModule(_LazyDescr):
def __init__(self, name, old, new=None):
super(MovedModule, self).__init__(name)
if PY3:
@@ -120,6 +122,7 @@ class MovedModule(_LazyDescr):
class _LazyModule(types.ModuleType):
def __init__(self, name):
super(_LazyModule, self).__init__(name)
self.__doc__ = self.__class__.__doc__
@@ -134,6 +137,7 @@ class _LazyModule(types.ModuleType):
class MovedAttribute(_LazyDescr):
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
super(MovedAttribute, self).__init__(name)
if PY3:
@@ -217,35 +221,28 @@ class _SixMetaPathImporter(object):
Required, if is_package is implemented"""
self.__get_module(fullname) # eventually raises ImportError
return None
get_source = get_code # same as get_code
_importer = _SixMetaPathImporter(__name__)
class _MovedItems(_LazyModule):
"""Lazy loading of moved objects"""
__path__ = [] # mark as package
_moved_attributes = [
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
MovedAttribute(
"filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"
),
MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
MovedAttribute("intern", "__builtin__", "sys"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute(
"reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"
),
MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
MovedAttribute("reduce", "__builtin__", "functools"),
MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
MovedAttribute("StringIO", "StringIO", "io"),
@@ -254,9 +251,7 @@ _moved_attributes = [
MovedAttribute("UserString", "UserString", "collections"),
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
MovedAttribute(
"zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"
),
MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
MovedModule("builtins", "__builtin__"),
MovedModule("configparser", "ConfigParser"),
MovedModule("copyreg", "copy_reg"),
@@ -268,9 +263,7 @@ _moved_attributes = [
MovedModule("html_parser", "HTMLParser", "html.parser"),
MovedModule("http_client", "httplib", "http.client"),
MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
MovedModule(
"email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"
),
MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
@@ -290,12 +283,15 @@ _moved_attributes = [
MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"),
MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"),
MovedModule("tkinter_colorchooser", "tkColorChooser",
"tkinter.colorchooser"),
MovedModule("tkinter_commondialog", "tkCommonDialog",
"tkinter.commondialog"),
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
MovedModule("tkinter_font", "tkFont", "tkinter.font"),
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"),
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
"tkinter.simpledialog"),
MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
@@ -305,7 +301,9 @@ _moved_attributes = [
]
# Add windows specific modules.
if sys.platform == "win32":
_moved_attributes += [MovedModule("winreg", "_winreg")]
_moved_attributes += [
MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr)
@@ -355,11 +353,8 @@ del attr
Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
_importer._add_module(
Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
"moves.urllib_parse",
"moves.urllib.parse",
)
_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
"moves.urllib_parse", "moves.urllib.parse")
class Module_six_moves_urllib_error(_LazyModule):
@@ -378,11 +373,8 @@ del attr
Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
_importer._add_module(
Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
"moves.urllib_error",
"moves.urllib.error",
)
_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
"moves.urllib_error", "moves.urllib.error")
class Module_six_moves_urllib_request(_LazyModule):
@@ -431,11 +423,8 @@ del attr
Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
_importer._add_module(
Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
"moves.urllib_request",
"moves.urllib.request",
)
_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
"moves.urllib_request", "moves.urllib.request")
class Module_six_moves_urllib_response(_LazyModule):
@@ -455,11 +444,8 @@ del attr
Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
_importer._add_module(
Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
"moves.urllib_response",
"moves.urllib.response",
)
_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
"moves.urllib_response", "moves.urllib.response")
class Module_six_moves_urllib_robotparser(_LazyModule):
@@ -468,27 +454,21 @@ class Module_six_moves_urllib_robotparser(_LazyModule):
_urllib_robotparser_moved_attributes = [
MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser")
MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
]
for attr in _urllib_robotparser_moved_attributes:
setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr
Module_six_moves_urllib_robotparser._moved_attributes = (
_urllib_robotparser_moved_attributes
)
Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
_importer._add_module(
Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
"moves.urllib_robotparser",
"moves.urllib.robotparser",
)
_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
"moves.urllib_robotparser", "moves.urllib.robotparser")
class Module_six_moves_urllib(types.ModuleType):
"""Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
__path__ = [] # mark as package
parse = _importer._get_module("moves.urllib_parse")
error = _importer._get_module("moves.urllib_error")
@@ -497,12 +477,10 @@ class Module_six_moves_urllib(types.ModuleType):
robotparser = _importer._get_module("moves.urllib_robotparser")
def __dir__(self):
return ["parse", "error", "request", "response", "robotparser"]
return ['parse', 'error', 'request', 'response', 'robotparser']
_importer._add_module(
Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib"
)
_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
"moves.urllib")
def add_move(move):
@@ -542,24 +520,19 @@ else:
try:
advance_iterator = next
except NameError:
def advance_iterator(it):
return it.next()
next = advance_iterator
try:
callable = callable
except NameError:
def callable(obj):
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
if PY3:
def get_unbound_function(unbound):
return unbound
@@ -570,7 +543,6 @@ if PY3:
Iterator = object
else:
def get_unbound_function(unbound):
return unbound.im_func
@@ -581,13 +553,13 @@ else:
return types.MethodType(func, None, cls)
class Iterator(object):
def next(self):
return type(self).__next__(self)
callable = callable
_add_doc(
get_unbound_function, """Get the function out of a possibly unbound function"""
)
_add_doc(get_unbound_function,
"""Get the function out of a possibly unbound function""")
get_method_function = operator.attrgetter(_meth_func)
@@ -599,7 +571,6 @@ get_function_globals = operator.attrgetter(_func_globals)
if PY3:
def iterkeys(d, **kw):
return iter(d.keys(**kw))
@@ -618,7 +589,6 @@ if PY3:
viewitems = operator.methodcaller("items")
else:
def iterkeys(d, **kw):
return d.iterkeys(**kw)
@@ -639,30 +609,26 @@ else:
_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(
iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary."
)
_add_doc(iteritems,
"Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(iterlists,
"Return an iterator over the (key, [values]) pairs of a dictionary.")
if PY3:
def b(s):
return s.encode("latin-1")
def u(s):
return s
unichr = chr
import struct
int2byte = struct.Struct(">B").pack
del struct
byte2int = operator.itemgetter(0)
indexbytes = operator.getitem
iterbytes = iter
import io
StringIO = io.StringIO
BytesIO = io.BytesIO
_assertCountEqual = "assertCountEqual"
@@ -673,15 +639,12 @@ if PY3:
_assertRaisesRegex = "assertRaisesRegex"
_assertRegex = "assertRegex"
else:
def b(s):
return s
# Workaround for standalone backslash
def u(s):
return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape")
return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
unichr = unichr
int2byte = chr
@@ -690,10 +653,8 @@ else:
def indexbytes(buf, i):
return ord(buf[i])
iterbytes = functools.partial(itertools.imap, ord)
import StringIO
StringIO = BytesIO = StringIO.StringIO
_assertCountEqual = "assertItemsEqual"
_assertRaisesRegex = "assertRaisesRegexp"
@@ -724,9 +685,7 @@ if PY3:
raise value.with_traceback(tb)
raise value
else:
def exec_(_code_, _globs_=None, _locs_=None):
"""Execute code in a namespace."""
if _globs_ is None:
@@ -739,36 +698,28 @@ else:
_locs_ = _globs_
exec("""exec _code_ in _globs_, _locs_""")
exec_(
"""def reraise(tp, value, tb=None):
exec_("""def reraise(tp, value, tb=None):
raise tp, value, tb
"""
)
""")
if sys.version_info[:2] == (3, 2):
exec_(
"""def raise_from(value, from_value):
exec_("""def raise_from(value, from_value):
if from_value is None:
raise value
raise value from from_value
"""
)
""")
elif sys.version_info[:2] > (3, 2):
exec_(
"""def raise_from(value, from_value):
exec_("""def raise_from(value, from_value):
raise value from from_value
"""
)
""")
else:
def raise_from(value, from_value):
raise value
print_ = getattr(moves.builtins, "print", None)
if print_ is None:
def print_(*args, **kwargs):
"""The new-style print function for Python 2.4 and 2.5."""
fp = kwargs.pop("file", sys.stdout)
@@ -779,17 +730,14 @@ if print_ is None:
if not isinstance(data, basestring):
data = str(data)
# If the file has an encoding, encode unicode with it.
if (
isinstance(fp, file)
and isinstance(data, unicode)
and fp.encoding is not None
):
if (isinstance(fp, file) and
isinstance(data, unicode) and
fp.encoding is not None):
errors = getattr(fp, "errors", None)
if errors is None:
errors = "strict"
data = data.encode(fp.encoding, errors)
fp.write(data)
want_unicode = False
sep = kwargs.pop("sep", None)
if sep is not None:
@@ -825,8 +773,6 @@ if print_ is None:
write(sep)
write(arg)
write(end)
if sys.version_info[:2] < (3, 3):
_print = print_
@@ -837,24 +783,16 @@ if sys.version_info[:2] < (3, 3):
if flush and fp is not None:
fp.flush()
_add_doc(reraise, """Reraise an exception.""")
if sys.version_info[0:2] < (3, 4):
def wraps(
wrapped,
assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES,
):
def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES):
def wrapper(f):
f = functools.wraps(wrapped, assigned, updated)(f)
f.__wrapped__ = wrapped
return f
return wrapper
else:
wraps = functools.wraps
@@ -865,27 +803,25 @@ def with_metaclass(meta, *bases):
# metaclass for one level of class instantiation that replaces itself with
# the actual metaclass.
class metaclass(meta):
def __new__(cls, name, this_bases, d):
return meta(name, bases, d)
return type.__new__(metaclass, "temporary_class", (), {})
return type.__new__(metaclass, 'temporary_class', (), {})
def add_metaclass(metaclass):
"""Class decorator for creating a class with a metaclass."""
def wrapper(cls):
orig_vars = cls.__dict__.copy()
slots = orig_vars.get("__slots__")
slots = orig_vars.get('__slots__')
if slots is not None:
if isinstance(slots, str):
slots = [slots]
for slots_var in slots:
orig_vars.pop(slots_var)
orig_vars.pop("__dict__", None)
orig_vars.pop("__weakref__", None)
orig_vars.pop('__dict__', None)
orig_vars.pop('__weakref__', None)
return metaclass(cls.__name__, cls.__bases__, orig_vars)
return wrapper
@@ -898,13 +834,12 @@ def python_2_unicode_compatible(klass):
returning text and apply this decorator to the class.
"""
if PY2:
if "__str__" not in klass.__dict__:
raise ValueError(
"@python_2_unicode_compatible cannot be applied "
"to %s because it doesn't define __str__()." % klass.__name__
)
if '__str__' not in klass.__dict__:
raise ValueError("@python_2_unicode_compatible cannot be applied "
"to %s because it doesn't define __str__()." %
klass.__name__)
klass.__unicode__ = klass.__str__
klass.__str__ = lambda self: self.__unicode__().encode("utf-8")
klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
return klass
@@ -924,10 +859,8 @@ if sys.meta_path:
# be floating around. Therefore, we can't use isinstance() to check for
# the six meta path importer, since the other six instance will have
# inserted an importer with different class.
if (
type(importer).__name__ == "_SixMetaPathImporter"
and importer.name == __name__
):
if (type(importer).__name__ == "_SixMetaPathImporter" and
importer.name == __name__):
del sys.meta_path[i]
break
del i, importer

2
lib/urllib3/packages/ssl_match_hostname/__init__.py Executable file → Normal file
View File

@@ -16,4 +16,4 @@ except ImportError:
from ._implementation import CertificateError, match_hostname
# Not needed, but documenting what we provide.
__all__ = ("CertificateError", "match_hostname")
__all__ = ('CertificateError', 'match_hostname')

58
lib/urllib3/packages/ssl_match_hostname/_implementation.py Executable file → Normal file
View File

@@ -15,7 +15,7 @@ try:
except ImportError:
ipaddress = None
__version__ = "3.5.0.1"
__version__ = '3.5.0.1'
class CertificateError(ValueError):
@@ -33,19 +33,18 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
# Ported from python3-syntax:
# leftmost, *remainder = dn.split(r'.')
parts = dn.split(r".")
parts = dn.split(r'.')
leftmost = parts[0]
remainder = parts[1:]
wildcards = leftmost.count("*")
wildcards = leftmost.count('*')
if wildcards > max_wildcards:
# Issue #17980: avoid denials of service by refusing more
# than one wildcard per fragment. A survey of established
# policy among SSL implementations showed it to be a
# reasonable choice.
raise CertificateError(
"too many wildcards in certificate DNS name: " + repr(dn)
)
"too many wildcards in certificate DNS name: " + repr(dn))
# speed up common case w/o wildcards
if not wildcards:
@@ -54,11 +53,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
# RFC 6125, section 6.4.3, subitem 1.
# The client SHOULD NOT attempt to match a presented identifier in which
# the wildcard character comprises a label other than the left-most label.
if leftmost == "*":
if leftmost == '*':
# When '*' is a fragment by itself, it matches a non-empty dotless
# fragment.
pats.append("[^.]+")
elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
pats.append('[^.]+')
elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
# RFC 6125, section 6.4.3, subitem 3.
# The client SHOULD NOT attempt to match a presented identifier
# where the wildcard character is embedded within an A-label or
@@ -66,22 +65,21 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
pats.append(re.escape(leftmost))
else:
# Otherwise, '*' matches any dotless string, e.g. www*
pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
# add the remaining fragments, ignore any wildcards
for frag in remainder:
pats.append(re.escape(frag))
pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
return pat.match(hostname)
def _to_unicode(obj):
if isinstance(obj, str) and sys.version_info < (3,):
obj = unicode(obj, encoding="ascii", errors="strict")
obj = unicode(obj, encoding='ascii', errors='strict')
return obj
def _ipaddress_match(ipname, host_ip):
"""Exact matching of IP addresses.
@@ -103,11 +101,9 @@ def match_hostname(cert, hostname):
returns nothing.
"""
if not cert:
raise ValueError(
"empty or no certificate, match_hostname needs a "
"SSL socket or SSL context with either "
"CERT_OPTIONAL or CERT_REQUIRED"
)
raise ValueError("empty or no certificate, match_hostname needs a "
"SSL socket or SSL context with either "
"CERT_OPTIONAL or CERT_REQUIRED")
try:
# Divergence from upstream: ipaddress can't handle byte str
host_ip = ipaddress.ip_address(_to_unicode(hostname))
@@ -126,37 +122,35 @@ def match_hostname(cert, hostname):
else:
raise
dnsnames = []
san = cert.get("subjectAltName", ())
san = cert.get('subjectAltName', ())
for key, value in san:
if key == "DNS":
if key == 'DNS':
if host_ip is None and _dnsname_match(value, hostname):
return
dnsnames.append(value)
elif key == "IP Address":
elif key == 'IP Address':
if host_ip is not None and _ipaddress_match(value, host_ip):
return
dnsnames.append(value)
if not dnsnames:
# The subject is only checked when there is no dNSName entry
# in subjectAltName
for sub in cert.get("subject", ()):
for sub in cert.get('subject', ()):
for key, value in sub:
# XXX according to RFC 2818, the most specific Common Name
# must be used.
if key == "commonName":
if key == 'commonName':
if _dnsname_match(value, hostname):
return
dnsnames.append(value)
if len(dnsnames) > 1:
raise CertificateError(
"hostname %r "
"doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
)
raise CertificateError("hostname %r "
"doesn't match either of %s"
% (hostname, ', '.join(map(repr, dnsnames))))
elif len(dnsnames) == 1:
raise CertificateError(
"hostname %r " "doesn't match %r" % (hostname, dnsnames[0])
)
raise CertificateError("hostname %r "
"doesn't match %r"
% (hostname, dnsnames[0]))
else:
raise CertificateError(
"no appropriate commonName or " "subjectAltName fields were found"
)
raise CertificateError("no appropriate commonName or "
"subjectAltName fields were found")

183
lib/urllib3/poolmanager.py Executable file → Normal file
View File

@@ -14,55 +14,48 @@ from .util.url import parse_url
from .util.retry import Retry
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
log = logging.getLogger(__name__)
SSL_KEYWORDS = (
"key_file",
"cert_file",
"cert_reqs",
"ca_certs",
"ssl_version",
"ca_cert_dir",
"ssl_context",
"key_password",
)
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
'ssl_version', 'ca_cert_dir', 'ssl_context',
'key_password')
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = (
"key_scheme", # str
"key_host", # str
"key_port", # int
"key_timeout", # int or float or Timeout
"key_retries", # int or Retry
"key_strict", # bool
"key_block", # bool
"key_source_address", # str
"key_key_file", # str
"key_key_password", # str
"key_cert_file", # str
"key_cert_reqs", # str
"key_ca_certs", # str
"key_ssl_version", # str
"key_ca_cert_dir", # str
"key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
"key_maxsize", # int
"key_headers", # dict
"key__proxy", # parsed proxy url
"key__proxy_headers", # dict
"key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
"key__socks_options", # dict
"key_assert_hostname", # bool or string
"key_assert_fingerprint", # str
"key_server_hostname", # str
'key_scheme', # str
'key_host', # str
'key_port', # int
'key_timeout', # int or float or Timeout
'key_retries', # int or Retry
'key_strict', # bool
'key_block', # bool
'key_source_address', # str
'key_key_file', # str
'key_key_password', # str
'key_cert_file', # str
'key_cert_reqs', # str
'key_ca_certs', # str
'key_ssl_version', # str
'key_ca_cert_dir', # str
'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
'key_maxsize', # int
'key_headers', # dict
'key__proxy', # parsed proxy url
'key__proxy_headers', # dict
'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples
'key__socks_options', # dict
'key_assert_hostname', # bool or string
'key_assert_fingerprint', # str
'key_server_hostname', # str
)
#: The namedtuple class used to construct keys for the connection pool.
#: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple("PoolKey", _key_fields)
PoolKey = collections.namedtuple('PoolKey', _key_fields)
def _default_key_normalizer(key_class, request_context):
@@ -87,24 +80,24 @@ def _default_key_normalizer(key_class, request_context):
"""
# Since we mutate the dictionary, make a copy first
context = request_context.copy()
context["scheme"] = context["scheme"].lower()
context["host"] = context["host"].lower()
context['scheme'] = context['scheme'].lower()
context['host'] = context['host'].lower()
# These are both dictionaries and need to be transformed into frozensets
for key in ("headers", "_proxy_headers", "_socks_options"):
for key in ('headers', '_proxy_headers', '_socks_options'):
if key in context and context[key] is not None:
context[key] = frozenset(context[key].items())
# The socket_options key may be a list and needs to be transformed into a
# tuple.
socket_opts = context.get("socket_options")
socket_opts = context.get('socket_options')
if socket_opts is not None:
context["socket_options"] = tuple(socket_opts)
context['socket_options'] = tuple(socket_opts)
# Map the kwargs to the names in the namedtuple - this is necessary since
# namedtuples can't have fields starting with '_'.
for key in list(context.keys()):
context["key_" + key] = context.pop(key)
context['key_' + key] = context.pop(key)
# Default to ``None`` for keys missing from the context
for field in key_class._fields:
@@ -119,11 +112,14 @@ def _default_key_normalizer(key_class, request_context):
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(_default_key_normalizer, PoolKey),
"https": functools.partial(_default_key_normalizer, PoolKey),
'http': functools.partial(_default_key_normalizer, PoolKey),
'https': functools.partial(_default_key_normalizer, PoolKey),
}
pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': HTTPSConnectionPool,
}
class PoolManager(RequestMethods):
@@ -159,7 +155,8 @@ class PoolManager(RequestMethods):
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
RequestMethods.__init__(self, headers)
self.connection_pool_kw = connection_pool_kw
self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close())
self.pools = RecentlyUsedContainer(num_pools,
dispose_func=lambda p: p.close())
# Locally set the pool classes and keys so other PoolManagers can
# override them.
@@ -192,10 +189,10 @@ class PoolManager(RequestMethods):
# this function has historically only used the scheme, host, and port
# in the positional args. When an API change is acceptable these can
# be removed.
for key in ("scheme", "host", "port"):
for key in ('scheme', 'host', 'port'):
request_context.pop(key, None)
if scheme == "http":
if scheme == 'http':
for kw in SSL_KEYWORDS:
request_context.pop(kw, None)
@@ -210,7 +207,7 @@ class PoolManager(RequestMethods):
"""
self.pools.clear()
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
"""
Get a :class:`ConnectionPool` based on the host, port, and scheme.
@@ -225,11 +222,11 @@ class PoolManager(RequestMethods):
raise LocationValueError("No host specified.")
request_context = self._merge_pool_kwargs(pool_kwargs)
request_context["scheme"] = scheme or "http"
request_context['scheme'] = scheme or 'http'
if not port:
port = port_by_scheme.get(request_context["scheme"].lower(), 80)
request_context["port"] = port
request_context["host"] = host
port = port_by_scheme.get(request_context['scheme'].lower(), 80)
request_context['port'] = port
request_context['host'] = host
return self.connection_from_context(request_context)
@@ -240,7 +237,7 @@ class PoolManager(RequestMethods):
``request_context`` must at least contain the ``scheme`` key and its
value must be a key in ``key_fn_by_scheme`` instance variable.
"""
scheme = request_context["scheme"].lower()
scheme = request_context['scheme'].lower()
pool_key_constructor = self.key_fn_by_scheme[scheme]
pool_key = pool_key_constructor(request_context)
@@ -262,9 +259,9 @@ class PoolManager(RequestMethods):
return pool
# Make a fresh ConnectionPool of the desired type
scheme = request_context["scheme"]
host = request_context["host"]
port = request_context["port"]
scheme = request_context['scheme']
host = request_context['host']
port = request_context['port']
pool = self._new_pool(scheme, host, port, request_context=request_context)
self.pools[pool_key] = pool
@@ -282,9 +279,8 @@ class PoolManager(RequestMethods):
not used.
"""
u = parse_url(url)
return self.connection_from_host(
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
)
return self.connection_from_host(u.host, port=u.port, scheme=u.scheme,
pool_kwargs=pool_kwargs)
def _merge_pool_kwargs(self, override):
"""
@@ -318,11 +314,11 @@ class PoolManager(RequestMethods):
u = parse_url(url)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw["assert_same_host"] = False
kw["redirect"] = False
kw['assert_same_host'] = False
kw['redirect'] = False
if "headers" not in kw:
kw["headers"] = self.headers.copy()
if 'headers' not in kw:
kw['headers'] = self.headers.copy()
if self.proxy is not None and u.scheme == "http":
response = conn.urlopen(method, url, **kw)
@@ -338,22 +334,21 @@ class PoolManager(RequestMethods):
# RFC 7231, Section 6.4.4
if response.status == 303:
method = "GET"
method = 'GET'
retries = kw.get("retries")
retries = kw.get('retries')
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect)
# Strip headers marked as unsafe to forward to the redirected location.
# Check remove_headers_on_redirect to avoid a potential network call within
# conn.is_same_host() which may use socket.gethostbyname() in the future.
if retries.remove_headers_on_redirect and not conn.is_same_host(
redirect_location
):
headers = list(six.iterkeys(kw["headers"]))
if (retries.remove_headers_on_redirect
and not conn.is_same_host(redirect_location)):
headers = list(six.iterkeys(kw['headers']))
for header in headers:
if header.lower() in retries.remove_headers_on_redirect:
kw["headers"].pop(header, None)
kw['headers'].pop(header, None)
try:
retries = retries.increment(method, url, response=response, _pool=conn)
@@ -362,8 +357,8 @@ class PoolManager(RequestMethods):
raise
return response
kw["retries"] = retries
kw["redirect"] = redirect
kw['retries'] = retries
kw['redirect'] = redirect
log.info("Redirecting %s -> %s", url, redirect_location)
return self.urlopen(method, redirect_location, **kw)
@@ -396,21 +391,12 @@ class ProxyManager(PoolManager):
"""
def __init__(
self,
proxy_url,
num_pools=10,
headers=None,
proxy_headers=None,
**connection_pool_kw
):
def __init__(self, proxy_url, num_pools=10, headers=None,
proxy_headers=None, **connection_pool_kw):
if isinstance(proxy_url, HTTPConnectionPool):
proxy_url = "%s://%s:%i" % (
proxy_url.scheme,
proxy_url.host,
proxy_url.port,
)
proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
proxy_url.port)
proxy = parse_url(proxy_url)
if not proxy.port:
port = port_by_scheme.get(proxy.scheme, 80)
@@ -422,31 +408,30 @@ class ProxyManager(PoolManager):
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw["_proxy_headers"] = self.proxy_headers
connection_pool_kw['_proxy'] = self.proxy
connection_pool_kw['_proxy_headers'] = self.proxy_headers
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
super(ProxyManager, self).__init__(
num_pools, headers, **connection_pool_kw)
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
if scheme == "https":
return super(ProxyManager, self).connection_from_host(
host, port, scheme, pool_kwargs=pool_kwargs
)
host, port, scheme, pool_kwargs=pool_kwargs)
return super(ProxyManager, self).connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
)
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs)
def _set_proxy_headers(self, url, headers=None):
"""
Sets headers needed by proxies: specifically, the Accept and Host
headers. Only sets headers not provided by the user.
"""
headers_ = {"Accept": "*/*"}
headers_ = {'Accept': '*/*'}
netloc = parse_url(url).netloc
if netloc:
headers_["Host"] = netloc
headers_['Host'] = netloc
if headers:
headers_.update(headers)
@@ -460,8 +445,8 @@ class ProxyManager(PoolManager):
# For proxied HTTPS requests, httplib sets the necessary headers
# on the CONNECT to the proxy. For HTTP, we'll definitely
# need to set 'Host' at the very least.
headers = kw.get("headers", self.headers)
kw["headers"] = self._set_proxy_headers(url, headers)
headers = kw.get('headers', self.headers)
kw['headers'] = self._set_proxy_headers(url, headers)
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)

79
lib/urllib3/request.py Executable file → Normal file
View File

@@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata
from .packages.six.moves.urllib.parse import urlencode
__all__ = ["RequestMethods"]
__all__ = ['RequestMethods']
class RequestMethods(object):
@@ -36,25 +36,16 @@ class RequestMethods(object):
explicitly.
"""
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
_encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'}
def __init__(self, headers=None):
self.headers = headers or {}
def urlopen(
self,
method,
url,
body=None,
headers=None,
encode_multipart=True,
multipart_boundary=None,
**kw
): # Abstract
raise NotImplementedError(
"Classes extending RequestMethods must implement "
"their own ``urlopen`` method."
)
def urlopen(self, method, url, body=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**kw): # Abstract
raise NotImplementedError("Classes extending RequestMethods must implement "
"their own ``urlopen`` method.")
def request(self, method, url, fields=None, headers=None, **urlopen_kw):
"""
@@ -69,18 +60,19 @@ class RequestMethods(object):
"""
method = method.upper()
urlopen_kw["request_url"] = url
urlopen_kw['request_url'] = url
if method in self._encode_url_methods:
return self.request_encode_url(
method, url, fields=fields, headers=headers, **urlopen_kw
)
return self.request_encode_url(method, url, fields=fields,
headers=headers,
**urlopen_kw)
else:
return self.request_encode_body(
method, url, fields=fields, headers=headers, **urlopen_kw
)
return self.request_encode_body(method, url, fields=fields,
headers=headers,
**urlopen_kw)
def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw):
def request_encode_url(self, method, url, fields=None, headers=None,
**urlopen_kw):
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
@@ -88,24 +80,17 @@ class RequestMethods(object):
if headers is None:
headers = self.headers
extra_kw = {"headers": headers}
extra_kw = {'headers': headers}
extra_kw.update(urlopen_kw)
if fields:
url += "?" + urlencode(fields)
url += '?' + urlencode(fields)
return self.urlopen(method, url, **extra_kw)
def request_encode_body(
self,
method,
url,
fields=None,
headers=None,
encode_multipart=True,
multipart_boundary=None,
**urlopen_kw
):
def request_encode_body(self, method, url, fields=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**urlopen_kw):
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the body. This is useful for request methods like POST, PUT, PATCH, etc.
@@ -144,28 +129,22 @@ class RequestMethods(object):
if headers is None:
headers = self.headers
extra_kw = {"headers": {}}
extra_kw = {'headers': {}}
if fields:
if "body" in urlopen_kw:
if 'body' in urlopen_kw:
raise TypeError(
"request got values for both 'fields' and 'body', can only specify one."
)
"request got values for both 'fields' and 'body', can only specify one.")
if encode_multipart:
body, content_type = encode_multipart_formdata(
fields, boundary=multipart_boundary
)
body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
else:
body, content_type = (
urlencode(fields),
"application/x-www-form-urlencoded",
)
body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
extra_kw["body"] = body
extra_kw["headers"] = {"Content-Type": content_type}
extra_kw['body'] = body
extra_kw['headers'] = {'Content-Type': content_type}
extra_kw["headers"].update(headers)
extra_kw['headers'].update(headers)
extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw)

187
lib/urllib3/response.py Executable file → Normal file
View File

@@ -13,13 +13,8 @@ except ImportError:
from ._collections import HTTPHeaderDict
from .exceptions import (
BodyNotHttplibCompatible,
ProtocolError,
DecodeError,
ReadTimeoutError,
ResponseNotChunked,
IncompleteRead,
InvalidHeader,
BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
ResponseNotChunked, IncompleteRead, InvalidHeader
)
from .packages.six import string_types as basestring, PY3
from .packages.six.moves import http_client as httplib
@@ -30,9 +25,10 @@ log = logging.getLogger(__name__)
class DeflateDecoder(object):
def __init__(self):
self._first_try = True
self._data = b""
self._data = b''
self._obj = zlib.decompressobj()
def __getattr__(self, name):
@@ -69,6 +65,7 @@ class GzipDecoderState(object):
class GzipDecoder(object):
def __init__(self):
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
self._state = GzipDecoderState.FIRST_MEMBER
@@ -99,23 +96,15 @@ class GzipDecoder(object):
if brotli is not None:
class BrotliDecoder(object):
# Supports both 'brotlipy' and 'Brotli' packages
# since they share an import name. The top branches
# are for 'brotlipy' and bottom branches for 'Brotli'
def __init__(self):
self._obj = brotli.Decompressor()
def decompress(self, data):
if hasattr(self._obj, "decompress"):
return self._obj.decompress(data)
return self._obj.process(data)
def __getattr__(self, name):
return getattr(self._obj, name)
def flush(self):
if hasattr(self._obj, "flush"):
return self._obj.flush()
return b""
def decompress(self, data):
return self._obj.decompress(data)
class MultiDecoder(object):
@@ -128,7 +117,7 @@ class MultiDecoder(object):
"""
def __init__(self, modes):
self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')]
def flush(self):
return self._decoders[0].flush()
@@ -140,13 +129,13 @@ class MultiDecoder(object):
def _get_decoder(mode):
if "," in mode:
if ',' in mode:
return MultiDecoder(mode)
if mode == "gzip":
if mode == 'gzip':
return GzipDecoder()
if brotli is not None and mode == "br":
if brotli is not None and mode == 'br':
return BrotliDecoder()
return DeflateDecoder()
@@ -185,30 +174,16 @@ class HTTPResponse(io.IOBase):
value of Content-Length header, if present. Otherwise, raise error.
"""
CONTENT_DECODERS = ["gzip", "deflate"]
CONTENT_DECODERS = ['gzip', 'deflate']
if brotli is not None:
CONTENT_DECODERS += ["br"]
CONTENT_DECODERS += ['br']
REDIRECT_STATUSES = [301, 302, 303, 307, 308]
def __init__(
self,
body="",
headers=None,
status=0,
version=0,
reason=None,
strict=0,
preload_content=True,
decode_content=True,
original_response=None,
pool=None,
connection=None,
msg=None,
retries=None,
enforce_content_length=False,
request_method=None,
request_url=None,
):
def __init__(self, body='', headers=None, status=0, version=0, reason=None,
strict=0, preload_content=True, decode_content=True,
original_response=None, pool=None, connection=None, msg=None,
retries=None, enforce_content_length=False,
request_method=None, request_url=None):
if isinstance(headers, HTTPHeaderDict):
self.headers = headers
@@ -236,13 +211,13 @@ class HTTPResponse(io.IOBase):
self._pool = pool
self._connection = connection
if hasattr(body, "read"):
if hasattr(body, 'read'):
self._fp = body
# Are we using the chunked-style of transfer encoding?
self.chunked = False
self.chunk_left = None
tr_enc = self.headers.get("transfer-encoding", "").lower()
tr_enc = self.headers.get('transfer-encoding', '').lower()
# Don't incur the penalty of creating a list and then discarding it
encodings = (enc.strip() for enc in tr_enc.split(","))
if "chunked" in encodings:
@@ -264,7 +239,7 @@ class HTTPResponse(io.IOBase):
location. ``False`` if not a redirect status code.
"""
if self.status in self.REDIRECT_STATUSES:
return self.headers.get("location")
return self.headers.get('location')
return False
@@ -303,20 +278,18 @@ class HTTPResponse(io.IOBase):
"""
Set initial length value for Response content if available.
"""
length = self.headers.get("content-length")
length = self.headers.get('content-length')
if length is not None:
if self.chunked:
# This Response will fail with an IncompleteRead if it can't be
# received as chunked. This method falls back to attempt reading
# the response before raising an exception.
log.warning(
"Received response with both Content-Length and "
"Transfer-Encoding set. This is expressly forbidden "
"by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
"attempting to process response as Transfer-Encoding: "
"chunked."
)
log.warning("Received response with both Content-Length and "
"Transfer-Encoding set. This is expressly forbidden "
"by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
"attempting to process response as Transfer-Encoding: "
"chunked.")
return None
try:
@@ -325,12 +298,10 @@ class HTTPResponse(io.IOBase):
# (e.g. Content-Length: 42, 42). This line ensures the values
# are all valid ints and that as long as the `set` length is 1,
# all values are the same. Otherwise, the header is invalid.
lengths = set([int(val) for val in length.split(",")])
lengths = set([int(val) for val in length.split(',')])
if len(lengths) > 1:
raise InvalidHeader(
"Content-Length contained multiple "
"unmatching values (%s)" % length
)
raise InvalidHeader("Content-Length contained multiple "
"unmatching values (%s)" % length)
length = lengths.pop()
except ValueError:
length = None
@@ -346,7 +317,7 @@ class HTTPResponse(io.IOBase):
status = 0
# Check for responses that shouldn't include a body
if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
length = 0
return length
@@ -357,22 +328,20 @@ class HTTPResponse(io.IOBase):
"""
# Note: content-encoding value should be case-insensitive, per RFC 7230
# Section 3.2
content_encoding = self.headers.get("content-encoding", "").lower()
content_encoding = self.headers.get('content-encoding', '').lower()
if self._decoder is None:
if content_encoding in self.CONTENT_DECODERS:
self._decoder = _get_decoder(content_encoding)
elif "," in content_encoding:
elif ',' in content_encoding:
encodings = [
e.strip()
for e in content_encoding.split(",")
if e.strip() in self.CONTENT_DECODERS
]
e.strip() for e in content_encoding.split(',')
if e.strip() in self.CONTENT_DECODERS]
if len(encodings):
self._decoder = _get_decoder(content_encoding)
DECODER_ERROR_CLASSES = (IOError, zlib.error)
if brotli is not None:
DECODER_ERROR_CLASSES += (brotli.error,)
DECODER_ERROR_CLASSES += (brotli.Error,)
def _decode(self, data, decode_content, flush_decoder):
"""
@@ -385,12 +354,10 @@ class HTTPResponse(io.IOBase):
if self._decoder:
data = self._decoder.decompress(data)
except self.DECODER_ERROR_CLASSES as e:
content_encoding = self.headers.get("content-encoding", "").lower()
content_encoding = self.headers.get('content-encoding', '').lower()
raise DecodeError(
"Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding,
e,
)
"failed to decode it." % content_encoding, e)
if flush_decoder:
data += self._flush_decoder()
@@ -402,10 +369,10 @@ class HTTPResponse(io.IOBase):
being used.
"""
if self._decoder:
buf = self._decoder.decompress(b"")
buf = self._decoder.decompress(b'')
return buf + self._decoder.flush()
return b""
return b''
@contextmanager
def _error_catcher(self):
@@ -425,20 +392,20 @@ class HTTPResponse(io.IOBase):
except SocketTimeout:
# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
# there is yet no clean way to get at it from this context.
raise ReadTimeoutError(self._pool, None, "Read timed out.")
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
except BaseSSLError as e:
# FIXME: Is there a better way to differentiate between SSLErrors?
if "read operation timed out" not in str(e): # Defensive:
if 'read operation timed out' not in str(e): # Defensive:
# This shouldn't happen but just in case we're missing an edge
# case, let's avoid swallowing SSL errors.
raise
raise ReadTimeoutError(self._pool, None, "Read timed out.")
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
except (HTTPException, SocketError) as e:
# This includes IncompleteRead.
raise ProtocolError("Connection broken: %r" % e, e)
raise ProtocolError('Connection broken: %r' % e, e)
# If no exception is thrown, we should avoid cleaning up
# unnecessarily.
@@ -503,9 +470,7 @@ class HTTPResponse(io.IOBase):
else:
cache_content = False
data = self._fp.read(amt)
if (
amt != 0 and not data
): # Platform-specific: Buggy versions of Python.
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
# Close the connection when no data is returned
#
# This is redundant to what httplib/http.client _should_
@@ -515,10 +480,7 @@ class HTTPResponse(io.IOBase):
# no harm in redundantly calling close.
self._fp.close()
flush_decoder = True
if self.enforce_content_length and self.length_remaining not in (
0,
None,
):
if self.enforce_content_length and self.length_remaining not in (0, None):
# This is an edge case that httplib failed to cover due
# to concerns of backward compatibility. We're
# addressing it here to make sure IncompleteRead is
@@ -538,7 +500,7 @@ class HTTPResponse(io.IOBase):
return data
def stream(self, amt=2 ** 16, decode_content=None):
def stream(self, amt=2**16, decode_content=None):
"""
A generator wrapper for the read() method. A call will block until
``amt`` bytes have been read from the connection or until the
@@ -583,17 +545,15 @@ class HTTPResponse(io.IOBase):
headers = HTTPHeaderDict.from_httplib(headers)
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, "strict", 0)
resp = ResponseCls(
body=r,
headers=headers,
status=r.status,
version=r.version,
reason=r.reason,
strict=strict,
original_response=r,
**response_kw
)
strict = getattr(r, 'strict', 0)
resp = ResponseCls(body=r,
headers=headers,
status=r.status,
version=r.version,
reason=r.reason,
strict=strict,
original_response=r,
**response_kw)
return resp
# Backwards-compatibility methods for httplib.HTTPResponse
@@ -619,9 +579,9 @@ class HTTPResponse(io.IOBase):
def closed(self):
if self._fp is None:
return True
elif hasattr(self._fp, "isclosed"):
elif hasattr(self._fp, 'isclosed'):
return self._fp.isclosed()
elif hasattr(self._fp, "closed"):
elif hasattr(self._fp, 'closed'):
return self._fp.closed
else:
return True
@@ -632,13 +592,11 @@ class HTTPResponse(io.IOBase):
elif hasattr(self._fp, "fileno"):
return self._fp.fileno()
else:
raise IOError(
"The file-like object this HTTPResponse is wrapped "
"around has no file descriptor"
)
raise IOError("The file-like object this HTTPResponse is wrapped "
"around has no file descriptor")
def flush(self):
if self._fp is not None and hasattr(self._fp, "flush"):
if self._fp is not None and hasattr(self._fp, 'flush'):
return self._fp.flush()
def readable(self):
@@ -651,7 +609,7 @@ class HTTPResponse(io.IOBase):
if len(temp) == 0:
return 0
else:
b[: len(temp)] = temp
b[:len(temp)] = temp
return len(temp)
def supports_chunked_reads(self):
@@ -661,7 +619,7 @@ class HTTPResponse(io.IOBase):
attribute. If it is present we assume it returns raw chunks as
processed by read_chunked().
"""
return hasattr(self._fp, "fp")
return hasattr(self._fp, 'fp')
def _update_chunk_length(self):
# First, we'll figure out length of a chunk and then
@@ -669,7 +627,7 @@ class HTTPResponse(io.IOBase):
if self.chunk_left is not None:
return
line = self._fp.fp.readline()
line = line.split(b";", 1)[0]
line = line.split(b';', 1)[0]
try:
self.chunk_left = int(line, 16)
except ValueError:
@@ -718,13 +676,11 @@ class HTTPResponse(io.IOBase):
if not self.chunked:
raise ResponseNotChunked(
"Response is not chunked. "
"Header 'transfer-encoding: chunked' is missing."
)
"Header 'transfer-encoding: chunked' is missing.")
if not self.supports_chunked_reads():
raise BodyNotHttplibCompatible(
"Body should be httplib.HTTPResponse like. "
"It should have have an fp attribute which returns raw chunks."
)
"It should have have an fp attribute which returns raw chunks.")
with self._error_catcher():
# Don't bother reading the body of a HEAD request.
@@ -742,9 +698,8 @@ class HTTPResponse(io.IOBase):
if self.chunk_left == 0:
break
chunk = self._handle_chunk(amt)
decoded = self._decode(
chunk, decode_content=decode_content, flush_decoder=False
)
decoded = self._decode(chunk, decode_content=decode_content,
flush_decoder=False)
if decoded:
yield decoded
@@ -762,7 +717,7 @@ class HTTPResponse(io.IOBase):
if not line:
# Some sites may not end with '\r\n'.
break
if line == b"\r\n":
if line == b'\r\n':
break
# We read everything; close the "file".

60
lib/urllib3/util/__init__.py Executable file → Normal file
View File

@@ -1,5 +1,4 @@
from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
from .request import make_headers
@@ -15,32 +14,43 @@ from .ssl_ import (
ssl_wrap_socket,
PROTOCOL_TLS,
)
from .timeout import current_time, Timeout
from .timeout import (
current_time,
Timeout,
)
from .retry import Retry
from .url import get_host, parse_url, split_first, Url
from .wait import wait_for_read, wait_for_write
from .url import (
get_host,
parse_url,
split_first,
Url,
)
from .wait import (
wait_for_read,
wait_for_write
)
__all__ = (
"HAS_SNI",
"IS_PYOPENSSL",
"IS_SECURETRANSPORT",
"SSLContext",
"PROTOCOL_TLS",
"Retry",
"Timeout",
"Url",
"assert_fingerprint",
"current_time",
"is_connection_dropped",
"is_fp_closed",
"get_host",
"parse_url",
"make_headers",
"resolve_cert_reqs",
"resolve_ssl_version",
"split_first",
"ssl_wrap_socket",
"wait_for_read",
"wait_for_write",
'HAS_SNI',
'IS_PYOPENSSL',
'IS_SECURETRANSPORT',
'SSLContext',
'PROTOCOL_TLS',
'Retry',
'Timeout',
'Url',
'assert_fingerprint',
'current_time',
'is_connection_dropped',
'is_fp_closed',
'get_host',
'parse_url',
'make_headers',
'resolve_cert_reqs',
'resolve_ssl_version',
'split_first',
'ssl_wrap_socket',
'wait_for_read',
'wait_for_write'
)

16
lib/urllib3/util/connection.py Executable file → Normal file
View File

@@ -14,7 +14,7 @@ def is_connection_dropped(conn): # Platform-specific
Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us.
"""
sock = getattr(conn, "sock", False)
sock = getattr(conn, 'sock', False)
if sock is False: # Platform-specific: AppEngine
return False
if sock is None: # Connection already closed (such as by httplib).
@@ -30,12 +30,8 @@ def is_connection_dropped(conn): # Platform-specific
# library test suite. Added to its signature is only `socket_options`.
# One additional modification is that we avoid binding to IPv6 servers
# discovered in DNS if the system doesn't have IPv6 functionality.
def create_connection(
address,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None,
socket_options=None,
):
def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None, socket_options=None):
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
@@ -49,8 +45,8 @@ def create_connection(
"""
host, port = address
if host.startswith("["):
host = host.strip("[]")
if host.startswith('['):
host = host.strip('[]')
err = None
# Using the value from allowed_gai_family() in the context of getaddrinfo lets
@@ -135,4 +131,4 @@ def _has_ipv6(host):
return has_ipv6
HAS_IPV6 = _has_ipv6("::1")
HAS_IPV6 = _has_ipv6('::1')

0
lib/urllib3/util/queue.py Executable file → Normal file
View File

52
lib/urllib3/util/request.py Executable file → Normal file
View File

@@ -4,25 +4,19 @@ from base64 import b64encode
from ..packages.six import b, integer_types
from ..exceptions import UnrewindableBodyError
ACCEPT_ENCODING = "gzip,deflate"
ACCEPT_ENCODING = 'gzip,deflate'
try:
import brotli as _unused_module_brotli # noqa: F401
except ImportError:
pass
else:
ACCEPT_ENCODING += ",br"
ACCEPT_ENCODING += ',br'
_FAILEDTELL = object()
def make_headers(
keep_alive=None,
accept_encoding=None,
user_agent=None,
basic_auth=None,
proxy_basic_auth=None,
disable_cache=None,
):
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
basic_auth=None, proxy_basic_auth=None, disable_cache=None):
"""
Shortcuts for generating request headers.
@@ -62,27 +56,27 @@ def make_headers(
if isinstance(accept_encoding, str):
pass
elif isinstance(accept_encoding, list):
accept_encoding = ",".join(accept_encoding)
accept_encoding = ','.join(accept_encoding)
else:
accept_encoding = ACCEPT_ENCODING
headers["accept-encoding"] = accept_encoding
headers['accept-encoding'] = accept_encoding
if user_agent:
headers["user-agent"] = user_agent
headers['user-agent'] = user_agent
if keep_alive:
headers["connection"] = "keep-alive"
headers['connection'] = 'keep-alive'
if basic_auth:
headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8")
headers['authorization'] = 'Basic ' + \
b64encode(b(basic_auth)).decode('utf-8')
if proxy_basic_auth:
headers["proxy-authorization"] = "Basic " + b64encode(
b(proxy_basic_auth)
).decode("utf-8")
headers['proxy-authorization'] = 'Basic ' + \
b64encode(b(proxy_basic_auth)).decode('utf-8')
if disable_cache:
headers["cache-control"] = "no-cache"
headers['cache-control'] = 'no-cache'
return headers
@@ -94,7 +88,7 @@ def set_file_position(body, pos):
"""
if pos is not None:
rewind_body(body, pos)
elif getattr(body, "tell", None) is not None:
elif getattr(body, 'tell', None) is not None:
try:
pos = body.tell()
except (IOError, OSError):
@@ -116,20 +110,16 @@ def rewind_body(body, body_pos):
:param int pos:
Position to seek to in file.
"""
body_seek = getattr(body, "seek", None)
body_seek = getattr(body, 'seek', None)
if body_seek is not None and isinstance(body_pos, integer_types):
try:
body_seek(body_pos)
except (IOError, OSError):
raise UnrewindableBodyError(
"An error occurred when rewinding request " "body for redirect/retry."
)
raise UnrewindableBodyError("An error occurred when rewinding request "
"body for redirect/retry.")
elif body_pos is _FAILEDTELL:
raise UnrewindableBodyError(
"Unable to record file position for rewinding "
"request body during a redirect/retry."
)
raise UnrewindableBodyError("Unable to record file position for rewinding "
"request body during a redirect/retry.")
else:
raise ValueError(
"body_pos must be of type integer, " "instead it was %s." % type(body_pos)
)
raise ValueError("body_pos must be of type integer, "
"instead it was %s." % type(body_pos))

9
lib/urllib3/util/response.py Executable file → Normal file
View File

@@ -52,10 +52,11 @@ def assert_header_parsing(headers):
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
raise TypeError("expected httplib.Message, got {0}.".format(type(headers)))
raise TypeError('expected httplib.Message, got {0}.'.format(
type(headers)))
defects = getattr(headers, "defects", None)
get_payload = getattr(headers, "get_payload", None)
defects = getattr(headers, 'defects', None)
get_payload = getattr(headers, 'get_payload', None)
unparsed_data = None
if get_payload:
@@ -83,4 +84,4 @@ def is_response_to_head(response):
method = response._method
if isinstance(method, int): # Platform-specific: Appengine
return method == 3
return method.upper() == "HEAD"
return method.upper() == 'HEAD'

100
lib/urllib3/util/retry.py Executable file → Normal file
View File

@@ -21,9 +21,8 @@ log = logging.getLogger(__name__)
# Data structure for representing the metadata of requests that result in a retry.
RequestHistory = namedtuple(
"RequestHistory", ["method", "url", "error", "status", "redirect_location"]
)
RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
"status", "redirect_location"])
class Retry(object):
@@ -147,33 +146,21 @@ class Retry(object):
request.
"""
DEFAULT_METHOD_WHITELIST = frozenset(
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
)
DEFAULT_METHOD_WHITELIST = frozenset([
'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"])
DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization'])
#: Maximum backoff time.
BACKOFF_MAX = 120
def __init__(
self,
total=10,
connect=None,
read=None,
redirect=None,
status=None,
method_whitelist=DEFAULT_METHOD_WHITELIST,
status_forcelist=None,
backoff_factor=0,
raise_on_redirect=True,
raise_on_status=True,
history=None,
respect_retry_after_header=True,
remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST,
):
def __init__(self, total=10, connect=None, read=None, redirect=None, status=None,
method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
history=None, respect_retry_after_header=True,
remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST):
self.total = total
self.connect = connect
@@ -192,25 +179,20 @@ class Retry(object):
self.raise_on_status = raise_on_status
self.history = history or tuple()
self.respect_retry_after_header = respect_retry_after_header
self.remove_headers_on_redirect = frozenset(
[h.lower() for h in remove_headers_on_redirect]
)
self.remove_headers_on_redirect = frozenset([
h.lower() for h in remove_headers_on_redirect])
def new(self, **kw):
params = dict(
total=self.total,
connect=self.connect,
read=self.read,
redirect=self.redirect,
status=self.status,
connect=self.connect, read=self.read, redirect=self.redirect, status=self.status,
method_whitelist=self.method_whitelist,
status_forcelist=self.status_forcelist,
backoff_factor=self.backoff_factor,
raise_on_redirect=self.raise_on_redirect,
raise_on_status=self.raise_on_status,
history=self.history,
remove_headers_on_redirect=self.remove_headers_on_redirect,
respect_retry_after_header=self.respect_retry_after_header,
remove_headers_on_redirect=self.remove_headers_on_redirect
)
params.update(kw)
return type(self)(**params)
@@ -235,11 +217,8 @@ class Retry(object):
:rtype: float
"""
# We want to consider only the last consecutive errors sequence (Ignore redirects).
consecutive_errors_len = len(
list(
takewhile(lambda x: x.redirect_location is None, reversed(self.history))
)
)
consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
reversed(self.history))))
if consecutive_errors_len <= 1:
return 0
@@ -295,7 +274,7 @@ class Retry(object):
this method will return immediately.
"""
if self.respect_retry_after_header and response:
if response:
slept = self.sleep_for_retry(response)
if slept:
return
@@ -336,12 +315,8 @@ class Retry(object):
if self.status_forcelist and status_code in self.status_forcelist:
return True
return (
self.total
and self.respect_retry_after_header
and has_retry_after
and (status_code in self.RETRY_AFTER_STATUS_CODES)
)
return (self.total and self.respect_retry_after_header and
has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
def is_exhausted(self):
""" Are we out of retries? """
@@ -352,15 +327,8 @@ class Retry(object):
return min(retry_counts) < 0
def increment(
self,
method=None,
url=None,
response=None,
error=None,
_pool=None,
_stacktrace=None,
):
def increment(self, method=None, url=None, response=None, error=None,
_pool=None, _stacktrace=None):
""" Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
@@ -383,7 +351,7 @@ class Retry(object):
read = self.read
redirect = self.redirect
status_count = self.status
cause = "unknown"
cause = 'unknown'
status = None
redirect_location = None
@@ -405,7 +373,7 @@ class Retry(object):
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
cause = 'too many redirects'
redirect_location = response.get_redirect_location()
status = response.status
@@ -416,21 +384,16 @@ class Retry(object):
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
cause = ResponseError.SPECIFIC_ERROR.format(
status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
history=history,
)
connect=connect, read=read, redirect=redirect, status=status_count,
history=history)
if new_retry.is_exhausted():
raise MaxRetryError(_pool, url, error or ResponseError(cause))
@@ -440,10 +403,9 @@ class Retry(object):
return new_retry
def __repr__(self):
return (
"{cls.__name__}(total={self.total}, connect={self.connect}, "
"read={self.read}, redirect={self.redirect}, status={self.status})"
).format(cls=type(self), self=self)
return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
'read={self.read}, redirect={self.redirect}, status={self.status})').format(
cls=type(self), self=self)
# For backwards compatibility (equivalent to pre-v1.9):

157
lib/urllib3/util/ssl_.py Executable file → Normal file
View File

@@ -18,7 +18,11 @@ IS_PYOPENSSL = False
IS_SECURETRANSPORT = False
# Maps the length of a digest to a possible hash function producing this digest
HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256}
HASHFUNC_MAP = {
32: md5,
40: sha1,
64: sha256,
}
def _const_compare_digest_backport(a, b):
@@ -34,13 +38,18 @@ def _const_compare_digest_backport(a, b):
return result == 0
_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport)
_const_compare_digest = getattr(hmac, 'compare_digest',
_const_compare_digest_backport)
# Borrow rfc3986's regular expressions for IPv4
# and IPv6 addresses for use in is_ipaddress()
_IP_ADDRESS_REGEX = re.compile(
r"^(?:%s|%s|%s)$"
% (abnf_regexp.IPv4_RE, abnf_regexp.IPv6_RE, abnf_regexp.IPv6_ADDRZ_RFC4007_RE)
r'^(?:%s|%s|%s|%s)$' % (
abnf_regexp.IPv4_RE,
abnf_regexp.IPv6_RE,
abnf_regexp.IPv6_ADDRZ_RE,
abnf_regexp.IPv_FUTURE_RE
)
)
try: # Test for SSL features
@@ -52,12 +61,10 @@ except ImportError:
try: # Platform-specific: Python 3.6
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
@@ -87,29 +94,26 @@ except ImportError:
# insecure ciphers for security reasons.
# - NOTE: TLS 1.3 cipher suites are managed through a different interface
# not exposed by CPython (yet!) and are enabled by default if they're available.
DEFAULT_CIPHERS = ":".join(
[
"ECDHE+AESGCM",
"ECDHE+CHACHA20",
"DHE+AESGCM",
"DHE+CHACHA20",
"ECDH+AESGCM",
"DH+AESGCM",
"ECDH+AES",
"DH+AES",
"RSA+AESGCM",
"RSA+AES",
"!aNULL",
"!eNULL",
"!MD5",
"!DSS",
]
)
DEFAULT_CIPHERS = ':'.join([
'ECDHE+AESGCM',
'ECDHE+CHACHA20',
'DHE+AESGCM',
'DHE+CHACHA20',
'ECDH+AESGCM',
'DH+AESGCM',
'ECDH+AES',
'DH+AES',
'RSA+AESGCM',
'RSA+AES',
'!aNULL',
'!eNULL',
'!MD5',
'!DSS',
])
try:
from ssl import SSLContext # Modern SSL?
except ImportError:
class SSLContext(object): # Platform-specific: Python 2
def __init__(self, protocol_version):
self.protocol = protocol_version
@@ -137,21 +141,21 @@ except ImportError:
def wrap_socket(self, socket, server_hostname=None, server_side=False):
warnings.warn(
"A true SSLContext object is not available. This prevents "
"urllib3 from configuring SSL appropriately and may cause "
"certain SSL connections to fail. You can upgrade to a newer "
"version of Python to solve this. For more information, see "
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
"#ssl-warnings",
InsecurePlatformWarning,
'A true SSLContext object is not available. This prevents '
'urllib3 from configuring SSL appropriately and may cause '
'certain SSL connections to fail. You can upgrade to a newer '
'version of Python to solve this. For more information, see '
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
'#ssl-warnings',
InsecurePlatformWarning
)
kwargs = {
"keyfile": self.keyfile,
"certfile": self.certfile,
"ca_certs": self.ca_certs,
"cert_reqs": self.verify_mode,
"ssl_version": self.protocol,
"server_side": server_side,
'keyfile': self.keyfile,
'certfile': self.certfile,
'ca_certs': self.ca_certs,
'cert_reqs': self.verify_mode,
'ssl_version': self.protocol,
'server_side': server_side,
}
return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
@@ -166,11 +170,12 @@ def assert_fingerprint(cert, fingerprint):
Fingerprint as string of hexdigits, can be interspersed by colons.
"""
fingerprint = fingerprint.replace(":", "").lower()
fingerprint = fingerprint.replace(':', '').lower()
digest_length = len(fingerprint)
hashfunc = HASHFUNC_MAP.get(digest_length)
if not hashfunc:
raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
raise SSLError(
'Fingerprint of invalid length: {0}'.format(fingerprint))
# We need encode() here for py32; works on py2 and p33.
fingerprint_bytes = unhexlify(fingerprint.encode())
@@ -178,11 +183,8 @@ def assert_fingerprint(cert, fingerprint):
cert_digest = hashfunc(cert).digest()
if not _const_compare_digest(cert_digest, fingerprint_bytes):
raise SSLError(
'Fingerprints did not match. Expected "{0}", got "{1}".'.format(
fingerprint, hexlify(cert_digest)
)
)
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
.format(fingerprint, hexlify(cert_digest)))
def resolve_cert_reqs(candidate):
@@ -202,7 +204,7 @@ def resolve_cert_reqs(candidate):
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "CERT_" + candidate)
res = getattr(ssl, 'CERT_' + candidate)
return res
return candidate
@@ -218,15 +220,14 @@ def resolve_ssl_version(candidate):
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "PROTOCOL_" + candidate)
res = getattr(ssl, 'PROTOCOL_' + candidate)
return res
return candidate
def create_urllib3_context(
ssl_version=None, cert_reqs=None, options=None, ciphers=None
):
def create_urllib3_context(ssl_version=None, cert_reqs=None,
options=None, ciphers=None):
"""All arguments have the same meaning as ``ssl_wrap_socket``.
By default, this function does a lot of the same work that
@@ -280,28 +281,17 @@ def create_urllib3_context(
context.options |= options
context.verify_mode = cert_reqs
if (
getattr(context, "check_hostname", None) is not None
): # Platform-specific: Python 3.2
if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2
# We do our own verification, including fingerprints and alternative
# hostnames. So disable it here
context.check_hostname = False
return context
def ssl_wrap_socket(
sock,
keyfile=None,
certfile=None,
cert_reqs=None,
ca_certs=None,
server_hostname=None,
ssl_version=None,
ciphers=None,
ssl_context=None,
ca_cert_dir=None,
key_password=None,
):
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
ca_certs=None, server_hostname=None,
ssl_version=None, ciphers=None, ssl_context=None,
ca_cert_dir=None, key_password=None):
"""
All arguments except for server_hostname, ssl_context, and ca_cert_dir have
the same meaning as they do when using :func:`ssl.wrap_socket`.
@@ -325,7 +315,8 @@ def ssl_wrap_socket(
# Note: This branch of code and all the variables in it are no longer
# used by urllib3 itself. We should consider deprecating and removing
# this code.
context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
context = create_urllib3_context(ssl_version, cert_reqs,
ciphers=ciphers)
if ca_certs or ca_cert_dir:
try:
@@ -339,7 +330,7 @@ def ssl_wrap_socket(
raise SSLError(e)
raise
elif ssl_context is None and hasattr(context, "load_default_certs"):
elif ssl_context is None and hasattr(context, 'load_default_certs'):
# try to load OS default certs; works well on Windows (require Python3.4+)
context.load_default_certs()
@@ -359,45 +350,43 @@ def ssl_wrap_socket(
# extension should not be used according to RFC3546 Section 3.1
# We shouldn't warn the user if SNI isn't available but we would
# not be using SNI anyways due to IP address for server_hostname.
if (
server_hostname is not None and not is_ipaddress(server_hostname)
) or IS_SECURETRANSPORT:
if ((server_hostname is not None and not is_ipaddress(server_hostname))
or IS_SECURETRANSPORT):
if HAS_SNI and server_hostname is not None:
return context.wrap_socket(sock, server_hostname=server_hostname)
warnings.warn(
"An HTTPS request has been made, but the SNI (Server Name "
"Indication) extension to TLS is not available on this platform. "
"This may cause the server to present an incorrect TLS "
"certificate, which can cause validation failures. You can upgrade to "
"a newer version of Python to solve this. For more information, see "
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
"#ssl-warnings",
SNIMissingWarning,
'An HTTPS request has been made, but the SNI (Server Name '
'Indication) extension to TLS is not available on this platform. '
'This may cause the server to present an incorrect TLS '
'certificate, which can cause validation failures. You can upgrade to '
'a newer version of Python to solve this. For more information, see '
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
'#ssl-warnings',
SNIMissingWarning
)
return context.wrap_socket(sock)
def is_ipaddress(hostname):
"""Detects whether the hostname given is an IPv4 or IPv6 address.
Also detects IPv6 addresses with Zone IDs.
"""Detects whether the hostname given is an IP address.
:param str hostname: Hostname to examine.
:return: True if the hostname is an IP address, False otherwise.
"""
if six.PY3 and isinstance(hostname, bytes):
# IDN A-label bytes are ASCII compatible.
hostname = hostname.decode("ascii")
hostname = hostname.decode('ascii')
return _IP_ADDRESS_REGEX.match(hostname) is not None
def _is_key_file_encrypted(key_file):
"""Detects if a key file is encrypted or not."""
with open(key_file, "r") as f:
with open(key_file, 'r') as f:
for line in f:
# Look for Proc-Type: 4,ENCRYPTED
if "ENCRYPTED" in line:
if 'ENCRYPTED' in line:
return True
return False

79
lib/urllib3/util/timeout.py Executable file → Normal file
View File

@@ -1,5 +1,4 @@
from __future__ import absolute_import
# The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT
@@ -46,20 +45,19 @@ class Timeout(object):
:type total: integer, float, or None
:param connect:
The maximum amount of time (in seconds) to wait for a connection
attempt to a server to succeed. Omitting the parameter will default the
connect timeout to the system default, probably `the global default
timeout in socket.py
The maximum amount of time to wait for a connection attempt to a server
to succeed. Omitting the parameter will default the connect timeout to
the system default, probably `the global default timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout for connection attempts.
:type connect: integer, float, or None
:param read:
The maximum amount of time (in seconds) to wait between consecutive
read operations for a response from the server. Omitting the parameter
will default the read timeout to the system default, probably `the
global default timeout in socket.py
The maximum amount of time to wait between consecutive
read operations for a response from the server. Omitting
the parameter will default the read timeout to the system
default, probably `the global default timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout.
@@ -93,18 +91,14 @@ class Timeout(object):
DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT
def __init__(self, total=None, connect=_Default, read=_Default):
self._connect = self._validate_timeout(connect, "connect")
self._read = self._validate_timeout(read, "read")
self.total = self._validate_timeout(total, "total")
self._connect = self._validate_timeout(connect, 'connect')
self._read = self._validate_timeout(read, 'read')
self.total = self._validate_timeout(total, 'total')
self._start_connect = None
def __str__(self):
return "%s(connect=%r, read=%r, total=%r)" % (
type(self).__name__,
self._connect,
self._read,
self.total,
)
return '%s(connect=%r, read=%r, total=%r)' % (
type(self).__name__, self._connect, self._read, self.total)
@classmethod
def _validate_timeout(cls, value, name):
@@ -124,31 +118,23 @@ class Timeout(object):
return value
if isinstance(value, bool):
raise ValueError(
"Timeout cannot be a boolean value. It must "
"be an int, float or None."
)
raise ValueError("Timeout cannot be a boolean value. It must "
"be an int, float or None.")
try:
float(value)
except (TypeError, ValueError):
raise ValueError(
"Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
)
raise ValueError("Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value))
try:
if value <= 0:
raise ValueError(
"Attempted to set %s timeout to %s, but the "
"timeout cannot be set to a value less "
"than or equal to 0." % (name, value)
)
raise ValueError("Attempted to set %s timeout to %s, but the "
"timeout cannot be set to a value less "
"than or equal to 0." % (name, value))
except TypeError:
# Python 3
raise ValueError(
"Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
)
raise ValueError("Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value))
return value
@@ -180,7 +166,8 @@ class Timeout(object):
# We can't use copy.deepcopy because that will also create a new object
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
# detect the user default.
return Timeout(connect=self._connect, read=self._read, total=self.total)
return Timeout(connect=self._connect, read=self._read,
total=self.total)
def start_connect(self):
""" Start the timeout clock, used during a connect() attempt
@@ -196,15 +183,14 @@ class Timeout(object):
def get_connect_duration(self):
""" Gets the time elapsed since the call to :meth:`start_connect`.
:return: Elapsed time in seconds.
:return: Elapsed time.
:rtype: float
:raises urllib3.exceptions.TimeoutStateError: if you attempt
to get duration for a timer that hasn't been started.
"""
if self._start_connect is None:
raise TimeoutStateError(
"Can't get connect duration for timer " "that has not started."
)
raise TimeoutStateError("Can't get connect duration for timer "
"that has not started.")
return current_time() - self._start_connect
@property
@@ -242,16 +228,15 @@ class Timeout(object):
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
has not yet been called on this object.
"""
if (
self.total is not None
and self.total is not self.DEFAULT_TIMEOUT
and self._read is not None
and self._read is not self.DEFAULT_TIMEOUT
):
if (self.total is not None and
self.total is not self.DEFAULT_TIMEOUT and
self._read is not None and
self._read is not self.DEFAULT_TIMEOUT):
# In case the connect timeout has not yet been established.
if self._start_connect is None:
return self._read
return max(0, min(self.total - self.get_connect_duration(), self._read))
return max(0, min(self.total - self.get_connect_duration(),
self._read))
elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT:
return max(0, self.total - self.get_connect_duration())
else:

130
lib/urllib3/util/url.py Executable file → Normal file
View File

@@ -6,50 +6,34 @@ from ..exceptions import LocationParseError
from ..packages import six, rfc3986
from ..packages.rfc3986.exceptions import RFC3986Exception, ValidationError
from ..packages.rfc3986.validators import Validator
from ..packages.rfc3986 import abnf_regexp, normalizers, compat, misc
url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"]
url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
# We only want to normalize urls with an HTTP(S) scheme.
# urllib3 infers URLs without a scheme (None) to be http.
NORMALIZABLE_SCHEMES = ("http", "https", None)
NORMALIZABLE_SCHEMES = ('http', 'https', None)
# Regex for detecting URLs with schemes. RFC 3986 Section 3.1
SCHEME_REGEX = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+\-]*:|/)")
PATH_CHARS = (
abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {":", "@", "/"}
)
QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"}
class Url(namedtuple("Url", url_attrs)):
class Url(namedtuple('Url', url_attrs)):
"""
Data structure for representing an HTTP URL. Used as a return value for
:func:`parse_url`. Both the scheme and host are normalized as they are
both case-insensitive according to RFC 3986.
"""
__slots__ = ()
def __new__(
cls,
scheme=None,
auth=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
):
if path and not path.startswith("/"):
path = "/" + path
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
query=None, fragment=None):
if path and not path.startswith('/'):
path = '/' + path
if scheme is not None:
scheme = scheme.lower()
return super(Url, cls).__new__(
cls, scheme, auth, host, port, path, query, fragment
)
return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
query, fragment)
@property
def hostname(self):
@@ -59,10 +43,10 @@ class Url(namedtuple("Url", url_attrs)):
@property
def request_uri(self):
"""Absolute path including the query string."""
uri = self.path or "/"
uri = self.path or '/'
if self.query is not None:
uri += "?" + self.query
uri += '?' + self.query
return uri
@@ -70,7 +54,7 @@ class Url(namedtuple("Url", url_attrs)):
def netloc(self):
"""Network location including host and port"""
if self.port:
return "%s:%d" % (self.host, self.port)
return '%s:%d' % (self.host, self.port)
return self.host
@property
@@ -93,23 +77,23 @@ class Url(namedtuple("Url", url_attrs)):
'http://username:password@host.com:80/path?query#fragment'
"""
scheme, auth, host, port, path, query, fragment = self
url = u""
url = u''
# We use "is not None" we want things to happen with empty strings (or 0 port)
if scheme is not None:
url += scheme + u"://"
url += scheme + u'://'
if auth is not None:
url += auth + u"@"
url += auth + u'@'
if host is not None:
url += host
if port is not None:
url += u":" + str(port)
url += u':' + str(port)
if path is not None:
url += path
if query is not None:
url += u"?" + query
url += u'?' + query
if fragment is not None:
url += u"#" + fragment
url += u'#' + fragment
return url
@@ -147,42 +131,9 @@ def split_first(s, delims):
min_delim = d
if min_idx is None or min_idx < 0:
return s, "", None
return s, '', None
return s[:min_idx], s[min_idx + 1 :], min_delim
def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"):
"""Percent-encodes a URI component without reapplying
onto an already percent-encoded component. Based on
rfc3986.normalizers.encode_component()
"""
if component is None:
return component
# Try to see if the component we're encoding is already percent-encoded
# so we can skip all '%' characters but still encode all others.
percent_encodings = len(
normalizers.PERCENT_MATCHER.findall(compat.to_str(component, encoding))
)
uri_bytes = component.encode("utf-8", "surrogatepass")
is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
encoded_component = bytearray()
for i in range(0, len(uri_bytes)):
# Will return a single character bytestring on both Python 2 & 3
byte = uri_bytes[i : i + 1]
byte_ord = ord(byte)
if (is_percent_encoded and byte == b"%") or (
byte_ord < 128 and byte.decode() in allowed_chars
):
encoded_component.extend(byte)
continue
encoded_component.extend("%{0:02x}".format(byte_ord).encode().upper())
return encoded_component.decode(encoding)
return s[:min_idx], s[min_idx + 1:], min_delim
def parse_url(url):
@@ -209,6 +160,8 @@ def parse_url(url):
return Url()
is_string = not isinstance(url, six.binary_type)
if not is_string:
url = url.decode("utf-8")
# RFC 3986 doesn't like URLs that have a host but don't start
# with a scheme and we support URLs like that so we need to
@@ -218,33 +171,25 @@ def parse_url(url):
if not SCHEME_REGEX.search(url):
url = "//" + url
try:
iri_ref = rfc3986.IRIReference.from_string(url, encoding="utf-8")
except (ValueError, RFC3986Exception):
six.raise_from(LocationParseError(url), None)
def idna_encode(name):
if name and any([ord(x) > 128 for x in name]):
try:
import idna
except ImportError:
raise LocationParseError(
"Unable to parse URL without the 'idna' module"
)
raise LocationParseError("Unable to parse URL without the 'idna' module")
try:
return idna.encode(name.lower(), strict=True, std3_rules=True)
except idna.IDNAError:
raise LocationParseError(u"Name '%s' is not a valid IDNA label" % name)
return name
try:
split_iri = misc.IRI_MATCHER.match(compat.to_str(url)).groupdict()
iri_ref = rfc3986.IRIReference(
split_iri["scheme"],
split_iri["authority"],
_encode_invalid_chars(split_iri["path"], PATH_CHARS),
_encode_invalid_chars(split_iri["query"], QUERY_CHARS),
_encode_invalid_chars(split_iri["fragment"], FRAGMENT_CHARS),
)
has_authority = iri_ref.authority is not None
uri_ref = iri_ref.encode(idna_encoder=idna_encode)
except (ValueError, RFC3986Exception):
return six.raise_from(LocationParseError(url), None)
has_authority = iri_ref.authority is not None
uri_ref = iri_ref.encode(idna_encoder=idna_encode)
# rfc3986 strips the authority if it's invalid
if has_authority and uri_ref.authority is None:
@@ -260,9 +205,11 @@ def parse_url(url):
# normalization has completed.
validator = Validator()
try:
validator.check_validity_of(*validator.COMPONENT_NAMES).validate(uri_ref)
validator.check_validity_of(
*validator.COMPONENT_NAMES
).validate(uri_ref)
except ValidationError:
return six.raise_from(LocationParseError(url), None)
six.raise_from(LocationParseError(url), None)
# For the sake of backwards compatibility we put empty
# string values for path if there are any defined values
@@ -270,7 +217,8 @@ def parse_url(url):
# TODO: Remove this when we break backwards compatibility.
path = uri_ref.path
if not path:
if uri_ref.query is not None or uri_ref.fragment is not None:
if (uri_ref.query is not None
or uri_ref.fragment is not None):
path = ""
else:
path = None
@@ -281,7 +229,7 @@ def parse_url(url):
if x is None:
return None
elif not is_string and not isinstance(x, six.binary_type):
return x.encode("utf-8")
return x.encode('utf-8')
return x
return Url(
@@ -291,7 +239,7 @@ def parse_url(url):
port=int(uri_ref.port) if uri_ref.port is not None else None,
path=to_input_type(path),
query=to_input_type(uri_ref.query),
fragment=to_input_type(uri_ref.fragment),
fragment=to_input_type(uri_ref.fragment)
)
@@ -300,4 +248,4 @@ def get_host(url):
Deprecated. Use :func:`parse_url` instead.
"""
p = parse_url(url)
return p.scheme or "http", p.hostname, p.port
return p.scheme or 'http', p.hostname, p.port

3
lib/urllib3/util/wait.py Executable file → Normal file
View File

@@ -2,7 +2,6 @@ import errno
from functools import partial
import select
import sys
try:
from time import monotonic
except ImportError:
@@ -41,8 +40,6 @@ if sys.version_info >= (3, 5):
# Modern Python, that retries syscalls by default
def _retry_on_intr(fn, timeout):
return fn(timeout)
else:
# Old and broken Pythons.
def _retry_on_intr(fn, timeout):