From de3d29c81864aae9f853f181434d48681caef023 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Tue, 25 Jun 2024 23:03:04 +0100 Subject: [PATCH] =?UTF-8?q?Update=20urllib3=202.2.1=20(54d6edf)=20?= =?UTF-8?q?=E2=86=92=202.2.2=20(27e2a5c).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 7 +++- lib/urllib3/_base_connection.py | 8 ++--- lib/urllib3/_collections.py | 6 ++-- lib/urllib3/_version.py | 2 +- lib/urllib3/connection.py | 9 +++-- lib/urllib3/connectionpool.py | 18 ++++------ lib/urllib3/contrib/socks.py | 4 +-- lib/urllib3/http2.py | 1 + lib/urllib3/poolmanager.py | 11 +++--- lib/urllib3/response.py | 62 +++++++++++++++++++++----------- lib/urllib3/util/request.py | 2 +- lib/urllib3/util/retry.py | 10 ++++-- lib/urllib3/util/ssl_.py | 4 +-- lib/urllib3/util/ssltransport.py | 9 +++-- 14 files changed, 88 insertions(+), 65 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7e30ea4..cc942793 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,9 @@ -### 3.32.0 (2024-06-25 21:15:00 UTC) +### 3.33.0 (2024-0x-xx xx:xx:00 UTC) + +* Update urllib3 2.2.1 (54d6edf) to 2.2.2 (27e2a5c) + + +### 3.32.0 (2024-06-25 21:15:00 UTC) * Update apprise 1.6.0 (0c0d5da) to 1.8.0 (81caf92) * Update attr 23.1.0 (67e4ff2) to 23.2.0 (b393d79) diff --git a/lib/urllib3/_base_connection.py b/lib/urllib3/_base_connection.py index bb349c74..29ca3348 100644 --- a/lib/urllib3/_base_connection.py +++ b/lib/urllib3/_base_connection.py @@ -12,7 +12,7 @@ _TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], class ProxyConfig(typing.NamedTuple): ssl_context: ssl.SSLContext | None use_forwarding_for_https: bool - assert_hostname: None | str | Literal[False] + assert_hostname: None | str | typing.Literal[False] assert_fingerprint: str | None @@ -28,7 +28,7 @@ class _ResponseOptions(typing.NamedTuple): if typing.TYPE_CHECKING: import ssl - from typing import Literal, Protocol + from typing import Protocol from .response import BaseHTTPResponse @@ -124,7 +124,7 @@ if typing.TYPE_CHECKING: # Certificate verification methods cert_reqs: int | str | None - assert_hostname: None | str | Literal[False] + assert_hostname: None | str | typing.Literal[False] assert_fingerprint: str | None ssl_context: ssl.SSLContext | None @@ -155,7 +155,7 @@ if typing.TYPE_CHECKING: proxy: Url | None = None, proxy_config: ProxyConfig | None = None, cert_reqs: int | str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, server_hostname: str | None = None, ssl_context: ssl.SSLContext | None = None, diff --git a/lib/urllib3/_collections.py b/lib/urllib3/_collections.py index 55b03247..8a4409a1 100644 --- a/lib/urllib3/_collections.py +++ b/lib/urllib3/_collections.py @@ -427,7 +427,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]): val = other.getlist(key) self._container[key.lower()] = [key, *val] - def copy(self) -> HTTPHeaderDict: + def copy(self) -> Self: clone = type(self)() clone._copy_from(self) return clone @@ -462,7 +462,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]): self.extend(maybe_constructable) return self - def __or__(self, other: object) -> HTTPHeaderDict: + def __or__(self, other: object) -> Self: # Supports merging header dicts using operator | # combining items with add instead of __setitem__ maybe_constructable = ensure_can_construct_http_header_dict(other) @@ -472,7 +472,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]): result.extend(maybe_constructable) return result - def __ror__(self, other: object) -> HTTPHeaderDict: + def __ror__(self, other: object) -> Self: # Supports merging header dicts using operator | when other is on left side # combining items with add instead of __setitem__ maybe_constructable = ensure_can_construct_http_header_dict(other) diff --git a/lib/urllib3/_version.py b/lib/urllib3/_version.py index 095cf3c1..7442f2b8 100644 --- a/lib/urllib3/_version.py +++ b/lib/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.2.1" +__version__ = "2.2.2" diff --git a/lib/urllib3/connection.py b/lib/urllib3/connection.py index aa5c547c..1b162792 100644 --- a/lib/urllib3/connection.py +++ b/lib/urllib3/connection.py @@ -14,8 +14,6 @@ from http.client import ResponseNotReady from socket import timeout as SocketTimeout if typing.TYPE_CHECKING: - from typing import Literal - from .response import HTTPResponse from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT from .util.ssltransport import SSLTransport @@ -482,6 +480,7 @@ class HTTPConnection(_HTTPConnection): headers=headers, status=httplib_response.status, version=httplib_response.version, + version_string=getattr(self, "_http_vsn_str", "HTTP/?"), reason=httplib_response.reason, preload_content=resp_options.preload_content, decode_content=resp_options.decode_content, @@ -523,7 +522,7 @@ class HTTPSConnection(HTTPConnection): proxy: Url | None = None, proxy_config: ProxyConfig | None = None, cert_reqs: int | str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, server_hostname: str | None = None, ssl_context: ssl.SSLContext | None = None, @@ -577,7 +576,7 @@ class HTTPSConnection(HTTPConnection): cert_reqs: int | str | None = None, key_password: str | None = None, ca_certs: str | None = None, - assert_hostname: None | str | Literal[False] = None, + assert_hostname: None | str | typing.Literal[False] = None, assert_fingerprint: str | None = None, ca_cert_dir: str | None = None, ca_cert_data: None | str | bytes = None, @@ -742,7 +741,7 @@ def _ssl_wrap_socket_and_match_hostname( ca_certs: str | None, ca_cert_dir: str | None, ca_cert_data: None | str | bytes, - assert_hostname: None | str | Literal[False], + assert_hostname: None | str | typing.Literal[False], assert_fingerprint: str | None, server_hostname: str | None, ssl_context: ssl.SSLContext | None, diff --git a/lib/urllib3/connectionpool.py b/lib/urllib3/connectionpool.py index bd58ff14..a2c3cf60 100644 --- a/lib/urllib3/connectionpool.py +++ b/lib/urllib3/connectionpool.py @@ -53,7 +53,8 @@ from .util.util import to_str if typing.TYPE_CHECKING: import ssl - from typing import Literal + + from typing_extensions import Self from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection @@ -61,8 +62,6 @@ log = logging.getLogger(__name__) _TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None] -_SelfT = typing.TypeVar("_SelfT") - # Pool objects class ConnectionPool: @@ -95,7 +94,7 @@ class ConnectionPool: def __str__(self) -> str: return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})" - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__( @@ -103,7 +102,7 @@ class ConnectionPool: exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, - ) -> Literal[False]: + ) -> typing.Literal[False]: self.close() # Return False to re-raise any potential exceptions return False @@ -544,17 +543,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response._connection = response_conn # type: ignore[attr-defined] response._pool = self # type: ignore[attr-defined] - # emscripten connection doesn't have _http_vsn_str - http_version = getattr(conn, "_http_vsn_str", "HTTP/?") log.debug( - '%s://%s:%s "%s %s %s" %s %s', + '%s://%s:%s "%s %s HTTP/%s" %s %s', self.scheme, self.host, self.port, method, url, - # HTTP version - http_version, + response.version, response.status, response.length_remaining, ) @@ -1002,7 +998,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): ssl_version: int | str | None = None, ssl_minimum_version: ssl.TLSVersion | None = None, ssl_maximum_version: ssl.TLSVersion | None = None, - assert_hostname: str | Literal[False] | None = None, + assert_hostname: str | typing.Literal[False] | None = None, assert_fingerprint: str | None = None, ca_cert_dir: str | None = None, **conn_kw: typing.Any, diff --git a/lib/urllib3/contrib/socks.py b/lib/urllib3/contrib/socks.py index 5a803916..c62b5e03 100644 --- a/lib/urllib3/contrib/socks.py +++ b/lib/urllib3/contrib/socks.py @@ -71,10 +71,8 @@ try: except ImportError: ssl = None # type: ignore[assignment] -from typing import TypedDict - -class _TYPE_SOCKS_OPTIONS(TypedDict): +class _TYPE_SOCKS_OPTIONS(typing.TypedDict): socks_version: int proxy_host: str | None proxy_port: str | None diff --git a/lib/urllib3/http2.py b/lib/urllib3/http2.py index 15fa9d91..ceb40602 100644 --- a/lib/urllib3/http2.py +++ b/lib/urllib3/http2.py @@ -195,6 +195,7 @@ class HTTP2Response(BaseHTTPResponse): headers=headers, # Following CPython, we map HTTP versions to major * 10 + minor integers version=20, + version_string="HTTP/2", # No reason phrase in HTTP/2 reason=None, decode_content=decode_content, diff --git a/lib/urllib3/poolmanager.py b/lib/urllib3/poolmanager.py index 32da0a00..085d1dba 100644 --- a/lib/urllib3/poolmanager.py +++ b/lib/urllib3/poolmanager.py @@ -26,7 +26,8 @@ from .util.url import Url, parse_url if typing.TYPE_CHECKING: import ssl - from typing import Literal + + from typing_extensions import Self __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] @@ -51,8 +52,6 @@ SSL_KEYWORDS = ( # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 _DEFAULT_BLOCKSIZE = 16384 -_SelfT = typing.TypeVar("_SelfT") - class PoolKey(typing.NamedTuple): """ @@ -214,7 +213,7 @@ class PoolManager(RequestMethods): self.pool_classes_by_scheme = pool_classes_by_scheme self.key_fn_by_scheme = key_fn_by_scheme.copy() - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__( @@ -222,7 +221,7 @@ class PoolManager(RequestMethods): exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, - ) -> Literal[False]: + ) -> typing.Literal[False]: self.clear() # Return False to re-raise any potential exceptions return False @@ -553,7 +552,7 @@ class ProxyManager(PoolManager): proxy_headers: typing.Mapping[str, str] | None = None, proxy_ssl_context: ssl.SSLContext | None = None, use_forwarding_for_https: bool = False, - proxy_assert_hostname: None | str | Literal[False] = None, + proxy_assert_hostname: None | str | typing.Literal[False] = None, proxy_assert_fingerprint: str | None = None, **connection_pool_kw: typing.Any, ) -> None: diff --git a/lib/urllib3/response.py b/lib/urllib3/response.py index d31fac9b..a0273d65 100644 --- a/lib/urllib3/response.py +++ b/lib/urllib3/response.py @@ -26,20 +26,21 @@ except ImportError: brotli = None try: - import zstandard as zstd # type: ignore[import-not-found] - + import zstandard as zstd +except (AttributeError, ImportError, ValueError): # Defensive: + HAS_ZSTD = False +else: # The package 'zstandard' added the 'eof' property starting # in v0.18.0 which we require to ensure a complete and # valid zstd stream was fed into the ZstdDecoder. # See: https://github.com/urllib3/urllib3/pull/2624 - _zstd_version = _zstd_version = tuple( + _zstd_version = tuple( map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] ) if _zstd_version < (0, 18): # Defensive: - zstd = None - -except (AttributeError, ImportError, ValueError): # Defensive: - zstd = None + HAS_ZSTD = False + else: + HAS_ZSTD = True from . import util from ._base_connection import _TYPE_BODY @@ -61,8 +62,6 @@ from .util.response import is_fp_closed, is_response_to_head from .util.retry import Retry if typing.TYPE_CHECKING: - from typing import Literal - from .connectionpool import HTTPConnectionPool log = logging.getLogger(__name__) @@ -163,7 +162,7 @@ if brotli is not None: return b"" -if zstd is not None: +if HAS_ZSTD: class ZstdDecoder(ContentDecoder): def __init__(self) -> None: @@ -183,7 +182,7 @@ if zstd is not None: ret = self._obj.flush() # note: this is a no-op if not self._obj.eof: raise DecodeError("Zstandard data is incomplete") - return ret # type: ignore[no-any-return] + return ret class MultiDecoder(ContentDecoder): @@ -219,7 +218,7 @@ def _get_decoder(mode: str) -> ContentDecoder: if brotli is not None and mode == "br": return BrotliDecoder() - if zstd is not None and mode == "zstd": + if HAS_ZSTD and mode == "zstd": return ZstdDecoder() return DeflateDecoder() @@ -302,7 +301,7 @@ class BaseHTTPResponse(io.IOBase): CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] if brotli is not None: CONTENT_DECODERS += ["br"] - if zstd is not None: + if HAS_ZSTD: CONTENT_DECODERS += ["zstd"] REDIRECT_STATUSES = [301, 302, 303, 307, 308] @@ -310,7 +309,7 @@ class BaseHTTPResponse(io.IOBase): if brotli is not None: DECODER_ERROR_CLASSES += (brotli.error,) - if zstd is not None: + if HAS_ZSTD: DECODER_ERROR_CLASSES += (zstd.ZstdError,) def __init__( @@ -319,6 +318,7 @@ class BaseHTTPResponse(io.IOBase): headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, status: int, version: int, + version_string: str, reason: str | None, decode_content: bool, request_url: str | None, @@ -330,6 +330,7 @@ class BaseHTTPResponse(io.IOBase): self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type] self.status = status self.version = version + self.version_string = version_string self.reason = reason self.decode_content = decode_content self._has_decoded_content = False @@ -346,7 +347,7 @@ class BaseHTTPResponse(io.IOBase): self._decoder: ContentDecoder | None = None self.length_remaining: int | None - def get_redirect_location(self) -> str | None | Literal[False]: + def get_redirect_location(self) -> str | None | typing.Literal[False]: """ Should we redirect and where to? @@ -364,13 +365,21 @@ class BaseHTTPResponse(io.IOBase): def json(self) -> typing.Any: """ - Parses the body of the HTTP response as JSON. + Deserializes the body of the HTTP response as a Python object. - To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder. + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 `_. - This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`. + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. - Read more :ref:`here `. + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here `. + + :returns: The body of the HTTP response as a Python object. """ data = self.data.decode("utf-8") return _json.loads(data) @@ -567,6 +576,7 @@ class HTTPResponse(BaseHTTPResponse): headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, status: int = 0, version: int = 0, + version_string: str = "HTTP/?", reason: str | None = None, preload_content: bool = True, decode_content: bool = True, @@ -584,6 +594,7 @@ class HTTPResponse(BaseHTTPResponse): headers=headers, status=status, version=version, + version_string=version_string, reason=reason, decode_content=decode_content, request_url=request_url, @@ -926,7 +937,10 @@ class HTTPResponse(BaseHTTPResponse): if decode_content is None: decode_content = self.decode_content - if amt is not None: + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + elif amt is not None: cache_content = False if len(self._decoded_buffer) >= amt: @@ -986,6 +1000,9 @@ class HTTPResponse(BaseHTTPResponse): """ if decode_content is None: decode_content = self.decode_content + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None # try and respond without going to the network if self._has_decoded_content: if not decode_content: @@ -1180,6 +1197,11 @@ class HTTPResponse(BaseHTTPResponse): if self._fp.fp is None: # type: ignore[union-attr] return None + if amt and amt < 0: + # Negative numbers and `None` should be treated the same, + # but httplib handles only `None` correctly. + amt = None + while True: self._update_chunk_length() if self.chunk_left == 0: diff --git a/lib/urllib3/util/request.py b/lib/urllib3/util/request.py index fe0e3485..859597e2 100644 --- a/lib/urllib3/util/request.py +++ b/lib/urllib3/util/request.py @@ -29,7 +29,7 @@ except ImportError: else: ACCEPT_ENCODING += ",br" try: - import zstandard as _unused_module_zstd # type: ignore[import-not-found] # noqa: F401 + import zstandard as _unused_module_zstd # noqa: F401 except ImportError: pass else: diff --git a/lib/urllib3/util/retry.py b/lib/urllib3/util/retry.py index 7572bfd2..0456cceb 100644 --- a/lib/urllib3/util/retry.py +++ b/lib/urllib3/util/retry.py @@ -21,6 +21,8 @@ from ..exceptions import ( from .util import reraise if typing.TYPE_CHECKING: + from typing_extensions import Self + from ..connectionpool import ConnectionPool from ..response import BaseHTTPResponse @@ -187,7 +189,9 @@ class Retry: RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) #: Default maximum backoff time. DEFAULT_BACKOFF_MAX = 120 @@ -240,7 +244,7 @@ class Retry: ) self.backoff_jitter = backoff_jitter - def new(self, **kw: typing.Any) -> Retry: + def new(self, **kw: typing.Any) -> Self: params = dict( total=self.total, connect=self.connect, @@ -429,7 +433,7 @@ class Retry: error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, - ) -> Retry: + ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not diff --git a/lib/urllib3/util/ssl_.py b/lib/urllib3/util/ssl_.py index b14cf27b..c46dd83e 100644 --- a/lib/urllib3/util/ssl_.py +++ b/lib/urllib3/util/ssl_.py @@ -78,7 +78,7 @@ def _is_has_never_check_common_name_reliable( if typing.TYPE_CHECKING: from ssl import VerifyMode - from typing import Literal, TypedDict + from typing import TypedDict from .ssltransport import SSLTransport as SSLTransportType @@ -365,7 +365,7 @@ def ssl_wrap_socket( ca_cert_dir: str | None = ..., key_password: str | None = ..., ca_cert_data: None | str | bytes = ..., - tls_in_tls: Literal[False] = ..., + tls_in_tls: typing.Literal[False] = ..., ) -> ssl.SSLSocket: ... diff --git a/lib/urllib3/util/ssltransport.py b/lib/urllib3/util/ssltransport.py index fa9f2b37..b52c477c 100644 --- a/lib/urllib3/util/ssltransport.py +++ b/lib/urllib3/util/ssltransport.py @@ -8,12 +8,11 @@ import typing from ..exceptions import ProxySchemeUnsupported if typing.TYPE_CHECKING: - from typing import Literal + from typing_extensions import Self from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT -_SelfT = typing.TypeVar("_SelfT", bound="SSLTransport") _WriteBuffer = typing.Union[bytearray, memoryview] _ReturnValue = typing.TypeVar("_ReturnValue") @@ -70,7 +69,7 @@ class SSLTransport: # Perform initial handshake. self._ssl_io_loop(self.sslobj.do_handshake) - def __enter__(self: _SelfT) -> _SelfT: + def __enter__(self) -> Self: return self def __exit__(self, *_: typing.Any) -> None: @@ -174,12 +173,12 @@ class SSLTransport: @typing.overload def getpeercert( - self, binary_form: Literal[False] = ... + self, binary_form: typing.Literal[False] = ... ) -> _TYPE_PEER_CERT_RET_DICT | None: ... @typing.overload - def getpeercert(self, binary_form: Literal[True]) -> bytes | None: + def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ... def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET: