Merge branch 'feature/UpdateUrllib3' into dev

CHANGES.md

@@ -1,4 +1,9 @@
-### 3.32.0 (2024-06-25 21:15:00 UTC)
+### 3.33.0 (2024-0x-xx xx:xx:00 UTC)
+
+* Update urllib3 2.2.1 (54d6edf) to 2.2.2 (27e2a5c)
+
+
+### 3.32.0 (2024-06-25 21:15:00 UTC)
 
 * Update apprise 1.6.0 (0c0d5da) to 1.8.0 (81caf92)
 * Update attr 23.1.0 (67e4ff2) to 23.2.0 (b393d79)

lib/urllib3/_base_connection.py

@@ -12,7 +12,7 @@ _TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes],
 class ProxyConfig(typing.NamedTuple):
     ssl_context: ssl.SSLContext | None
     use_forwarding_for_https: bool
-    assert_hostname: None | str | Literal[False]
+    assert_hostname: None | str | typing.Literal[False]
     assert_fingerprint: str | None
 
 
@@ -28,7 +28,7 @@ class _ResponseOptions(typing.NamedTuple):
 
 if typing.TYPE_CHECKING:
     import ssl
-    from typing import Literal, Protocol
+    from typing import Protocol
 
     from .response import BaseHTTPResponse
 
@@ -124,7 +124,7 @@ if typing.TYPE_CHECKING:
 
         # Certificate verification methods
         cert_reqs: int | str | None
-        assert_hostname: None | str | Literal[False]
+        assert_hostname: None | str | typing.Literal[False]
         assert_fingerprint: str | None
         ssl_context: ssl.SSLContext | None
 
@@ -155,7 +155,7 @@ if typing.TYPE_CHECKING:
             proxy: Url | None = None,
             proxy_config: ProxyConfig | None = None,
             cert_reqs: int | str | None = None,
-            assert_hostname: None | str | Literal[False] = None,
+            assert_hostname: None | str | typing.Literal[False] = None,
             assert_fingerprint: str | None = None,
             server_hostname: str | None = None,
             ssl_context: ssl.SSLContext | None = None,

lib/urllib3/_collections.py

@@ -427,7 +427,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
             val = other.getlist(key)
             self._container[key.lower()] = [key, *val]
 
-    def copy(self) -> HTTPHeaderDict:
+    def copy(self) -> Self:
         clone = type(self)()
         clone._copy_from(self)
         return clone
@@ -462,7 +462,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
         self.extend(maybe_constructable)
         return self
 
-    def __or__(self, other: object) -> HTTPHeaderDict:
+    def __or__(self, other: object) -> Self:
         # Supports merging header dicts using operator |
         # combining items with add instead of __setitem__
         maybe_constructable = ensure_can_construct_http_header_dict(other)
@@ -472,7 +472,7 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
         result.extend(maybe_constructable)
         return result
 
-    def __ror__(self, other: object) -> HTTPHeaderDict:
+    def __ror__(self, other: object) -> Self:
         # Supports merging header dicts using operator | when other is on left side
         # combining items with add instead of __setitem__
         maybe_constructable = ensure_can_construct_http_header_dict(other)

lib/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.2.1"
+__version__ = "2.2.2"

lib/urllib3/connection.py

@@ -14,8 +14,6 @@ from http.client import ResponseNotReady
 from socket import timeout as SocketTimeout
 
 if typing.TYPE_CHECKING:
-    from typing import Literal
-
     from .response import HTTPResponse
     from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
     from .util.ssltransport import SSLTransport
@@ -482,6 +480,7 @@ class HTTPConnection(_HTTPConnection):
             headers=headers,
             status=httplib_response.status,
             version=httplib_response.version,
+            version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
             reason=httplib_response.reason,
             preload_content=resp_options.preload_content,
             decode_content=resp_options.decode_content,
@@ -523,7 +522,7 @@ class HTTPSConnection(HTTPConnection):
         proxy: Url | None = None,
         proxy_config: ProxyConfig | None = None,
         cert_reqs: int | str | None = None,
-        assert_hostname: None | str | Literal[False] = None,
+        assert_hostname: None | str | typing.Literal[False] = None,
         assert_fingerprint: str | None = None,
         server_hostname: str | None = None,
         ssl_context: ssl.SSLContext | None = None,
@@ -577,7 +576,7 @@ class HTTPSConnection(HTTPConnection):
         cert_reqs: int | str | None = None,
         key_password: str | None = None,
         ca_certs: str | None = None,
-        assert_hostname: None | str | Literal[False] = None,
+        assert_hostname: None | str | typing.Literal[False] = None,
         assert_fingerprint: str | None = None,
         ca_cert_dir: str | None = None,
         ca_cert_data: None | str | bytes = None,
@@ -742,7 +741,7 @@ def _ssl_wrap_socket_and_match_hostname(
     ca_certs: str | None,
     ca_cert_dir: str | None,
     ca_cert_data: None | str | bytes,
-    assert_hostname: None | str | Literal[False],
+    assert_hostname: None | str | typing.Literal[False],
     assert_fingerprint: str | None,
     server_hostname: str | None,
     ssl_context: ssl.SSLContext | None,

lib/urllib3/connectionpool.py

@@ -53,7 +53,8 @@ from .util.util import to_str
 
 if typing.TYPE_CHECKING:
     import ssl
-    from typing import Literal
+
+    from typing_extensions import Self
 
     from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
 
@@ -61,8 +62,6 @@ log = logging.getLogger(__name__)
 
 _TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
 
-_SelfT = typing.TypeVar("_SelfT")
-
 
 # Pool objects
 class ConnectionPool:
@@ -95,7 +94,7 @@ class ConnectionPool:
     def __str__(self) -> str:
         return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
 
-    def __enter__(self: _SelfT) -> _SelfT:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(
@@ -103,7 +102,7 @@ class ConnectionPool:
         exc_type: type[BaseException] | None,
         exc_val: BaseException | None,
         exc_tb: TracebackType | None,
-    ) -> Literal[False]:
+    ) -> typing.Literal[False]:
         self.close()
         # Return False to re-raise any potential exceptions
         return False
@@ -544,17 +543,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         response._connection = response_conn  # type: ignore[attr-defined]
         response._pool = self  # type: ignore[attr-defined]
 
-        # emscripten connection doesn't have _http_vsn_str
-        http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
         log.debug(
-            '%s://%s:%s "%s %s %s" %s %s',
+            '%s://%s:%s "%s %s HTTP/%s" %s %s',
             self.scheme,
             self.host,
             self.port,
             method,
             url,
-            # HTTP version
+            response.version,
-            http_version,
             response.status,
             response.length_remaining,
         )
@@ -1002,7 +998,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         ssl_version: int | str | None = None,
         ssl_minimum_version: ssl.TLSVersion | None = None,
         ssl_maximum_version: ssl.TLSVersion | None = None,
-        assert_hostname: str | Literal[False] | None = None,
+        assert_hostname: str | typing.Literal[False] | None = None,
         assert_fingerprint: str | None = None,
         ca_cert_dir: str | None = None,
         **conn_kw: typing.Any,

lib/urllib3/contrib/socks.py

@@ -71,10 +71,8 @@ try:
 except ImportError:
     ssl = None  # type: ignore[assignment]
 
-from typing import TypedDict
 
-
+class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
-class _TYPE_SOCKS_OPTIONS(TypedDict):
     socks_version: int
     proxy_host: str | None
     proxy_port: str | None

lib/urllib3/http2.py

@@ -195,6 +195,7 @@ class HTTP2Response(BaseHTTPResponse):
             headers=headers,
             # Following CPython, we map HTTP versions to major * 10 + minor integers
             version=20,
+            version_string="HTTP/2",
             # No reason phrase in HTTP/2
             reason=None,
             decode_content=decode_content,

lib/urllib3/poolmanager.py

@@ -26,7 +26,8 @@ from .util.url import Url, parse_url
 
 if typing.TYPE_CHECKING:
     import ssl
-    from typing import Literal
+
+    from typing_extensions import Self
 
 __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
 
@@ -51,8 +52,6 @@ SSL_KEYWORDS = (
 # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
 _DEFAULT_BLOCKSIZE = 16384
 
-_SelfT = typing.TypeVar("_SelfT")
-
 
 class PoolKey(typing.NamedTuple):
     """
@@ -214,7 +213,7 @@ class PoolManager(RequestMethods):
         self.pool_classes_by_scheme = pool_classes_by_scheme
         self.key_fn_by_scheme = key_fn_by_scheme.copy()
 
-    def __enter__(self: _SelfT) -> _SelfT:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(
@@ -222,7 +221,7 @@ class PoolManager(RequestMethods):
         exc_type: type[BaseException] | None,
         exc_val: BaseException | None,
         exc_tb: TracebackType | None,
-    ) -> Literal[False]:
+    ) -> typing.Literal[False]:
         self.clear()
         # Return False to re-raise any potential exceptions
         return False
@@ -553,7 +552,7 @@ class ProxyManager(PoolManager):
         proxy_headers: typing.Mapping[str, str] | None = None,
         proxy_ssl_context: ssl.SSLContext | None = None,
         use_forwarding_for_https: bool = False,
-        proxy_assert_hostname: None | str | Literal[False] = None,
+        proxy_assert_hostname: None | str | typing.Literal[False] = None,
         proxy_assert_fingerprint: str | None = None,
         **connection_pool_kw: typing.Any,
     ) -> None:

lib/urllib3/response.py

@@ -26,20 +26,21 @@ except ImportError:
     brotli = None
 
 try:
-    import zstandard as zstd  # type: ignore[import-not-found]
+    import zstandard as zstd
-
+except (AttributeError, ImportError, ValueError):  # Defensive:
+    HAS_ZSTD = False
+else:
     # The package 'zstandard' added the 'eof' property starting
     # in v0.18.0 which we require to ensure a complete and
     # valid zstd stream was fed into the ZstdDecoder.
     # See: https://github.com/urllib3/urllib3/pull/2624
-    _zstd_version = _zstd_version = tuple(
+    _zstd_version = tuple(
         map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups())  # type: ignore[union-attr]
     )
     if _zstd_version < (0, 18):  # Defensive:
-        zstd = None
+        HAS_ZSTD = False
-
+    else:
-except (AttributeError, ImportError, ValueError):  # Defensive:
+        HAS_ZSTD = True
-    zstd = None
 
 from . import util
 from ._base_connection import _TYPE_BODY
@@ -61,8 +62,6 @@ from .util.response import is_fp_closed, is_response_to_head
 from .util.retry import Retry
 
 if typing.TYPE_CHECKING:
-    from typing import Literal
-
     from .connectionpool import HTTPConnectionPool
 
 log = logging.getLogger(__name__)
@@ -163,7 +162,7 @@ if brotli is not None:
             return b""
 
 
-if zstd is not None:
+if HAS_ZSTD:
 
     class ZstdDecoder(ContentDecoder):
         def __init__(self) -> None:
@@ -183,7 +182,7 @@ if zstd is not None:
             ret = self._obj.flush()  # note: this is a no-op
             if not self._obj.eof:
                 raise DecodeError("Zstandard data is incomplete")
-            return ret  # type: ignore[no-any-return]
+            return ret
 
 
 class MultiDecoder(ContentDecoder):
@@ -219,7 +218,7 @@ def _get_decoder(mode: str) -> ContentDecoder:
     if brotli is not None and mode == "br":
         return BrotliDecoder()
 
-    if zstd is not None and mode == "zstd":
+    if HAS_ZSTD and mode == "zstd":
         return ZstdDecoder()
 
     return DeflateDecoder()
@@ -302,7 +301,7 @@ class BaseHTTPResponse(io.IOBase):
     CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
     if brotli is not None:
         CONTENT_DECODERS += ["br"]
-    if zstd is not None:
+    if HAS_ZSTD:
         CONTENT_DECODERS += ["zstd"]
     REDIRECT_STATUSES = [301, 302, 303, 307, 308]
 
@@ -310,7 +309,7 @@ class BaseHTTPResponse(io.IOBase):
     if brotli is not None:
         DECODER_ERROR_CLASSES += (brotli.error,)
 
-    if zstd is not None:
+    if HAS_ZSTD:
         DECODER_ERROR_CLASSES += (zstd.ZstdError,)
 
     def __init__(
@@ -319,6 +318,7 @@ class BaseHTTPResponse(io.IOBase):
         headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
         status: int,
         version: int,
+        version_string: str,
         reason: str | None,
         decode_content: bool,
         request_url: str | None,
@@ -330,6 +330,7 @@ class BaseHTTPResponse(io.IOBase):
             self.headers = HTTPHeaderDict(headers)  # type: ignore[arg-type]
         self.status = status
         self.version = version
+        self.version_string = version_string
         self.reason = reason
         self.decode_content = decode_content
         self._has_decoded_content = False
@@ -346,7 +347,7 @@ class BaseHTTPResponse(io.IOBase):
         self._decoder: ContentDecoder | None = None
         self.length_remaining: int | None
 
-    def get_redirect_location(self) -> str | None | Literal[False]:
+    def get_redirect_location(self) -> str | None | typing.Literal[False]:
         """
         Should we redirect and where to?
 
@@ -364,13 +365,21 @@ class BaseHTTPResponse(io.IOBase):
 
     def json(self) -> typing.Any:
         """
-        Parses the body of the HTTP response as JSON.
+        Deserializes the body of the HTTP response as a Python object.
 
-        To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder.
+        The body of the HTTP response must be encoded using UTF-8, as per
+        `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
 
-        This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`.
+        To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
+        your custom decoder instead.
 
-        Read more :ref:`here <json>`.
+        If the body of the HTTP response is not decodable to UTF-8, a
+        `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
+        valid JSON document, a `json.JSONDecodeError` will be raised.
+
+        Read more :ref:`here <json_content>`.
+
+        :returns: The body of the HTTP response as a Python object.
         """
         data = self.data.decode("utf-8")
         return _json.loads(data)
@@ -567,6 +576,7 @@ class HTTPResponse(BaseHTTPResponse):
         headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
         status: int = 0,
         version: int = 0,
+        version_string: str = "HTTP/?",
         reason: str | None = None,
         preload_content: bool = True,
         decode_content: bool = True,
@@ -584,6 +594,7 @@ class HTTPResponse(BaseHTTPResponse):
             headers=headers,
             status=status,
             version=version,
+            version_string=version_string,
             reason=reason,
             decode_content=decode_content,
             request_url=request_url,
@@ -926,7 +937,10 @@ class HTTPResponse(BaseHTTPResponse):
         if decode_content is None:
             decode_content = self.decode_content
 
-        if amt is not None:
+        if amt and amt < 0:
+            # Negative numbers and `None` should be treated the same.
+            amt = None
+        elif amt is not None:
             cache_content = False
 
             if len(self._decoded_buffer) >= amt:
@@ -986,6 +1000,9 @@ class HTTPResponse(BaseHTTPResponse):
         """
         if decode_content is None:
             decode_content = self.decode_content
+        if amt and amt < 0:
+            # Negative numbers and `None` should be treated the same.
+            amt = None
         # try and respond without going to the network
         if self._has_decoded_content:
             if not decode_content:
@@ -1180,6 +1197,11 @@ class HTTPResponse(BaseHTTPResponse):
             if self._fp.fp is None:  # type: ignore[union-attr]
                 return None
 
+            if amt and amt < 0:
+                # Negative numbers and `None` should be treated the same,
+                # but httplib handles only `None` correctly.
+                amt = None
+
             while True:
                 self._update_chunk_length()
                 if self.chunk_left == 0:

lib/urllib3/util/request.py

@@ -29,7 +29,7 @@ except ImportError:
 else:
     ACCEPT_ENCODING += ",br"
 try:
-    import zstandard as _unused_module_zstd  # type: ignore[import-not-found] # noqa: F401
+    import zstandard as _unused_module_zstd  # noqa: F401
 except ImportError:
     pass
 else:

lib/urllib3/util/retry.py

@@ -21,6 +21,8 @@ from ..exceptions import (
 from .util import reraise
 
 if typing.TYPE_CHECKING:
+    from typing_extensions import Self
+
     from ..connectionpool import ConnectionPool
     from ..response import BaseHTTPResponse
 
@@ -187,7 +189,9 @@ class Retry:
     RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
 
     #: Default headers to be used for ``remove_headers_on_redirect``
-    DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])
+    DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
+        ["Cookie", "Authorization", "Proxy-Authorization"]
+    )
 
     #: Default maximum backoff time.
     DEFAULT_BACKOFF_MAX = 120
@@ -240,7 +244,7 @@ class Retry:
         )
         self.backoff_jitter = backoff_jitter
 
-    def new(self, **kw: typing.Any) -> Retry:
+    def new(self, **kw: typing.Any) -> Self:
         params = dict(
             total=self.total,
             connect=self.connect,
@@ -429,7 +433,7 @@ class Retry:
         error: Exception | None = None,
         _pool: ConnectionPool | None = None,
         _stacktrace: TracebackType | None = None,
-    ) -> Retry:
+    ) -> Self:
         """Return a new Retry object with incremented retry counters.
 
         :param response: A response object, or None, if the server did not

lib/urllib3/util/ssl_.py

@@ -78,7 +78,7 @@ def _is_has_never_check_common_name_reliable(
 
 if typing.TYPE_CHECKING:
     from ssl import VerifyMode
-    from typing import Literal, TypedDict
+    from typing import TypedDict
 
     from .ssltransport import SSLTransport as SSLTransportType
 
@@ -365,7 +365,7 @@ def ssl_wrap_socket(
     ca_cert_dir: str | None = ...,
     key_password: str | None = ...,
     ca_cert_data: None | str | bytes = ...,
-    tls_in_tls: Literal[False] = ...,
+    tls_in_tls: typing.Literal[False] = ...,
 ) -> ssl.SSLSocket:
     ...
 

lib/urllib3/util/ssltransport.py

@@ -8,12 +8,11 @@ import typing
 from ..exceptions import ProxySchemeUnsupported
 
 if typing.TYPE_CHECKING:
-    from typing import Literal
+    from typing_extensions import Self
 
     from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT
 
 
-_SelfT = typing.TypeVar("_SelfT", bound="SSLTransport")
 _WriteBuffer = typing.Union[bytearray, memoryview]
 _ReturnValue = typing.TypeVar("_ReturnValue")
 
@@ -70,7 +69,7 @@ class SSLTransport:
         # Perform initial handshake.
         self._ssl_io_loop(self.sslobj.do_handshake)
 
-    def __enter__(self: _SelfT) -> _SelfT:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(self, *_: typing.Any) -> None:
@@ -174,12 +173,12 @@ class SSLTransport:
 
     @typing.overload
     def getpeercert(
-        self, binary_form: Literal[False] = ...
+        self, binary_form: typing.Literal[False] = ...
     ) -> _TYPE_PEER_CERT_RET_DICT | None:
         ...
 
     @typing.overload
-    def getpeercert(self, binary_form: Literal[True]) -> bytes | None:
+    def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None:
         ...
 
     def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET: