From 3ee5ce8ba89f92b5a68c152306613575e69fee10 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Sun, 10 Dec 2023 05:31:11 +0000 Subject: [PATCH] =?UTF-8?q?Update=20Tornado=20Web=20Server=206.3.3=20(e4d6?= =?UTF-8?q?984)=20=E2=86=92=206.4=20(b3f2a4b).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 + lib/tornado/__init__.py | 4 +- lib/tornado/__init__.pyi | 33 ++++ lib/tornado/auth.py | 132 +++++++++++----- lib/tornado/autoreload.py | 148 +++++++++--------- lib/tornado/concurrent.py | 16 +- lib/tornado/escape.py | 129 ++++++++-------- lib/tornado/gen.py | 18 ++- lib/tornado/httpserver.py | 2 +- lib/tornado/httputil.py | 3 +- lib/tornado/ioloop.py | 40 +++-- lib/tornado/iostream.py | 39 +---- lib/tornado/locale.py | 12 +- lib/tornado/netutil.py | 24 ++- lib/tornado/platform/asyncio.py | 258 +++++++++++++++++++------------- lib/tornado/process.py | 14 +- lib/tornado/tcpserver.py | 2 +- lib/tornado/testing.py | 9 +- lib/tornado/web.py | 33 ++-- lib/tornado/websocket.py | 12 +- 20 files changed, 540 insertions(+), 389 deletions(-) create mode 100644 lib/tornado/__init__.pyi diff --git a/CHANGES.md b/CHANGES.md index c9f395f8..3f06be28 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ * Update Rarfile 4.1a1 (8a72967) to 4.1 (c9140d8) * Update soupsieve 2.4.1 (2e66beb) to 2.5.0 (dc71495) * Update thefuzz 0.19.0 (c2cd4f4) to 0.21.0 (0b49e4a) +* Update Tornado Web Server 6.3.3 (e4d6984) to 6.4 (b3f2a4b) * Update urllib3 2.0.5 (d9f85a7) to 2.0.7 (56f01e0) * Add support for Brotli compression * Add ignore Plex extras diff --git a/lib/tornado/__init__.py b/lib/tornado/__init__.py index c2a8f25b..a0ae714d 100644 --- a/lib/tornado/__init__.py +++ b/lib/tornado/__init__.py @@ -22,8 +22,8 @@ # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "6.3.3" -version_info = (6, 3, 3, 0) +version = "6.4" +version_info = (6, 4, 0, 0) import importlib import typing diff --git a/lib/tornado/__init__.pyi b/lib/tornado/__init__.pyi new file mode 100644 index 00000000..60c2a7e7 --- /dev/null +++ b/lib/tornado/__init__.pyi @@ -0,0 +1,33 @@ +import typing + +version: str +version_info: typing.Tuple[int, int, int, int] + +from . import auth +from . import autoreload +from . import concurrent +from . import curl_httpclient +from . import escape +from . import gen +from . import http1connection +from . import httpclient +from . import httpserver +from . import httputil +from . import ioloop +from . import iostream +from . import locale +from . import locks +from . import log +from . import netutil +from . import options +from . import platform +from . import process +from . import queues +from . import routing +from . import simple_httpclient +from . import tcpclient +from . import tcpserver +from . import template +from . import testing +from . import util +from . import web diff --git a/lib/tornado/auth.py b/lib/tornado/auth.py index 59501f56..d1edcc65 100644 --- a/lib/tornado/auth.py +++ b/lib/tornado/auth.py @@ -33,23 +33,39 @@ See the individual service classes below for complete documentation. Example usage for Google OAuth: +.. testsetup:: + + import urllib + .. testcode:: class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleOAuth2Mixin): + tornado.auth.GoogleOAuth2Mixin): async def get(self): - if self.get_argument('code', False): - user = await self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - # Save the user with e.g. set_signed_cookie - else: - self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.settings['google_oauth']['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) + # Google requires an exact match for redirect_uri, so it's + # best to get it from your app configuration instead of from + # self.request.full_uri(). + redirect_uri = urllib.parse.urljoin(self.application.settings['redirect_base_uri'], + self.reverse_url('google_oauth')) + async def get(self): + if self.get_argument('code', False): + access = await self.get_authenticated_user( + redirect_uri=redirect_uri, + code=self.get_argument('code')) + user = await self.oauth2_request( + "https://www.googleapis.com/oauth2/v1/userinfo", + access_token=access["access_token"]) + # Save the user and access token. For example: + user_cookie = dict(id=user["id"], access_token=access["access_token"]) + self.set_signed_cookie("user", json.dumps(user_cookie)) + self.redirect("/") + else: + self.authorize_redirect( + redirect_uri=redirect_uri, + client_id=self.get_google_oauth_settings()['key'], + scope=['profile', 'email'], + response_type='code', + extra_params={'approval_prompt': 'auto'}) .. testoutput:: :hide: @@ -63,6 +79,7 @@ import hmac import time import urllib.parse import uuid +import warnings from tornado import httpclient from tornado import escape @@ -571,7 +588,13 @@ class OAuth2Mixin(object): The ``callback`` argument and returned awaitable were removed; this is now an ordinary synchronous function. + + .. deprecated:: 6.4 + The ``client_secret`` argument (which has never had any effect) + is deprecated and will be removed in Tornado 7.0. """ + if client_secret is not None: + warnings.warn("client_secret argument is deprecated", DeprecationWarning) handler = cast(RequestHandler, self) args = {"response_type": response_type} if redirect_uri is not None: @@ -705,6 +728,12 @@ class TwitterMixin(OAuthMixin): includes the attributes ``username``, ``name``, ``access_token``, and all of the custom Twitter user attributes described at https://dev.twitter.com/docs/api/1.1/get/users/show + + .. deprecated:: 6.3 + This class refers to version 1.1 of the Twitter API, which has been + deprecated by Twitter. Since Twitter has begun to limit access to its + API, this class will no longer be updated and will be removed in the + future. """ _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" @@ -839,12 +868,18 @@ class GoogleOAuth2Mixin(OAuth2Mixin): * Go to the Google Dev Console at http://console.developers.google.com * Select a project, or create a new one. + * Depending on permissions required, you may need to set your app to + "testing" mode and add your account as a test user, or go through + a verfication process. You may also need to use the "Enable + APIs and Services" command to enable specific services. * In the sidebar on the left, select Credentials. * Click CREATE CREDENTIALS and click OAuth client ID. * Under Application type, select Web application. * Name OAuth 2.0 client and click Create. * Copy the "Client secret" and "Client ID" to the application settings as ``{"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}}`` + * You must register the ``redirect_uri`` you plan to use with this class + on the Credentials page. .. versionadded:: 3.2 """ @@ -890,27 +925,39 @@ class GoogleOAuth2Mixin(OAuth2Mixin): Example usage: + .. testsetup:: + + import urllib + .. testcode:: class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, tornado.auth.GoogleOAuth2Mixin): async def get(self): - if self.get_argument('code', False): - access = await self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - user = await self.oauth2_request( - "https://www.googleapis.com/oauth2/v1/userinfo", - access_token=access["access_token"]) - # Save the user and access token with - # e.g. set_signed_cookie. - else: - self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.get_google_oauth_settings()['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) + # Google requires an exact match for redirect_uri, so it's + # best to get it from your app configuration instead of from + # self.request.full_uri(). + redirect_uri = urllib.parse.urljoin(self.application.settings['redirect_base_uri'], + self.reverse_url('google_oauth')) + async def get(self): + if self.get_argument('code', False): + access = await self.get_authenticated_user( + redirect_uri=redirect_uri, + code=self.get_argument('code')) + user = await self.oauth2_request( + "https://www.googleapis.com/oauth2/v1/userinfo", + access_token=access["access_token"]) + # Save the user and access token. For example: + user_cookie = dict(id=user["id"], access_token=access["access_token"]) + self.set_signed_cookie("user", json.dumps(user_cookie)) + self.redirect("/") + else: + self.authorize_redirect( + redirect_uri=redirect_uri, + client_id=self.get_google_oauth_settings()['key'], + scope=['profile', 'email'], + response_type='code', + extra_params={'approval_prompt': 'auto'}) .. testoutput:: :hide: @@ -971,18 +1018,21 @@ class FacebookGraphMixin(OAuth2Mixin): class FacebookGraphLoginHandler(tornado.web.RequestHandler, tornado.auth.FacebookGraphMixin): async def get(self): - if self.get_argument("code", False): - user = await self.get_authenticated_user( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - client_secret=self.settings["facebook_secret"], - code=self.get_argument("code")) - # Save the user with e.g. set_signed_cookie - else: - self.authorize_redirect( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - extra_params={"scope": "read_stream,offline_access"}) + redirect_uri = urllib.parse.urljoin( + self.application.settings['redirect_base_uri'], + self.reverse_url('facebook_oauth')) + if self.get_argument("code", False): + user = await self.get_authenticated_user( + redirect_uri=redirect_uri, + client_id=self.settings["facebook_api_key"], + client_secret=self.settings["facebook_secret"], + code=self.get_argument("code")) + # Save the user with e.g. set_signed_cookie + else: + self.authorize_redirect( + redirect_uri=redirect_uri, + client_id=self.settings["facebook_api_key"], + extra_params={"scope": "user_posts"}) .. testoutput:: :hide: diff --git a/lib/tornado/autoreload.py b/lib/tornado/autoreload.py index 0ac44966..c6a6e82d 100644 --- a/lib/tornado/autoreload.py +++ b/lib/tornado/autoreload.py @@ -60,8 +60,7 @@ import sys # may become relative in spite of the future import. # # We address the former problem by reconstructing the original command -# line (Python >= 3.4) or by setting the $PYTHONPATH environment -# variable (Python < 3.4) before re-execution so the new process will +# line before re-execution so the new process will # see the correct path. We attempt to address the latter problem when # tornado.autoreload is run as __main__. @@ -76,8 +75,9 @@ if __name__ == "__main__": del sys.path[0] import functools +import importlib.abc import os -import pkgutil # type: ignore +import pkgutil import sys import traceback import types @@ -87,18 +87,13 @@ import weakref from tornado import ioloop from tornado.log import gen_log from tornado import process -from tornado.util import exec_in try: import signal except ImportError: signal = None # type: ignore -import typing -from typing import Callable, Dict - -if typing.TYPE_CHECKING: - from typing import List, Optional, Union # noqa: F401 +from typing import Callable, Dict, Optional, List, Union # os.execv is broken on Windows and can't properly parse command line # arguments and executable name if they contain whitespaces. subprocess @@ -108,9 +103,11 @@ _has_execv = sys.platform != "win32" _watched_files = set() _reload_hooks = [] _reload_attempted = False -_io_loops = weakref.WeakKeyDictionary() # type: ignore +_io_loops: "weakref.WeakKeyDictionary[ioloop.IOLoop, bool]" = ( + weakref.WeakKeyDictionary() +) _autoreload_is_main = False -_original_argv = None # type: Optional[List[str]] +_original_argv: Optional[List[str]] = None _original_spec = None @@ -126,7 +123,7 @@ def start(check_time: int = 500) -> None: _io_loops[io_loop] = True if len(_io_loops) > 1: gen_log.warning("tornado.autoreload started more than once in the same process") - modify_times = {} # type: Dict[str, float] + modify_times: Dict[str, float] = {} callback = functools.partial(_reload_on_update, modify_times) scheduler = ioloop.PeriodicCallback(callback, check_time) scheduler.start() @@ -214,10 +211,7 @@ def _reload() -> None: # sys.path fixes: see comments at top of file. If __main__.__spec__ # exists, we were invoked with -m and the effective path is about to # change on re-exec. Reconstruct the original command line to - # ensure that the new process sees the same path we did. If - # __spec__ is not available (Python < 3.4), check instead if - # sys.path[0] is an empty string and add the current directory to - # $PYTHONPATH. + # ensure that the new process sees the same path we did. if _autoreload_is_main: assert _original_argv is not None spec = _original_spec @@ -225,43 +219,25 @@ def _reload() -> None: else: spec = getattr(sys.modules["__main__"], "__spec__", None) argv = sys.argv - if spec: + if spec and spec.name != "__main__": + # __spec__ is set in two cases: when running a module, and when running a directory. (when + # running a file, there is no spec). In the former case, we must pass -m to maintain the + # module-style behavior (setting sys.path), even though python stripped -m from its argv at + # startup. If sys.path is exactly __main__, we're running a directory and should fall + # through to the non-module behavior. + # + # Some of this, including the use of exactly __main__ as a spec for directory mode, + # is documented at https://docs.python.org/3/library/runpy.html#runpy.run_path argv = ["-m", spec.name] + argv[1:] - else: - path_prefix = "." + os.pathsep - if sys.path[0] == "" and not os.environ.get("PYTHONPATH", "").startswith( - path_prefix - ): - os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "") + if not _has_execv: subprocess.Popen([sys.executable] + argv) os._exit(0) else: - try: - os.execv(sys.executable, [sys.executable] + argv) - except OSError: - # Mac OS X versions prior to 10.6 do not support execv in - # a process that contains multiple threads. Instead of - # re-executing in the current process, start a new one - # and cause the current process to exit. This isn't - # ideal since the new process is detached from the parent - # terminal and thus cannot easily be killed with ctrl-C, - # but it's better than not being able to autoreload at - # all. - # Unfortunately the errno returned in this case does not - # appear to be consistent, so we can't easily check for - # this error specifically. - os.spawnv( - os.P_NOWAIT, sys.executable, [sys.executable] + argv # type: ignore - ) - # At this point the IOLoop has been closed and finally - # blocks will experience errors if we allow the stack to - # unwind, so just exit uncleanly. - os._exit(0) + os.execv(sys.executable, [sys.executable] + argv) -_USAGE = """\ -Usage: +_USAGE = """ python -m tornado.autoreload -m module.to.run [args...] python -m tornado.autoreload path/to/script.py [args...] """ @@ -283,6 +259,12 @@ def main() -> None: # Remember that we were launched with autoreload as main. # The main module can be tricky; set the variables both in our globals # (which may be __main__) and the real importable version. + # + # We use optparse instead of the newer argparse because we want to + # mimic the python command-line interface which requires stopping + # parsing at the first positional argument. optparse supports + # this but as far as I can tell argparse does not. + import optparse import tornado.autoreload global _autoreload_is_main @@ -292,38 +274,43 @@ def main() -> None: tornado.autoreload._original_argv = _original_argv = original_argv original_spec = getattr(sys.modules["__main__"], "__spec__", None) tornado.autoreload._original_spec = _original_spec = original_spec - sys.argv = sys.argv[:] - if len(sys.argv) >= 3 and sys.argv[1] == "-m": - mode = "module" - module = sys.argv[2] - del sys.argv[1:3] - elif len(sys.argv) >= 2: - mode = "script" - script = sys.argv[1] - sys.argv = sys.argv[1:] + + parser = optparse.OptionParser( + prog="python -m tornado.autoreload", + usage=_USAGE, + epilog="Either -m or a path must be specified, but not both", + ) + parser.disable_interspersed_args() + parser.add_option("-m", dest="module", metavar="module", help="module to run") + parser.add_option( + "--until-success", + action="store_true", + help="stop reloading after the program exist successfully (status code 0)", + ) + opts, rest = parser.parse_args() + if opts.module is None: + if not rest: + print("Either -m or a path must be specified", file=sys.stderr) + sys.exit(1) + path = rest[0] + sys.argv = rest[:] else: - print(_USAGE, file=sys.stderr) - sys.exit(1) + path = None + sys.argv = [sys.argv[0]] + rest + # SystemExit.code is typed funny: https://github.com/python/typeshed/issues/8513 + # All we care about is truthiness + exit_status: Union[int, str, None] = 1 try: - if mode == "module": - import runpy + import runpy - runpy.run_module(module, run_name="__main__", alter_sys=True) - elif mode == "script": - with open(script) as f: - # Execute the script in our namespace instead of creating - # a new one so that something that tries to import __main__ - # (e.g. the unittest module) will see names defined in the - # script instead of just those defined in this module. - global __file__ - __file__ = script - # If __package__ is defined, imports may be incorrectly - # interpreted as relative to this module. - global __package__ - del __package__ - exec_in(f.read(), globals(), globals()) + if opts.module is not None: + runpy.run_module(opts.module, run_name="__main__", alter_sys=True) + else: + assert path is not None + runpy.run_path(path, run_name="__main__") except SystemExit as e: + exit_status = e.code gen_log.info("Script exited with status %s", e.code) except Exception as e: gen_log.warning("Script exited with uncaught exception", exc_info=True) @@ -331,7 +318,7 @@ def main() -> None: # never made it into sys.modules and so we won't know to watch it. # Just to make sure we've covered everything, walk the stack trace # from the exception and watch every file. - for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]): + for filename, lineno, name, line in traceback.extract_tb(sys.exc_info()[2]): watch(filename) if isinstance(e, SyntaxError): # SyntaxErrors are special: their innermost stack frame is fake @@ -340,17 +327,20 @@ def main() -> None: if e.filename is not None: watch(e.filename) else: + exit_status = 0 gen_log.info("Script exited normally") # restore sys.argv so subsequent executions will include autoreload sys.argv = original_argv - if mode == "module": + if opts.module is not None: + assert opts.module is not None # runpy did a fake import of the module as __main__, but now it's # no longer in sys.modules. Figure out where it is and watch it. - loader = pkgutil.get_loader(module) - if loader is not None: - watch(loader.get_filename()) # type: ignore - + loader = pkgutil.get_loader(opts.module) + if loader is not None and isinstance(loader, importlib.abc.FileLoader): + watch(loader.get_filename()) + if opts.until_success and not exit_status: + return wait() diff --git a/lib/tornado/concurrent.py b/lib/tornado/concurrent.py index 6e05346b..86bbd703 100644 --- a/lib/tornado/concurrent.py +++ b/lib/tornado/concurrent.py @@ -54,7 +54,7 @@ def is_future(x: Any) -> bool: class DummyExecutor(futures.Executor): - def submit( + def submit( # type: ignore[override] self, fn: Callable[..., _T], *args: Any, **kwargs: Any ) -> "futures.Future[_T]": future = futures.Future() # type: futures.Future[_T] @@ -64,8 +64,15 @@ class DummyExecutor(futures.Executor): future_set_exc_info(future, sys.exc_info()) return future - def shutdown(self, wait: bool = True) -> None: - pass + if sys.version_info >= (3, 9): + + def shutdown(self, wait: bool = True, cancel_futures: bool = False) -> None: + pass + + else: + + def shutdown(self, wait: bool = True) -> None: + pass dummy_executor = DummyExecutor() @@ -150,8 +157,7 @@ def chain_future(a: "Future[_T]", b: "Future[_T]") -> None: """ - def copy(future: "Future[_T]") -> None: - assert future is a + def copy(a: "Future[_T]") -> None: if b.done(): return if hasattr(a, "exc_info") and a.exc_info() is not None: # type: ignore diff --git a/lib/tornado/escape.py b/lib/tornado/escape.py index 55354c30..84abfca6 100644 --- a/lib/tornado/escape.py +++ b/lib/tornado/escape.py @@ -17,9 +17,15 @@ Also includes a few other miscellaneous string manipulation functions that have crept in over time. + +Many functions in this module have near-equivalents in the standard library +(the differences mainly relate to handling of bytes and unicode strings, +and were more relevant in Python 2). In new code, the standard library +functions are encouraged instead of this module where applicable. See the +docstrings on each function for details. """ -import html.entities +import html import json import re import urllib.parse @@ -30,16 +36,6 @@ import typing from typing import Union, Any, Optional, Dict, List, Callable -_XHTML_ESCAPE_RE = re.compile("[&<>\"']") -_XHTML_ESCAPE_DICT = { - "&": "&", - "<": "<", - ">": ">", - '"': """, - "'": "'", -} - - def xhtml_escape(value: Union[str, bytes]) -> str: """Escapes a string so it is valid within HTML or XML. @@ -47,25 +43,50 @@ def xhtml_escape(value: Union[str, bytes]) -> str: When used in attribute values the escaped strings must be enclosed in quotes. + Equivalent to `html.escape` except that this function always returns + type `str` while `html.escape` returns `bytes` if its input is `bytes`. + .. versionchanged:: 3.2 Added the single quote to the list of escaped characters. + + .. versionchanged:: 6.4 + + Now simply wraps `html.escape`. This is equivalent to the old behavior + except that single quotes are now escaped as ``'`` instead of + ``'`` and performance may be different. """ - return _XHTML_ESCAPE_RE.sub( - lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value) - ) + return html.escape(to_unicode(value)) def xhtml_unescape(value: Union[str, bytes]) -> str: - """Un-escapes an XML-escaped string.""" - return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) + """Un-escapes an XML-escaped string. + + Equivalent to `html.unescape` except that this function always returns + type `str` while `html.unescape` returns `bytes` if its input is `bytes`. + + .. versionchanged:: 6.4 + + Now simply wraps `html.unescape`. This changes behavior for some inputs + as required by the HTML 5 specification + https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state + + Some invalid inputs such as surrogates now raise an error, and numeric + references to certain ISO-8859-1 characters are now handled correctly. + """ + return html.unescape(to_unicode(value)) # The fact that json_encode wraps json.dumps is an implementation detail. # Please see https://github.com/tornadoweb/tornado/pull/706 # before sending a pull request that adds **kwargs to this function. def json_encode(value: Any) -> str: - """JSON-encodes the given Python object.""" + """JSON-encodes the given Python object. + + Equivalent to `json.dumps` with the additional guarantee that the output + will never contain the character sequence ```` tag. + """ # JSON permits but does not require forward slashes to be escaped. # This is useful when json data is emitted in a tags from prematurely terminating @@ -78,9 +99,9 @@ def json_encode(value: Any) -> str: def json_decode(value: Union[str, bytes]) -> Any: """Returns Python objects for the given JSON string. - Supports both `str` and `bytes` inputs. + Supports both `str` and `bytes` inputs. Equvalent to `json.loads`. """ - return json.loads(to_basestring(value)) + return json.loads(value) def squeeze(value: str) -> str: @@ -91,16 +112,20 @@ def squeeze(value: str) -> str: def url_escape(value: Union[str, bytes], plus: bool = True) -> str: """Returns a URL-encoded version of the given value. - If ``plus`` is true (the default), spaces will be represented - as "+" instead of "%20". This is appropriate for query strings - but not for the path component of a URL. Note that this default - is the reverse of Python's urllib module. + Equivalent to either `urllib.parse.quote_plus` or `urllib.parse.quote` depending on the ``plus`` + argument. + + If ``plus`` is true (the default), spaces will be represented as ``+`` and slashes will be + represented as ``%2F``. This is appropriate for query strings. If ``plus`` is false, spaces + will be represented as ``%20`` and slashes are left as-is. This is appropriate for the path + component of a URL. Note that the default of ``plus=True`` is effectively the + reverse of Python's urllib module. .. versionadded:: 3.1 The ``plus`` argument """ quote = urllib.parse.quote_plus if plus else urllib.parse.quote - return quote(utf8(value)) + return quote(value) @typing.overload @@ -108,28 +133,29 @@ def url_unescape(value: Union[str, bytes], encoding: None, plus: bool = True) -> pass -@typing.overload # noqa: F811 +@typing.overload def url_unescape( value: Union[str, bytes], encoding: str = "utf-8", plus: bool = True ) -> str: pass -def url_unescape( # noqa: F811 +def url_unescape( value: Union[str, bytes], encoding: Optional[str] = "utf-8", plus: bool = True ) -> Union[str, bytes]: """Decodes the given value from a URL. The argument may be either a byte or unicode string. - If encoding is None, the result will be a byte string. Otherwise, - the result is a unicode string in the specified encoding. + If encoding is None, the result will be a byte string and this function is equivalent to + `urllib.parse.unquote_to_bytes` if ``plus=False``. Otherwise, the result is a unicode string in + the specified encoding and this function is equivalent to either `urllib.parse.unquote_plus` or + `urllib.parse.unquote` except that this function also accepts `bytes` as input. - If ``plus`` is true (the default), plus signs will be interpreted - as spaces (literal plus signs must be represented as "%2B"). This - is appropriate for query strings and form-encoded values but not - for the path component of a URL. Note that this default is the - reverse of Python's urllib module. + If ``plus`` is true (the default), plus signs will be interpreted as spaces (literal plus signs + must be represented as "%2B"). This is appropriate for query strings and form-encoded values + but not for the path component of a URL. Note that this default is the reverse of Python's + urllib module. .. versionadded:: 3.1 The ``plus`` argument @@ -175,17 +201,17 @@ def utf8(value: bytes) -> bytes: pass -@typing.overload # noqa: F811 +@typing.overload def utf8(value: str) -> bytes: pass -@typing.overload # noqa: F811 +@typing.overload def utf8(value: None) -> None: pass -def utf8(value: Union[None, str, bytes]) -> Optional[bytes]: # noqa: F811 +def utf8(value: Union[None, str, bytes]) -> Optional[bytes]: """Converts a string argument to a byte string. If the argument is already a byte string or None, it is returned unchanged. @@ -206,17 +232,17 @@ def to_unicode(value: str) -> str: pass -@typing.overload # noqa: F811 +@typing.overload def to_unicode(value: bytes) -> str: pass -@typing.overload # noqa: F811 +@typing.overload def to_unicode(value: None) -> None: pass -def to_unicode(value: Union[None, str, bytes]) -> Optional[str]: # noqa: F811 +def to_unicode(value: Union[None, str, bytes]) -> Optional[str]: """Converts a string argument to a unicode string. If the argument is already a unicode string or None, it is returned @@ -375,28 +401,3 @@ def linkify( # that we won't pick up ", etc. text = _unicode(xhtml_escape(text)) return _URL_RE.sub(make_link, text) - - -def _convert_entity(m: typing.Match) -> str: - if m.group(1) == "#": - try: - if m.group(2)[:1].lower() == "x": - return chr(int(m.group(2)[1:], 16)) - else: - return chr(int(m.group(2))) - except ValueError: - return "&#%s;" % m.group(2) - try: - return _HTML_UNICODE_MAP[m.group(2)] - except KeyError: - return "&%s;" % m.group(2) - - -def _build_unicode_map() -> Dict[str, str]: - unicode_map = {} - for name, value in html.entities.name2codepoint.items(): - unicode_map[name] = chr(value) - return unicode_map - - -_HTML_UNICODE_MAP = _build_unicode_map() diff --git a/lib/tornado/gen.py b/lib/tornado/gen.py index 4819b857..dab4fd09 100644 --- a/lib/tornado/gen.py +++ b/lib/tornado/gen.py @@ -840,13 +840,17 @@ class Runner(object): return False -# Convert Awaitables into Futures. -try: - _wrap_awaitable = asyncio.ensure_future -except AttributeError: - # asyncio.ensure_future was introduced in Python 3.4.4, but - # Debian jessie still ships with 3.4.2 so try the old name. - _wrap_awaitable = getattr(asyncio, "async") +def _wrap_awaitable(awaitable: Awaitable) -> Future: + # Convert Awaitables into Futures. + # Note that we use ensure_future, which handles both awaitables + # and coroutines, rather than create_task, which only accepts + # coroutines. (ensure_future calls create_task if given a coroutine) + fut = asyncio.ensure_future(awaitable) + # See comments on IOLoop._pending_tasks. + loop = IOLoop.current() + loop._register_task(fut) + fut.add_done_callback(lambda f: loop._unregister_task(f)) + return fut def convert_yielded(yielded: _Yieldable) -> Future: diff --git a/lib/tornado/httpserver.py b/lib/tornado/httpserver.py index 77dc541e..757f711b 100644 --- a/lib/tornado/httpserver.py +++ b/lib/tornado/httpserver.py @@ -74,7 +74,7 @@ class HTTPServer(TCPServer, Configurable, httputil.HTTPServerConnectionDelegate) To make this server serve SSL traffic, send the ``ssl_options`` keyword argument with an `ssl.SSLContext` object. For compatibility with older versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: + arguments for the `ssl.SSLContext.wrap_socket` method.:: ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), diff --git a/lib/tornado/httputil.py b/lib/tornado/httputil.py index 9c341d47..b21d8046 100644 --- a/lib/tornado/httputil.py +++ b/lib/tornado/httputil.py @@ -856,7 +856,8 @@ def format_timestamp( The argument may be a numeric timestamp as returned by `time.time`, a time tuple as returned by `time.gmtime`, or a `datetime.datetime` - object. + object. Naive `datetime.datetime` objects are assumed to represent + UTC; aware objects are converted to UTC before formatting. >>> format_timestamp(1359312200) 'Sun, 27 Jan 2013 18:43:20 GMT' diff --git a/lib/tornado/ioloop.py b/lib/tornado/ioloop.py index bcdcca09..3fb1359a 100644 --- a/lib/tornado/ioloop.py +++ b/lib/tornado/ioloop.py @@ -50,7 +50,7 @@ import typing from typing import Union, Any, Type, Optional, Callable, TypeVar, Tuple, Awaitable if typing.TYPE_CHECKING: - from typing import Dict, List # noqa: F401 + from typing import Dict, List, Set # noqa: F401 from typing_extensions import Protocol else: @@ -159,6 +159,18 @@ class IOLoop(Configurable): # In Python 3, _ioloop_for_asyncio maps from asyncio loops to IOLoops. _ioloop_for_asyncio = dict() # type: Dict[asyncio.AbstractEventLoop, IOLoop] + # Maintain a set of all pending tasks to follow the warning in the docs + # of asyncio.create_tasks: + # https://docs.python.org/3.11/library/asyncio-task.html#asyncio.create_task + # This ensures that all pending tasks have a strong reference so they + # will not be garbage collected before they are finished. + # (Thus avoiding "task was destroyed but it is pending" warnings) + # An analogous change has been proposed in cpython for 3.13: + # https://github.com/python/cpython/issues/91887 + # If that change is accepted, this can eventually be removed. + # If it is not, we will consider the rationale and may remove this. + _pending_tasks = set() # type: Set[Future] + @classmethod def configure( cls, impl: "Union[None, str, Type[Configurable]]", **kwargs: Any @@ -632,9 +644,6 @@ class IOLoop(Configurable): other interaction with the `IOLoop` must be done from that `IOLoop`'s thread. `add_callback()` may be used to transfer control from other threads to the `IOLoop`'s thread. - - To add a callback from a signal handler, see - `add_callback_from_signal`. """ raise NotImplementedError() @@ -643,8 +652,13 @@ class IOLoop(Configurable): ) -> None: """Calls the given callback on the next I/O loop iteration. - Safe for use from a Python signal handler; should not be used - otherwise. + Intended to be afe for use from a Python signal handler; should not be + used otherwise. + + .. deprecated:: 6.4 + Use ``asyncio.AbstractEventLoop.add_signal_handler`` instead. + This method is suspected to have been broken since Tornado 5.0 and + will be removed in version 7.0. """ raise NotImplementedError() @@ -682,22 +696,20 @@ class IOLoop(Configurable): # the error logging (i.e. it goes to tornado.log.app_log # instead of asyncio's log). future.add_done_callback( - lambda f: self._run_callback(functools.partial(callback, future)) + lambda f: self._run_callback(functools.partial(callback, f)) ) else: assert is_future(future) # For concurrent futures, we use self.add_callback, so # it's fine if future_add_done_callback inlines that call. - future_add_done_callback( - future, lambda f: self.add_callback(callback, future) - ) + future_add_done_callback(future, lambda f: self.add_callback(callback, f)) def run_in_executor( self, executor: Optional[concurrent.futures.Executor], func: Callable[..., _T], *args: Any - ) -> Awaitable[_T]: + ) -> "Future[_T]": """Runs a function in a ``concurrent.futures.Executor``. If ``executor`` is ``None``, the IO loop's default executor will be used. @@ -803,6 +815,12 @@ class IOLoop(Configurable): except OSError: pass + def _register_task(self, f: Future) -> None: + self._pending_tasks.add(f) + + def _unregister_task(self, f: Future) -> None: + self._pending_tasks.discard(f) + class _Timeout(object): """An IOLoop timeout, a UNIX timestamp and a callback""" diff --git a/lib/tornado/iostream.py b/lib/tornado/iostream.py index e7291263..6cee1fe4 100644 --- a/lib/tornado/iostream.py +++ b/lib/tornado/iostream.py @@ -1219,7 +1219,7 @@ class IOStream(BaseIOStream): The ``ssl_options`` argument may be either an `ssl.SSLContext` object or a dictionary of keyword arguments for the - `ssl.wrap_socket` function. The ``server_hostname`` argument + `ssl.SSLContext.wrap_socket` function. The ``server_hostname`` argument will be used for certificate validation unless disabled in the ``ssl_options``. @@ -1324,7 +1324,7 @@ class SSLIOStream(IOStream): If the socket passed to the constructor is already connected, it should be wrapped with:: - ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) + ssl.SSLContext(...).wrap_socket(sock, do_handshake_on_connect=False, **kwargs) before constructing the `SSLIOStream`. Unconnected sockets will be wrapped when `IOStream.connect` is finished. @@ -1335,7 +1335,7 @@ class SSLIOStream(IOStream): def __init__(self, *args: Any, **kwargs: Any) -> None: """The ``ssl_options`` keyword argument may either be an `ssl.SSLContext` object or a dictionary of keywords arguments - for `ssl.wrap_socket` + for `ssl.SSLContext.wrap_socket` """ self._ssl_options = kwargs.pop("ssl_options", _client_ssl_defaults) super().__init__(*args, **kwargs) @@ -1413,9 +1413,9 @@ class SSLIOStream(IOStream): return self.close(exc_info=err) else: self._ssl_accepting = False - if not self._verify_cert(self.socket.getpeercert()): - self.close() - return + # Prior to the introduction of SNI, this is where we would check + # the server's claimed hostname. + assert ssl.HAS_SNI self._finish_ssl_connect() def _finish_ssl_connect(self) -> None: @@ -1424,33 +1424,6 @@ class SSLIOStream(IOStream): self._ssl_connect_future = None future_set_result_unless_cancelled(future, self) - def _verify_cert(self, peercert: Any) -> bool: - """Returns ``True`` if peercert is valid according to the configured - validation mode and hostname. - - The ssl handshake already tested the certificate for a valid - CA signature; the only thing that remains is to check - the hostname. - """ - if isinstance(self._ssl_options, dict): - verify_mode = self._ssl_options.get("cert_reqs", ssl.CERT_NONE) - elif isinstance(self._ssl_options, ssl.SSLContext): - verify_mode = self._ssl_options.verify_mode - assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL) - if verify_mode == ssl.CERT_NONE or self._server_hostname is None: - return True - cert = self.socket.getpeercert() - if cert is None and verify_mode == ssl.CERT_REQUIRED: - gen_log.warning("No SSL certificate given") - return False - try: - ssl.match_hostname(peercert, self._server_hostname) - except ssl.CertificateError as e: - gen_log.warning("Invalid SSL certificate: %s" % e) - return False - else: - return True - def _handle_read(self) -> None: if self._ssl_accepting: self._do_ssl_handshake() diff --git a/lib/tornado/locale.py b/lib/tornado/locale.py index 55072af2..c5526703 100644 --- a/lib/tornado/locale.py +++ b/lib/tornado/locale.py @@ -333,7 +333,7 @@ class Locale(object): shorter: bool = False, full_format: bool = False, ) -> str: - """Formats the given date (which should be GMT). + """Formats the given date. By default, we return a relative time (e.g., "2 minutes ago"). You can return an absolute date string with ``relative=False``. @@ -343,10 +343,16 @@ class Locale(object): This method is primarily intended for dates in the past. For dates in the future, we fall back to full format. + + .. versionchanged:: 6.4 + Aware `datetime.datetime` objects are now supported (naive + datetimes are still assumed to be UTC). """ if isinstance(date, (int, float)): - date = datetime.datetime.utcfromtimestamp(date) - now = datetime.datetime.utcnow() + date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc) + if date.tzinfo is None: + date = date.replace(tzinfo=datetime.timezone.utc) + now = datetime.datetime.now(datetime.timezone.utc) if date > now: if relative and (date - now).seconds < 60: # Due to click skew, things are some things slightly diff --git a/lib/tornado/netutil.py b/lib/tornado/netutil.py index 04db085a..18c91e67 100644 --- a/lib/tornado/netutil.py +++ b/lib/tornado/netutil.py @@ -594,7 +594,7 @@ def ssl_options_to_context( `~ssl.SSLContext` object. The ``ssl_options`` dictionary contains keywords to be passed to - `ssl.wrap_socket`. In Python 2.7.9+, `ssl.SSLContext` objects can + ``ssl.SSLContext.wrap_socket``. In Python 2.7.9+, `ssl.SSLContext` objects can be used instead. This function converts the dict form to its `~ssl.SSLContext` equivalent, and may be used when a component which accepts both forms needs to upgrade to the `~ssl.SSLContext` version @@ -652,9 +652,7 @@ def ssl_wrap_socket( ``ssl_options`` may be either an `ssl.SSLContext` object or a dictionary (as accepted by `ssl_options_to_context`). Additional - keyword arguments are passed to ``wrap_socket`` (either the - `~ssl.SSLContext` method or the `ssl` module function as - appropriate). + keyword arguments are passed to `ssl.SSLContext.wrap_socket`. .. versionchanged:: 6.2 @@ -664,14 +662,10 @@ def ssl_wrap_socket( context = ssl_options_to_context(ssl_options, server_side=server_side) if server_side is None: server_side = False - if ssl.HAS_SNI: - # In python 3.4, wrap_socket only accepts the server_hostname - # argument if HAS_SNI is true. - # TODO: add a unittest (python added server-side SNI support in 3.4) - # In the meantime it can be manually tested with - # python3 -m tornado.httpclient https://sni.velox.ch - return context.wrap_socket( - socket, server_hostname=server_hostname, server_side=server_side, **kwargs - ) - else: - return context.wrap_socket(socket, server_side=server_side, **kwargs) + assert ssl.HAS_SNI + # TODO: add a unittest for hostname validation (python added server-side SNI support in 3.4) + # In the meantime it can be manually tested with + # python3 -m tornado.httpclient https://sni.velox.ch + return context.wrap_socket( + socket, server_hostname=server_hostname, server_side=server_side, **kwargs + ) diff --git a/lib/tornado/platform/asyncio.py b/lib/tornado/platform/asyncio.py index a15a74df..79e60848 100644 --- a/lib/tornado/platform/asyncio.py +++ b/lib/tornado/platform/asyncio.py @@ -36,23 +36,32 @@ import warnings from tornado.gen import convert_yielded from tornado.ioloop import IOLoop, _Selectable -from typing import Any, TypeVar, Awaitable, Callable, Union, Optional, List, Dict +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Protocol, + Set, + Tuple, + TypeVar, + Union, +) -if typing.TYPE_CHECKING: - from typing import Set, Tuple # noqa: F401 - from typing_extensions import Protocol - class _HasFileno(Protocol): - def fileno(self) -> int: - pass +class _HasFileno(Protocol): + def fileno(self) -> int: + pass - _FileDescriptorLike = Union[int, _HasFileno] + +_FileDescriptorLike = Union[int, _HasFileno] _T = TypeVar("_T") # Collection of selector thread event loops to shut down on exit. -_selector_loops = set() # type: Set[AddThreadSelectorEventLoop] +_selector_loops: Set["SelectorThread"] = set() def _atexit_callback() -> None: @@ -64,11 +73,12 @@ def _atexit_callback() -> None: loop._waker_w.send(b"a") except BlockingIOError: pass - # If we don't join our (daemon) thread here, we may get a deadlock - # during interpreter shutdown. I don't really understand why. This - # deadlock happens every time in CI (both travis and appveyor) but - # I've never been able to reproduce locally. - loop._thread.join() + if loop._thread is not None: + # If we don't join our (daemon) thread here, we may get a deadlock + # during interpreter shutdown. I don't really understand why. This + # deadlock happens every time in CI (both travis and appveyor) but + # I've never been able to reproduce locally. + loop._thread.join() _selector_loops.clear() @@ -87,16 +97,16 @@ class BaseAsyncIOLoop(IOLoop): # as windows where the default event loop does not implement these methods. self.selector_loop = asyncio_loop if hasattr(asyncio, "ProactorEventLoop") and isinstance( - asyncio_loop, asyncio.ProactorEventLoop # type: ignore + asyncio_loop, asyncio.ProactorEventLoop ): # Ignore this line for mypy because the abstract method checker # doesn't understand dynamic proxies. self.selector_loop = AddThreadSelectorEventLoop(asyncio_loop) # type: ignore # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler) - self.handlers = {} # type: Dict[int, Tuple[Union[int, _Selectable], Callable]] + self.handlers: Dict[int, Tuple[Union[int, _Selectable], Callable]] = {} # Set of fds listening for reads/writes - self.readers = set() # type: Set[int] - self.writers = set() # type: Set[int] + self.readers: Set[int] = set() + self.writers: Set[int] = set() self.closing = False # If an asyncio loop was closed through an asyncio interface # instead of IOLoop.close(), we'd never hear about it and may @@ -239,6 +249,7 @@ class BaseAsyncIOLoop(IOLoop): def add_callback_from_signal( self, callback: Callable, *args: Any, **kwargs: Any ) -> None: + warnings.warn("add_callback_from_signal is deprecated", DeprecationWarning) try: self.asyncio_loop.call_soon_threadsafe( self._run_callback, functools.partial(callback, *args, **kwargs) @@ -251,7 +262,7 @@ class BaseAsyncIOLoop(IOLoop): executor: Optional[concurrent.futures.Executor], func: Callable[..., _T], *args: Any, - ) -> Awaitable[_T]: + ) -> "asyncio.Future[_T]": return self.asyncio_loop.run_in_executor(executor, func, *args) def set_default_executor(self, executor: concurrent.futures.Executor) -> None: @@ -417,87 +428,51 @@ class AnyThreadEventLoopPolicy(_BasePolicy): # type: ignore def get_event_loop(self) -> asyncio.AbstractEventLoop: try: return super().get_event_loop() - except (RuntimeError, AssertionError): - # This was an AssertionError in Python 3.4.2 (which ships with Debian Jessie) - # and changed to a RuntimeError in 3.4.3. + except RuntimeError: # "There is no current event loop in thread %r" loop = self.new_event_loop() self.set_event_loop(loop) return loop -class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): - """Wrap an event loop to add implementations of the ``add_reader`` method family. +class SelectorThread: + """Define ``add_reader`` methods to be called in a background select thread. Instances of this class start a second thread to run a selector. - This thread is completely hidden from the user; all callbacks are - run on the wrapped event loop's thread. - - This class is used automatically by Tornado; applications should not need - to refer to it directly. - - It is safe to wrap any event loop with this class, although it only makes sense - for event loops that do not implement the ``add_reader`` family of methods - themselves (i.e. ``WindowsProactorEventLoop``) - - Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop. + This thread is completely hidden from the user; + all callbacks are run on the wrapped event loop's thread. + Typically used via ``AddThreadSelectorEventLoop``, + but can be attached to a running asyncio loop. """ - # This class is a __getattribute__-based proxy. All attributes other than those - # in this set are proxied through to the underlying loop. - MY_ATTRIBUTES = { - "_consume_waker", - "_select_cond", - "_select_args", - "_closing_selector", - "_thread", - "_handle_event", - "_readers", - "_real_loop", - "_start_select", - "_run_select", - "_handle_select", - "_wake_selector", - "_waker_r", - "_waker_w", - "_writers", - "add_reader", - "add_writer", - "close", - "remove_reader", - "remove_writer", - } - - def __getattribute__(self, name: str) -> Any: - if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES: - return super().__getattribute__(name) - return getattr(self._real_loop, name) + _closed = False def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None: self._real_loop = real_loop - # Create a thread to run the select system call. We manage this thread - # manually so we can trigger a clean shutdown from an atexit hook. Note - # that due to the order of operations at shutdown, only daemon threads - # can be shut down in this way (non-daemon threads would require the - # introduction of a new hook: https://bugs.python.org/issue41962) self._select_cond = threading.Condition() - self._select_args = ( - None - ) # type: Optional[Tuple[List[_FileDescriptorLike], List[_FileDescriptorLike]]] + self._select_args: Optional[ + Tuple[List[_FileDescriptorLike], List[_FileDescriptorLike]] + ] = None self._closing_selector = False - self._thread = threading.Thread( - name="Tornado selector", - daemon=True, - target=self._run_select, - ) - self._thread.start() - # Start the select loop once the loop is started. - self._real_loop.call_soon(self._start_select) + self._thread: Optional[threading.Thread] = None + self._thread_manager_handle = self._thread_manager() - self._readers = {} # type: Dict[_FileDescriptorLike, Callable] - self._writers = {} # type: Dict[_FileDescriptorLike, Callable] + async def thread_manager_anext() -> None: + # the anext builtin wasn't added until 3.10. We just need to iterate + # this generator one step. + await self._thread_manager_handle.__anext__() + + # When the loop starts, start the thread. Not too soon because we can't + # clean up if we get to this point but the event loop is closed without + # starting. + self._real_loop.call_soon( + lambda: self._real_loop.create_task(thread_manager_anext()) + ) + + self._readers: Dict[_FileDescriptorLike, Callable] = {} + self._writers: Dict[_FileDescriptorLike, Callable] = {} # Writing to _waker_w will wake up the selector thread, which # watches for _waker_r to be readable. @@ -507,28 +482,49 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): _selector_loops.add(self) self.add_reader(self._waker_r, self._consume_waker) - def __del__(self) -> None: - # If the top-level application code uses asyncio interfaces to - # start and stop the event loop, no objects created in Tornado - # can get a clean shutdown notification. If we're just left to - # be GC'd, we must explicitly close our sockets to avoid - # logging warnings. - _selector_loops.discard(self) - self._waker_r.close() - self._waker_w.close() - def close(self) -> None: + if self._closed: + return with self._select_cond: self._closing_selector = True self._select_cond.notify() self._wake_selector() - self._thread.join() + if self._thread is not None: + self._thread.join() _selector_loops.discard(self) + self.remove_reader(self._waker_r) self._waker_r.close() self._waker_w.close() - self._real_loop.close() + self._closed = True + + async def _thread_manager(self) -> typing.AsyncGenerator[None, None]: + # Create a thread to run the select system call. We manage this thread + # manually so we can trigger a clean shutdown from an atexit hook. Note + # that due to the order of operations at shutdown, only daemon threads + # can be shut down in this way (non-daemon threads would require the + # introduction of a new hook: https://bugs.python.org/issue41962) + self._thread = threading.Thread( + name="Tornado selector", + daemon=True, + target=self._run_select, + ) + self._thread.start() + self._start_select() + try: + # The presense of this yield statement means that this coroutine + # is actually an asynchronous generator, which has a special + # shutdown protocol. We wait at this yield point until the + # event loop's shutdown_asyncgens method is called, at which point + # we will get a GeneratorExit exception and can shut down the + # selector thread. + yield + except GeneratorExit: + self.close() + raise def _wake_selector(self) -> None: + if self._closed: + return try: self._waker_w.send(b"a") except BlockingIOError: @@ -614,7 +610,7 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): pass def _handle_select( - self, rs: List["_FileDescriptorLike"], ws: List["_FileDescriptorLike"] + self, rs: List[_FileDescriptorLike], ws: List[_FileDescriptorLike] ) -> None: for r in rs: self._handle_event(r, self._readers) @@ -624,8 +620,8 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): def _handle_event( self, - fd: "_FileDescriptorLike", - cb_map: Dict["_FileDescriptorLike", Callable], + fd: _FileDescriptorLike, + cb_map: Dict[_FileDescriptorLike, Callable], ) -> None: try: callback = cb_map[fd] @@ -634,18 +630,18 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): callback() def add_reader( - self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any ) -> None: self._readers[fd] = functools.partial(callback, *args) self._wake_selector() def add_writer( - self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any ) -> None: self._writers[fd] = functools.partial(callback, *args) self._wake_selector() - def remove_reader(self, fd: "_FileDescriptorLike") -> bool: + def remove_reader(self, fd: _FileDescriptorLike) -> bool: try: del self._readers[fd] except KeyError: @@ -653,10 +649,70 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): self._wake_selector() return True - def remove_writer(self, fd: "_FileDescriptorLike") -> bool: + def remove_writer(self, fd: _FileDescriptorLike) -> bool: try: del self._writers[fd] except KeyError: return False self._wake_selector() return True + + +class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop): + """Wrap an event loop to add implementations of the ``add_reader`` method family. + + Instances of this class start a second thread to run a selector. + This thread is completely hidden from the user; all callbacks are + run on the wrapped event loop's thread. + + This class is used automatically by Tornado; applications should not need + to refer to it directly. + + It is safe to wrap any event loop with this class, although it only makes sense + for event loops that do not implement the ``add_reader`` family of methods + themselves (i.e. ``WindowsProactorEventLoop``) + + Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop. + + """ + + # This class is a __getattribute__-based proxy. All attributes other than those + # in this set are proxied through to the underlying loop. + MY_ATTRIBUTES = { + "_real_loop", + "_selector", + "add_reader", + "add_writer", + "close", + "remove_reader", + "remove_writer", + } + + def __getattribute__(self, name: str) -> Any: + if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES: + return super().__getattribute__(name) + return getattr(self._real_loop, name) + + def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None: + self._real_loop = real_loop + self._selector = SelectorThread(real_loop) + + def close(self) -> None: + self._selector.close() + self._real_loop.close() + + def add_reader( + self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + ) -> None: + return self._selector.add_reader(fd, callback, *args) + + def add_writer( + self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + ) -> None: + return self._selector.add_writer(fd, callback, *args) + + def remove_reader(self, fd: "_FileDescriptorLike") -> bool: + return self._selector.remove_reader(fd) + + def remove_writer(self, fd: "_FileDescriptorLike") -> bool: + return self._selector.remove_writer(fd) diff --git a/lib/tornado/process.py b/lib/tornado/process.py index 26428feb..12e3eb64 100644 --- a/lib/tornado/process.py +++ b/lib/tornado/process.py @@ -17,6 +17,7 @@ the server into multiple processes and managing subprocesses. """ +import asyncio import os import multiprocessing import signal @@ -210,7 +211,6 @@ class Subprocess(object): _initialized = False _waiting = {} # type: ignore - _old_sigchld = None def __init__(self, *args: Any, **kwargs: Any) -> None: self.io_loop = ioloop.IOLoop.current() @@ -322,11 +322,8 @@ class Subprocess(object): """ if cls._initialized: return - io_loop = ioloop.IOLoop.current() - cls._old_sigchld = signal.signal( - signal.SIGCHLD, - lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup), - ) + loop = asyncio.get_event_loop() + loop.add_signal_handler(signal.SIGCHLD, cls._cleanup) cls._initialized = True @classmethod @@ -334,7 +331,8 @@ class Subprocess(object): """Removes the ``SIGCHLD`` handler.""" if not cls._initialized: return - signal.signal(signal.SIGCHLD, cls._old_sigchld) + loop = asyncio.get_event_loop() + loop.remove_signal_handler(signal.SIGCHLD) cls._initialized = False @classmethod @@ -352,7 +350,7 @@ class Subprocess(object): return assert ret_pid == pid subproc = cls._waiting.pop(pid) - subproc.io_loop.add_callback_from_signal(subproc._set_returncode, status) + subproc.io_loop.add_callback(subproc._set_returncode, status) def _set_returncode(self, status: int) -> None: if sys.platform == "win32": diff --git a/lib/tornado/tcpserver.py b/lib/tornado/tcpserver.py index deab8f2a..02c0ca0c 100644 --- a/lib/tornado/tcpserver.py +++ b/lib/tornado/tcpserver.py @@ -61,7 +61,7 @@ class TCPServer(object): To make this server serve SSL traffic, send the ``ssl_options`` keyword argument with an `ssl.SSLContext` object. For compatibility with older versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: + arguments for the `ssl.SSLContext.wrap_socket` method.:: ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), diff --git a/lib/tornado/testing.py b/lib/tornado/testing.py index 9bfadf45..bdbff87b 100644 --- a/lib/tornado/testing.py +++ b/lib/tornado/testing.py @@ -206,10 +206,7 @@ class AsyncTestCase(unittest.TestCase): # this always happens in tests, so cancel any tasks that are # still pending by the time we get here. asyncio_loop = self.io_loop.asyncio_loop # type: ignore - if hasattr(asyncio, "all_tasks"): # py37 - tasks = asyncio.all_tasks(asyncio_loop) # type: ignore - else: - tasks = asyncio.Task.all_tasks(asyncio_loop) + tasks = asyncio.all_tasks(asyncio_loop) # Tasks that are done may still appear here and may contain # non-cancellation exceptions, so filter them out. tasks = [t for t in tasks if not t.done()] # type: ignore @@ -520,7 +517,9 @@ class AsyncHTTPSTestCase(AsyncHTTPTestCase): def default_ssl_options() -> Dict[str, Any]: # Testing keys were generated with: # openssl req -new -keyout tornado/test/test.key \ - # -out tornado/test/test.crt -nodes -days 3650 -x509 + # -out tornado/test/test.crt \ + # -nodes -days 3650 -x509 \ + # -subj "/CN=foo.example.com" -addext "subjectAltName = DNS:foo.example.com" module_dir = os.path.dirname(__file__) return dict( certfile=os.path.join(module_dir, "test", "test.crt"), diff --git a/lib/tornado/web.py b/lib/tornado/web.py index 56514049..03939647 100644 --- a/lib/tornado/web.py +++ b/lib/tornado/web.py @@ -647,7 +647,9 @@ class RequestHandler(object): if domain: morsel["domain"] = domain if expires_days is not None and not expires: - expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) + expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + days=expires_days + ) if expires: morsel["expires"] = httputil.format_timestamp(expires) if path: @@ -698,7 +700,9 @@ class RequestHandler(object): raise TypeError( f"clear_cookie() got an unexpected keyword argument '{excluded_arg}'" ) - expires = datetime.datetime.utcnow() - datetime.timedelta(days=365) + expires = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + days=365 + ) self.set_cookie(name, value="", expires=expires, **kwargs) def clear_all_cookies(self, **kwargs: Any) -> None: @@ -2793,7 +2797,8 @@ class StaticFileHandler(RequestHandler): if cache_time > 0: self.set_header( "Expires", - datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time), + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=cache_time), ) self.set_header("Cache-Control", "max-age=" + str(cache_time)) @@ -2812,12 +2817,12 @@ class StaticFileHandler(RequestHandler): # content has not been modified ims_value = self.request.headers.get("If-Modified-Since") if ims_value is not None: - date_tuple = email.utils.parsedate(ims_value) - if date_tuple is not None: - if_since = datetime.datetime(*date_tuple[:6]) - assert self.modified is not None - if if_since >= self.modified: - return True + if_since = email.utils.parsedate_to_datetime(ims_value) + if if_since.tzinfo is None: + if_since = if_since.replace(tzinfo=datetime.timezone.utc) + assert self.modified is not None + if if_since >= self.modified: + return True return False @@ -2981,6 +2986,10 @@ class StaticFileHandler(RequestHandler): object or None. .. versionadded:: 3.1 + + .. versionchanged:: 6.4 + Now returns an aware datetime object instead of a naive one. + Subclasses that override this method may return either kind. """ stat_result = self._stat() # NOTE: Historically, this used stat_result[stat.ST_MTIME], @@ -2991,7 +3000,9 @@ class StaticFileHandler(RequestHandler): # consistency with the past (and because we have a unit test # that relies on this), we truncate the float here, although # I'm not sure that's the right thing to do. - modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime)) + modified = datetime.datetime.fromtimestamp( + int(stat_result.st_mtime), datetime.timezone.utc + ) return modified def get_content_type(self) -> str: @@ -3125,7 +3136,7 @@ class FallbackHandler(RequestHandler): django.core.handlers.wsgi.WSGIHandler()) application = tornado.web.Application([ (r"/foo", FooHandler), - (r".*", FallbackHandler, dict(fallback=wsgi_app), + (r".*", FallbackHandler, dict(fallback=wsgi_app)), ]) """ diff --git a/lib/tornado/websocket.py b/lib/tornado/websocket.py index d0abd425..fbfd7008 100644 --- a/lib/tornado/websocket.py +++ b/lib/tornado/websocket.py @@ -20,6 +20,7 @@ import sys import struct import tornado from urllib.parse import urlparse +import warnings import zlib from tornado.concurrent import Future, future_set_result_unless_cancelled @@ -1356,7 +1357,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): ping_interval: Optional[float] = None, ping_timeout: Optional[float] = None, max_message_size: int = _default_max_message_size, - subprotocols: Optional[List[str]] = [], + subprotocols: Optional[List[str]] = None, resolver: Optional[Resolver] = None, ) -> None: self.connect_future = Future() # type: Future[WebSocketClientConnection] @@ -1410,6 +1411,15 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): 104857600, ) + def __del__(self) -> None: + if self.protocol is not None: + # Unclosed client connections can sometimes log "task was destroyed but + # was pending" warnings if shutdown strikes at the wrong time (such as + # while a ping is being processed due to ping_interval). Log our own + # warning to make it a little more deterministic (although it's still + # dependent on GC timing). + warnings.warn("Unclosed WebSocketClientConnection", ResourceWarning) + def close(self, code: Optional[int] = None, reason: Optional[str] = None) -> None: """Closes the websocket connection.