Merge branch 'UpdateTornado' into dev

CHANGES.md

@@ -11,6 +11,7 @@
 * Update Rarfile 4.1a1 (8a72967) to 4.1 (c9140d8)
 * Update soupsieve 2.4.1 (2e66beb) to 2.5.0 (dc71495)
 * Update thefuzz 0.19.0 (c2cd4f4) to 0.21.0 (0b49e4a)
+* Update Tornado Web Server 6.3.3 (e4d6984) to 6.4 (b3f2a4b)
 * Update urllib3 2.0.5 (d9f85a7) to 2.0.7 (56f01e0)
 * Add support for Brotli compression
 * Add ignore Plex extras

lib/tornado/__init__.py

@@ -22,8 +22,8 @@
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "6.3.3"
+version = "6.4"
-version_info = (6, 3, 3, 0)
+version_info = (6, 4, 0, 0)
 
 import importlib
 import typing

lib/tornado/__init__.pyi

@@ -0,0 +1,33 @@
+import typing
+
+version: str
+version_info: typing.Tuple[int, int, int, int]
+
+from . import auth
+from . import autoreload
+from . import concurrent
+from . import curl_httpclient
+from . import escape
+from . import gen
+from . import http1connection
+from . import httpclient
+from . import httpserver
+from . import httputil
+from . import ioloop
+from . import iostream
+from . import locale
+from . import locks
+from . import log
+from . import netutil
+from . import options
+from . import platform
+from . import process
+from . import queues
+from . import routing
+from . import simple_httpclient
+from . import tcpclient
+from . import tcpserver
+from . import template
+from . import testing
+from . import util
+from . import web

lib/tornado/auth.py

@@ -33,23 +33,39 @@ See the individual service classes below for complete documentation.
 
 Example usage for Google OAuth:
 
+.. testsetup::
+
+    import urllib
+
 .. testcode::
 
     class GoogleOAuth2LoginHandler(tornado.web.RequestHandler,
-                                   tornado.auth.GoogleOAuth2Mixin):
+                                    tornado.auth.GoogleOAuth2Mixin):
         async def get(self):
-            if self.get_argument('code', False):
+            # Google requires an exact match for redirect_uri, so it's
-                user = await self.get_authenticated_user(
+            # best to get it from your app configuration instead of from
-                    redirect_uri='http://your.site.com/auth/google',
+            # self.request.full_uri().
-                    code=self.get_argument('code'))
+            redirect_uri = urllib.parse.urljoin(self.application.settings['redirect_base_uri'],
-                # Save the user with e.g. set_signed_cookie
+                self.reverse_url('google_oauth'))
-            else:
+            async def get(self):
-                self.authorize_redirect(
+                if self.get_argument('code', False):
-                    redirect_uri='http://your.site.com/auth/google',
+                    access = await self.get_authenticated_user(
-                    client_id=self.settings['google_oauth']['key'],
+                        redirect_uri=redirect_uri,
-                    scope=['profile', 'email'],
+                        code=self.get_argument('code'))
-                    response_type='code',
+                    user = await self.oauth2_request(
-                    extra_params={'approval_prompt': 'auto'})
+                        "https://www.googleapis.com/oauth2/v1/userinfo",
+                        access_token=access["access_token"])
+                    # Save the user and access token. For example:
+                    user_cookie = dict(id=user["id"], access_token=access["access_token"])
+                    self.set_signed_cookie("user", json.dumps(user_cookie))
+                    self.redirect("/")
+                else:
+                    self.authorize_redirect(
+                        redirect_uri=redirect_uri,
+                        client_id=self.get_google_oauth_settings()['key'],
+                        scope=['profile', 'email'],
+                        response_type='code',
+                        extra_params={'approval_prompt': 'auto'})
 
 .. testoutput::
    :hide:
@@ -63,6 +79,7 @@ import hmac
 import time
 import urllib.parse
 import uuid
+import warnings
 
 from tornado import httpclient
 from tornado import escape
@@ -571,7 +588,13 @@ class OAuth2Mixin(object):
 
            The ``callback`` argument and returned awaitable were removed;
            this is now an ordinary synchronous function.
+
+        .. deprecated:: 6.4
+           The ``client_secret`` argument (which has never had any effect)
+           is deprecated and will be removed in Tornado 7.0.
         """
+        if client_secret is not None:
+            warnings.warn("client_secret argument is deprecated", DeprecationWarning)
         handler = cast(RequestHandler, self)
         args = {"response_type": response_type}
         if redirect_uri is not None:
@@ -705,6 +728,12 @@ class TwitterMixin(OAuthMixin):
     includes the attributes ``username``, ``name``, ``access_token``,
     and all of the custom Twitter user attributes described at
     https://dev.twitter.com/docs/api/1.1/get/users/show
+
+    .. deprecated:: 6.3
+       This class refers to version 1.1 of the Twitter API, which has been
+       deprecated by Twitter. Since Twitter has begun to limit access to its
+       API, this class will no longer be updated and will be removed in the
+       future.
     """
 
     _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token"
@@ -839,12 +868,18 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
 
     * Go to the Google Dev Console at http://console.developers.google.com
     * Select a project, or create a new one.
+    * Depending on permissions required, you may need to set your app to
+      "testing" mode and add your account as a test user, or go through
+      a verfication process. You may also need to use the "Enable
+      APIs and Services" command to enable specific services.
     * In the sidebar on the left, select Credentials.
     * Click CREATE CREDENTIALS and click OAuth client ID.
     * Under Application type, select Web application.
     * Name OAuth 2.0 client and click Create.
     * Copy the "Client secret" and "Client ID" to the application settings as
       ``{"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}}``
+    * You must register the ``redirect_uri`` you plan to use with this class
+      on the Credentials page.
 
     .. versionadded:: 3.2
     """
@@ -890,27 +925,39 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
 
         Example usage:
 
+        .. testsetup::
+
+            import urllib
+
         .. testcode::
 
             class GoogleOAuth2LoginHandler(tornado.web.RequestHandler,
                                            tornado.auth.GoogleOAuth2Mixin):
                 async def get(self):
-                    if self.get_argument('code', False):
+                    # Google requires an exact match for redirect_uri, so it's
-                        access = await self.get_authenticated_user(
+                    # best to get it from your app configuration instead of from
-                            redirect_uri='http://your.site.com/auth/google',
+                    # self.request.full_uri().
-                            code=self.get_argument('code'))
+                    redirect_uri = urllib.parse.urljoin(self.application.settings['redirect_base_uri'],
-                        user = await self.oauth2_request(
+                        self.reverse_url('google_oauth'))
-                            "https://www.googleapis.com/oauth2/v1/userinfo",
+                    async def get(self):
-                            access_token=access["access_token"])
+                        if self.get_argument('code', False):
-                        # Save the user and access token with
+                            access = await self.get_authenticated_user(
-                        # e.g. set_signed_cookie.
+                                redirect_uri=redirect_uri,
-                    else:
+                                code=self.get_argument('code'))
-                        self.authorize_redirect(
+                            user = await self.oauth2_request(
-                            redirect_uri='http://your.site.com/auth/google',
+                                "https://www.googleapis.com/oauth2/v1/userinfo",
-                            client_id=self.get_google_oauth_settings()['key'],
+                                access_token=access["access_token"])
-                            scope=['profile', 'email'],
+                            # Save the user and access token. For example:
-                            response_type='code',
+                            user_cookie = dict(id=user["id"], access_token=access["access_token"])
-                            extra_params={'approval_prompt': 'auto'})
+                            self.set_signed_cookie("user", json.dumps(user_cookie))
+                            self.redirect("/")
+                        else:
+                            self.authorize_redirect(
+                                redirect_uri=redirect_uri,
+                                client_id=self.get_google_oauth_settings()['key'],
+                                scope=['profile', 'email'],
+                                response_type='code',
+                                extra_params={'approval_prompt': 'auto'})
 
         .. testoutput::
            :hide:
@@ -971,18 +1018,21 @@ class FacebookGraphMixin(OAuth2Mixin):
             class FacebookGraphLoginHandler(tornado.web.RequestHandler,
                                             tornado.auth.FacebookGraphMixin):
               async def get(self):
-                  if self.get_argument("code", False):
+                redirect_uri = urllib.parse.urljoin(
-                      user = await self.get_authenticated_user(
+                    self.application.settings['redirect_base_uri'],
-                          redirect_uri='/auth/facebookgraph/',
+                    self.reverse_url('facebook_oauth'))
-                          client_id=self.settings["facebook_api_key"],
+                if self.get_argument("code", False):
-                          client_secret=self.settings["facebook_secret"],
+                    user = await self.get_authenticated_user(
-                          code=self.get_argument("code"))
+                        redirect_uri=redirect_uri,
-                      # Save the user with e.g. set_signed_cookie
+                        client_id=self.settings["facebook_api_key"],
-                  else:
+                        client_secret=self.settings["facebook_secret"],
-                      self.authorize_redirect(
+                        code=self.get_argument("code"))
-                          redirect_uri='/auth/facebookgraph/',
+                    # Save the user with e.g. set_signed_cookie
-                          client_id=self.settings["facebook_api_key"],
+                else:
-                          extra_params={"scope": "read_stream,offline_access"})
+                    self.authorize_redirect(
+                        redirect_uri=redirect_uri,
+                        client_id=self.settings["facebook_api_key"],
+                        extra_params={"scope": "user_posts"})
 
         .. testoutput::
            :hide:

lib/tornado/autoreload.py

@@ -60,8 +60,7 @@ import sys
 # may become relative in spite of the future import.
 #
 # We address the former problem by reconstructing the original command
-# line (Python >= 3.4) or by setting the $PYTHONPATH environment
+# line before re-execution so the new process will
-# variable (Python < 3.4) before re-execution so the new process will
 # see the correct path.  We attempt to address the latter problem when
 # tornado.autoreload is run as __main__.
 
@@ -76,8 +75,9 @@ if __name__ == "__main__":
         del sys.path[0]
 
 import functools
+import importlib.abc
 import os
-import pkgutil  # type: ignore
+import pkgutil
 import sys
 import traceback
 import types
@@ -87,18 +87,13 @@ import weakref
 from tornado import ioloop
 from tornado.log import gen_log
 from tornado import process
-from tornado.util import exec_in
 
 try:
     import signal
 except ImportError:
     signal = None  # type: ignore
 
-import typing
+from typing import Callable, Dict, Optional, List, Union
-from typing import Callable, Dict
-
-if typing.TYPE_CHECKING:
-    from typing import List, Optional, Union  # noqa: F401
 
 # os.execv is broken on Windows and can't properly parse command line
 # arguments and executable name if they contain whitespaces. subprocess
@@ -108,9 +103,11 @@ _has_execv = sys.platform != "win32"
 _watched_files = set()
 _reload_hooks = []
 _reload_attempted = False
-_io_loops = weakref.WeakKeyDictionary()  # type: ignore
+_io_loops: "weakref.WeakKeyDictionary[ioloop.IOLoop, bool]" = (
+    weakref.WeakKeyDictionary()
+)
 _autoreload_is_main = False
-_original_argv = None  # type: Optional[List[str]]
+_original_argv: Optional[List[str]] = None
 _original_spec = None
 
 
@@ -126,7 +123,7 @@ def start(check_time: int = 500) -> None:
     _io_loops[io_loop] = True
     if len(_io_loops) > 1:
         gen_log.warning("tornado.autoreload started more than once in the same process")
-    modify_times = {}  # type: Dict[str, float]
+    modify_times: Dict[str, float] = {}
     callback = functools.partial(_reload_on_update, modify_times)
     scheduler = ioloop.PeriodicCallback(callback, check_time)
     scheduler.start()
@@ -214,10 +211,7 @@ def _reload() -> None:
     # sys.path fixes: see comments at top of file.  If __main__.__spec__
     # exists, we were invoked with -m and the effective path is about to
     # change on re-exec.  Reconstruct the original command line to
-    # ensure that the new process sees the same path we did.  If
+    # ensure that the new process sees the same path we did.
-    # __spec__ is not available (Python < 3.4), check instead if
-    # sys.path[0] is an empty string and add the current directory to
-    # $PYTHONPATH.
     if _autoreload_is_main:
         assert _original_argv is not None
         spec = _original_spec
@@ -225,43 +219,25 @@ def _reload() -> None:
     else:
         spec = getattr(sys.modules["__main__"], "__spec__", None)
         argv = sys.argv
-    if spec:
+    if spec and spec.name != "__main__":
+        # __spec__ is set in two cases: when running a module, and when running a directory. (when
+        # running a file, there is no spec). In the former case, we must pass -m to maintain the
+        # module-style behavior (setting sys.path), even though python stripped -m from its argv at
+        # startup. If sys.path is exactly __main__, we're running a directory and should fall
+        # through to the non-module behavior.
+        #
+        # Some of this, including the use of exactly __main__ as a spec for directory mode,
+        # is documented at https://docs.python.org/3/library/runpy.html#runpy.run_path
         argv = ["-m", spec.name] + argv[1:]
-    else:
+
-        path_prefix = "." + os.pathsep
-        if sys.path[0] == "" and not os.environ.get("PYTHONPATH", "").startswith(
-            path_prefix
-        ):
-            os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "")
     if not _has_execv:
         subprocess.Popen([sys.executable] + argv)
         os._exit(0)
     else:
-        try:
+        os.execv(sys.executable, [sys.executable] + argv)
-            os.execv(sys.executable, [sys.executable] + argv)
-        except OSError:
-            # Mac OS X versions prior to 10.6 do not support execv in
-            # a process that contains multiple threads.  Instead of
-            # re-executing in the current process, start a new one
-            # and cause the current process to exit.  This isn't
-            # ideal since the new process is detached from the parent
-            # terminal and thus cannot easily be killed with ctrl-C,
-            # but it's better than not being able to autoreload at
-            # all.
-            # Unfortunately the errno returned in this case does not
-            # appear to be consistent, so we can't easily check for
-            # this error specifically.
-            os.spawnv(
-                os.P_NOWAIT, sys.executable, [sys.executable] + argv  # type: ignore
-            )
-            # At this point the IOLoop has been closed and finally
-            # blocks will experience errors if we allow the stack to
-            # unwind, so just exit uncleanly.
-            os._exit(0)
 
 
-_USAGE = """\
+_USAGE = """
-Usage:
   python -m tornado.autoreload -m module.to.run [args...]
   python -m tornado.autoreload path/to/script.py [args...]
 """
@@ -283,6 +259,12 @@ def main() -> None:
     # Remember that we were launched with autoreload as main.
     # The main module can be tricky; set the variables both in our globals
     # (which may be __main__) and the real importable version.
+    #
+    # We use optparse instead of the newer argparse because we want to
+    # mimic the python command-line interface which requires stopping
+    # parsing at the first positional argument. optparse supports
+    # this but as far as I can tell argparse does not.
+    import optparse
     import tornado.autoreload
 
     global _autoreload_is_main
@@ -292,38 +274,43 @@ def main() -> None:
     tornado.autoreload._original_argv = _original_argv = original_argv
     original_spec = getattr(sys.modules["__main__"], "__spec__", None)
     tornado.autoreload._original_spec = _original_spec = original_spec
-    sys.argv = sys.argv[:]
+
-    if len(sys.argv) >= 3 and sys.argv[1] == "-m":
+    parser = optparse.OptionParser(
-        mode = "module"
+        prog="python -m tornado.autoreload",
-        module = sys.argv[2]
+        usage=_USAGE,
-        del sys.argv[1:3]
+        epilog="Either -m or a path must be specified, but not both",
-    elif len(sys.argv) >= 2:
+    )
-        mode = "script"
+    parser.disable_interspersed_args()
-        script = sys.argv[1]
+    parser.add_option("-m", dest="module", metavar="module", help="module to run")
-        sys.argv = sys.argv[1:]
+    parser.add_option(
+        "--until-success",
+        action="store_true",
+        help="stop reloading after the program exist successfully (status code 0)",
+    )
+    opts, rest = parser.parse_args()
+    if opts.module is None:
+        if not rest:
+            print("Either -m or a path must be specified", file=sys.stderr)
+            sys.exit(1)
+        path = rest[0]
+        sys.argv = rest[:]
     else:
-        print(_USAGE, file=sys.stderr)
+        path = None
-        sys.exit(1)
+        sys.argv = [sys.argv[0]] + rest
 
+    # SystemExit.code is typed funny: https://github.com/python/typeshed/issues/8513
+    # All we care about is truthiness
+    exit_status: Union[int, str, None] = 1
     try:
-        if mode == "module":
+        import runpy
-            import runpy
 
-            runpy.run_module(module, run_name="__main__", alter_sys=True)
+        if opts.module is not None:
-        elif mode == "script":
+            runpy.run_module(opts.module, run_name="__main__", alter_sys=True)
-            with open(script) as f:
+        else:
-                # Execute the script in our namespace instead of creating
+            assert path is not None
-                # a new one so that something that tries to import __main__
+            runpy.run_path(path, run_name="__main__")
-                # (e.g. the unittest module) will see names defined in the
-                # script instead of just those defined in this module.
-                global __file__
-                __file__ = script
-                # If __package__ is defined, imports may be incorrectly
-                # interpreted as relative to this module.
-                global __package__
-                del __package__
-                exec_in(f.read(), globals(), globals())
     except SystemExit as e:
+        exit_status = e.code
         gen_log.info("Script exited with status %s", e.code)
     except Exception as e:
         gen_log.warning("Script exited with uncaught exception", exc_info=True)
@@ -331,7 +318,7 @@ def main() -> None:
         # never made it into sys.modules and so we won't know to watch it.
         # Just to make sure we've covered everything, walk the stack trace
         # from the exception and watch every file.
-        for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]):
+        for filename, lineno, name, line in traceback.extract_tb(sys.exc_info()[2]):
             watch(filename)
         if isinstance(e, SyntaxError):
             # SyntaxErrors are special:  their innermost stack frame is fake
@@ -340,17 +327,20 @@ def main() -> None:
             if e.filename is not None:
                 watch(e.filename)
     else:
+        exit_status = 0
         gen_log.info("Script exited normally")
     # restore sys.argv so subsequent executions will include autoreload
     sys.argv = original_argv
 
-    if mode == "module":
+    if opts.module is not None:
+        assert opts.module is not None
         # runpy did a fake import of the module as __main__, but now it's
         # no longer in sys.modules.  Figure out where it is and watch it.
-        loader = pkgutil.get_loader(module)
+        loader = pkgutil.get_loader(opts.module)
-        if loader is not None:
+        if loader is not None and isinstance(loader, importlib.abc.FileLoader):
-            watch(loader.get_filename())  # type: ignore
+            watch(loader.get_filename())
-
+    if opts.until_success and not exit_status:
+        return
     wait()
 
 

lib/tornado/concurrent.py

@@ -54,7 +54,7 @@ def is_future(x: Any) -> bool:
 
 
 class DummyExecutor(futures.Executor):
-    def submit(
+    def submit(  # type: ignore[override]
         self, fn: Callable[..., _T], *args: Any, **kwargs: Any
     ) -> "futures.Future[_T]":
         future = futures.Future()  # type: futures.Future[_T]
@@ -64,8 +64,15 @@ class DummyExecutor(futures.Executor):
             future_set_exc_info(future, sys.exc_info())
         return future
 
-    def shutdown(self, wait: bool = True) -> None:
+    if sys.version_info >= (3, 9):
-        pass
+
+        def shutdown(self, wait: bool = True, cancel_futures: bool = False) -> None:
+            pass
+
+    else:
+
+        def shutdown(self, wait: bool = True) -> None:
+            pass
 
 
 dummy_executor = DummyExecutor()
@@ -150,8 +157,7 @@ def chain_future(a: "Future[_T]", b: "Future[_T]") -> None:
 
     """
 
-    def copy(future: "Future[_T]") -> None:
+    def copy(a: "Future[_T]") -> None:
-        assert future is a
         if b.done():
             return
         if hasattr(a, "exc_info") and a.exc_info() is not None:  # type: ignore

lib/tornado/escape.py

@@ -17,9 +17,15 @@
 
 Also includes a few other miscellaneous string manipulation functions that
 have crept in over time.
+
+Many functions in this module have near-equivalents in the standard library
+(the differences mainly relate to handling of bytes and unicode strings,
+and were more relevant in Python 2). In new code, the standard library
+functions are encouraged instead of this module where applicable. See the
+docstrings on each function for details.
 """
 
-import html.entities
+import html
 import json
 import re
 import urllib.parse
@@ -30,16 +36,6 @@ import typing
 from typing import Union, Any, Optional, Dict, List, Callable
 
 
-_XHTML_ESCAPE_RE = re.compile("[&<>\"']")
-_XHTML_ESCAPE_DICT = {
-    "&": "&amp;",
-    "<": "&lt;",
-    ">": "&gt;",
-    '"': "&quot;",
-    "'": "&#39;",
-}
-
-
 def xhtml_escape(value: Union[str, bytes]) -> str:
     """Escapes a string so it is valid within HTML or XML.
 
@@ -47,25 +43,50 @@ def xhtml_escape(value: Union[str, bytes]) -> str:
     When used in attribute values the escaped strings must be enclosed
     in quotes.
 
+    Equivalent to `html.escape` except that this function always returns
+    type `str` while `html.escape` returns `bytes` if its input is `bytes`.
+
     .. versionchanged:: 3.2
 
        Added the single quote to the list of escaped characters.
+
+    .. versionchanged:: 6.4
+
+       Now simply wraps `html.escape`. This is equivalent to the old behavior
+       except that single quotes are now escaped as ``&#x27;`` instead of
+       ``&#39;`` and performance may be different.
     """
-    return _XHTML_ESCAPE_RE.sub(
+    return html.escape(to_unicode(value))
-        lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value)
-    )
 
 
 def xhtml_unescape(value: Union[str, bytes]) -> str:
-    """Un-escapes an XML-escaped string."""
+    """Un-escapes an XML-escaped string.
-    return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value))
+
+    Equivalent to `html.unescape` except that this function always returns
+    type `str` while `html.unescape` returns `bytes` if its input is `bytes`.
+
+    .. versionchanged:: 6.4
+
+       Now simply wraps `html.unescape`. This changes behavior for some inputs
+       as required by the HTML 5 specification
+       https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+
+       Some invalid inputs such as surrogates now raise an error, and numeric
+       references to certain ISO-8859-1 characters are now handled correctly.
+    """
+    return html.unescape(to_unicode(value))
 
 
 # The fact that json_encode wraps json.dumps is an implementation detail.
 # Please see https://github.com/tornadoweb/tornado/pull/706
 # before sending a pull request that adds **kwargs to this function.
 def json_encode(value: Any) -> str:
-    """JSON-encodes the given Python object."""
+    """JSON-encodes the given Python object.
+
+    Equivalent to `json.dumps` with the additional guarantee that the output
+    will never contain the character sequence ``</`` which can be problematic
+    when JSON is embedded in an HTML ``<script>`` tag.
+    """
     # JSON permits but does not require forward slashes to be escaped.
     # This is useful when json data is emitted in a <script> tag
     # in HTML, as it prevents </script> tags from prematurely terminating
@@ -78,9 +99,9 @@ def json_encode(value: Any) -> str:
 def json_decode(value: Union[str, bytes]) -> Any:
     """Returns Python objects for the given JSON string.
 
-    Supports both `str` and `bytes` inputs.
+    Supports both `str` and `bytes` inputs. Equvalent to `json.loads`.
     """
-    return json.loads(to_basestring(value))
+    return json.loads(value)
 
 
 def squeeze(value: str) -> str:
@@ -91,16 +112,20 @@ def squeeze(value: str) -> str:
 def url_escape(value: Union[str, bytes], plus: bool = True) -> str:
     """Returns a URL-encoded version of the given value.
 
-    If ``plus`` is true (the default), spaces will be represented
+    Equivalent to either `urllib.parse.quote_plus` or `urllib.parse.quote` depending on the ``plus``
-    as "+" instead of "%20".  This is appropriate for query strings
+    argument.
-    but not for the path component of a URL.  Note that this default
+
-    is the reverse of Python's urllib module.
+    If ``plus`` is true (the default), spaces will be represented as ``+`` and slashes will be
+    represented as ``%2F``.  This is appropriate for query strings. If ``plus`` is false, spaces
+    will be represented as ``%20`` and slashes are left as-is. This is appropriate for the path
+    component of a URL. Note that the default of ``plus=True`` is effectively the
+    reverse of Python's urllib module.
 
     .. versionadded:: 3.1
         The ``plus`` argument
     """
     quote = urllib.parse.quote_plus if plus else urllib.parse.quote
-    return quote(utf8(value))
+    return quote(value)
 
 
 @typing.overload
@@ -108,28 +133,29 @@ def url_unescape(value: Union[str, bytes], encoding: None, plus: bool = True) ->
     pass
 
 
-@typing.overload  # noqa: F811
+@typing.overload
 def url_unescape(
     value: Union[str, bytes], encoding: str = "utf-8", plus: bool = True
 ) -> str:
     pass
 
 
-def url_unescape(  # noqa: F811
+def url_unescape(
     value: Union[str, bytes], encoding: Optional[str] = "utf-8", plus: bool = True
 ) -> Union[str, bytes]:
     """Decodes the given value from a URL.
 
     The argument may be either a byte or unicode string.
 
-    If encoding is None, the result will be a byte string.  Otherwise,
+    If encoding is None, the result will be a byte string and this function is equivalent to
-    the result is a unicode string in the specified encoding.
+    `urllib.parse.unquote_to_bytes` if ``plus=False``.  Otherwise, the result is a unicode string in
+    the specified encoding and this function is equivalent to either `urllib.parse.unquote_plus` or
+    `urllib.parse.unquote` except that this function also accepts `bytes` as input.
 
-    If ``plus`` is true (the default), plus signs will be interpreted
+    If ``plus`` is true (the default), plus signs will be interpreted as spaces (literal plus signs
-    as spaces (literal plus signs must be represented as "%2B").  This
+    must be represented as "%2B").  This is appropriate for query strings and form-encoded values
-    is appropriate for query strings and form-encoded values but not
+    but not for the path component of a URL.  Note that this default is the reverse of Python's
-    for the path component of a URL.  Note that this default is the
+    urllib module.
-    reverse of Python's urllib module.
 
     .. versionadded:: 3.1
        The ``plus`` argument
@@ -175,17 +201,17 @@ def utf8(value: bytes) -> bytes:
     pass
 
 
-@typing.overload  # noqa: F811
+@typing.overload
 def utf8(value: str) -> bytes:
     pass
 
 
-@typing.overload  # noqa: F811
+@typing.overload
 def utf8(value: None) -> None:
     pass
 
 
-def utf8(value: Union[None, str, bytes]) -> Optional[bytes]:  # noqa: F811
+def utf8(value: Union[None, str, bytes]) -> Optional[bytes]:
     """Converts a string argument to a byte string.
 
     If the argument is already a byte string or None, it is returned unchanged.
@@ -206,17 +232,17 @@ def to_unicode(value: str) -> str:
     pass
 
 
-@typing.overload  # noqa: F811
+@typing.overload
 def to_unicode(value: bytes) -> str:
     pass
 
 
-@typing.overload  # noqa: F811
+@typing.overload
 def to_unicode(value: None) -> None:
     pass
 
 
-def to_unicode(value: Union[None, str, bytes]) -> Optional[str]:  # noqa: F811
+def to_unicode(value: Union[None, str, bytes]) -> Optional[str]:
     """Converts a string argument to a unicode string.
 
     If the argument is already a unicode string or None, it is returned
@@ -375,28 +401,3 @@ def linkify(
     # that we won't pick up &quot;, etc.
     text = _unicode(xhtml_escape(text))
     return _URL_RE.sub(make_link, text)
-
-
-def _convert_entity(m: typing.Match) -> str:
-    if m.group(1) == "#":
-        try:
-            if m.group(2)[:1].lower() == "x":
-                return chr(int(m.group(2)[1:], 16))
-            else:
-                return chr(int(m.group(2)))
-        except ValueError:
-            return "&#%s;" % m.group(2)
-    try:
-        return _HTML_UNICODE_MAP[m.group(2)]
-    except KeyError:
-        return "&%s;" % m.group(2)
-
-
-def _build_unicode_map() -> Dict[str, str]:
-    unicode_map = {}
-    for name, value in html.entities.name2codepoint.items():
-        unicode_map[name] = chr(value)
-    return unicode_map
-
-
-_HTML_UNICODE_MAP = _build_unicode_map()

lib/tornado/gen.py

@@ -840,13 +840,17 @@ class Runner(object):
             return False
 
 
-# Convert Awaitables into Futures.
+def _wrap_awaitable(awaitable: Awaitable) -> Future:
-try:
+    # Convert Awaitables into Futures.
-    _wrap_awaitable = asyncio.ensure_future
+    # Note that we use ensure_future, which handles both awaitables
-except AttributeError:
+    # and coroutines, rather than create_task, which only accepts
-    # asyncio.ensure_future was introduced in Python 3.4.4, but
+    # coroutines. (ensure_future calls create_task if given a coroutine)
-    # Debian jessie still ships with 3.4.2 so try the old name.
+    fut = asyncio.ensure_future(awaitable)
-    _wrap_awaitable = getattr(asyncio, "async")
+    # See comments on IOLoop._pending_tasks.
+    loop = IOLoop.current()
+    loop._register_task(fut)
+    fut.add_done_callback(lambda f: loop._unregister_task(f))
+    return fut
 
 
 def convert_yielded(yielded: _Yieldable) -> Future:

lib/tornado/httpserver.py

@@ -74,7 +74,7 @@ class HTTPServer(TCPServer, Configurable, httputil.HTTPServerConnectionDelegate)
     To make this server serve SSL traffic, send the ``ssl_options`` keyword
     argument with an `ssl.SSLContext` object. For compatibility with older
     versions of Python ``ssl_options`` may also be a dictionary of keyword
-    arguments for the `ssl.wrap_socket` method.::
+    arguments for the `ssl.SSLContext.wrap_socket` method.::
 
        ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
        ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"),

lib/tornado/httputil.py

@@ -856,7 +856,8 @@ def format_timestamp(
 
     The argument may be a numeric timestamp as returned by `time.time`,
     a time tuple as returned by `time.gmtime`, or a `datetime.datetime`
-    object.
+    object. Naive `datetime.datetime` objects are assumed to represent
+    UTC; aware objects are converted to UTC before formatting.
 
     >>> format_timestamp(1359312200)
     'Sun, 27 Jan 2013 18:43:20 GMT'

lib/tornado/ioloop.py

@@ -50,7 +50,7 @@ import typing
 from typing import Union, Any, Type, Optional, Callable, TypeVar, Tuple, Awaitable
 
 if typing.TYPE_CHECKING:
-    from typing import Dict, List  # noqa: F401
+    from typing import Dict, List, Set  # noqa: F401
 
     from typing_extensions import Protocol
 else:
@@ -159,6 +159,18 @@ class IOLoop(Configurable):
     # In Python 3, _ioloop_for_asyncio maps from asyncio loops to IOLoops.
     _ioloop_for_asyncio = dict()  # type: Dict[asyncio.AbstractEventLoop, IOLoop]
 
+    # Maintain a set of all pending tasks to follow the warning in the docs
+    # of asyncio.create_tasks:
+    # https://docs.python.org/3.11/library/asyncio-task.html#asyncio.create_task
+    # This ensures that all pending tasks have a strong reference so they
+    # will not be garbage collected before they are finished.
+    # (Thus avoiding "task was destroyed but it is pending" warnings)
+    # An analogous change has been proposed in cpython for 3.13:
+    # https://github.com/python/cpython/issues/91887
+    # If that change is accepted, this can eventually be removed.
+    # If it is not, we will consider the rationale and may remove this.
+    _pending_tasks = set()  # type: Set[Future]
+
     @classmethod
     def configure(
         cls, impl: "Union[None, str, Type[Configurable]]", **kwargs: Any
@@ -632,9 +644,6 @@ class IOLoop(Configurable):
         other interaction with the `IOLoop` must be done from that
         `IOLoop`'s thread.  `add_callback()` may be used to transfer
         control from other threads to the `IOLoop`'s thread.
-
-        To add a callback from a signal handler, see
-        `add_callback_from_signal`.
         """
         raise NotImplementedError()
 
@@ -643,8 +652,13 @@ class IOLoop(Configurable):
     ) -> None:
         """Calls the given callback on the next I/O loop iteration.
 
-        Safe for use from a Python signal handler; should not be used
+        Intended to be afe for use from a Python signal handler; should not be
-        otherwise.
+        used otherwise.
+
+        .. deprecated:: 6.4
+           Use ``asyncio.AbstractEventLoop.add_signal_handler`` instead.
+           This method is suspected to have been broken since Tornado 5.0 and
+           will be removed in version 7.0.
         """
         raise NotImplementedError()
 
@@ -682,22 +696,20 @@ class IOLoop(Configurable):
             # the error logging (i.e. it goes to tornado.log.app_log
             # instead of asyncio's log).
             future.add_done_callback(
-                lambda f: self._run_callback(functools.partial(callback, future))
+                lambda f: self._run_callback(functools.partial(callback, f))
             )
         else:
             assert is_future(future)
             # For concurrent futures, we use self.add_callback, so
             # it's fine if future_add_done_callback inlines that call.
-            future_add_done_callback(
+            future_add_done_callback(future, lambda f: self.add_callback(callback, f))
-                future, lambda f: self.add_callback(callback, future)
-            )
 
     def run_in_executor(
         self,
         executor: Optional[concurrent.futures.Executor],
         func: Callable[..., _T],
         *args: Any
-    ) -> Awaitable[_T]:
+    ) -> "Future[_T]":
         """Runs a function in a ``concurrent.futures.Executor``. If
         ``executor`` is ``None``, the IO loop's default executor will be used.
 
@@ -803,6 +815,12 @@ class IOLoop(Configurable):
         except OSError:
             pass
 
+    def _register_task(self, f: Future) -> None:
+        self._pending_tasks.add(f)
+
+    def _unregister_task(self, f: Future) -> None:
+        self._pending_tasks.discard(f)
+
 
 class _Timeout(object):
     """An IOLoop timeout, a UNIX timestamp and a callback"""

lib/tornado/iostream.py

@@ -1219,7 +1219,7 @@ class IOStream(BaseIOStream):
 
         The ``ssl_options`` argument may be either an `ssl.SSLContext`
         object or a dictionary of keyword arguments for the
-        `ssl.wrap_socket` function.  The ``server_hostname`` argument
+        `ssl.SSLContext.wrap_socket` function.  The ``server_hostname`` argument
         will be used for certificate validation unless disabled
         in the ``ssl_options``.
 
@@ -1324,7 +1324,7 @@ class SSLIOStream(IOStream):
     If the socket passed to the constructor is already connected,
     it should be wrapped with::
 
-        ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs)
+        ssl.SSLContext(...).wrap_socket(sock, do_handshake_on_connect=False, **kwargs)
 
     before constructing the `SSLIOStream`.  Unconnected sockets will be
     wrapped when `IOStream.connect` is finished.
@@ -1335,7 +1335,7 @@ class SSLIOStream(IOStream):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         """The ``ssl_options`` keyword argument may either be an
         `ssl.SSLContext` object or a dictionary of keywords arguments
-        for `ssl.wrap_socket`
+        for `ssl.SSLContext.wrap_socket`
         """
         self._ssl_options = kwargs.pop("ssl_options", _client_ssl_defaults)
         super().__init__(*args, **kwargs)
@@ -1413,9 +1413,9 @@ class SSLIOStream(IOStream):
             return self.close(exc_info=err)
         else:
             self._ssl_accepting = False
-            if not self._verify_cert(self.socket.getpeercert()):
+            # Prior to the introduction of SNI, this is where we would check
-                self.close()
+            # the server's claimed hostname.
-                return
+            assert ssl.HAS_SNI
             self._finish_ssl_connect()
 
     def _finish_ssl_connect(self) -> None:
@@ -1424,33 +1424,6 @@ class SSLIOStream(IOStream):
             self._ssl_connect_future = None
             future_set_result_unless_cancelled(future, self)
 
-    def _verify_cert(self, peercert: Any) -> bool:
-        """Returns ``True`` if peercert is valid according to the configured
-        validation mode and hostname.
-
-        The ssl handshake already tested the certificate for a valid
-        CA signature; the only thing that remains is to check
-        the hostname.
-        """
-        if isinstance(self._ssl_options, dict):
-            verify_mode = self._ssl_options.get("cert_reqs", ssl.CERT_NONE)
-        elif isinstance(self._ssl_options, ssl.SSLContext):
-            verify_mode = self._ssl_options.verify_mode
-        assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL)
-        if verify_mode == ssl.CERT_NONE or self._server_hostname is None:
-            return True
-        cert = self.socket.getpeercert()
-        if cert is None and verify_mode == ssl.CERT_REQUIRED:
-            gen_log.warning("No SSL certificate given")
-            return False
-        try:
-            ssl.match_hostname(peercert, self._server_hostname)
-        except ssl.CertificateError as e:
-            gen_log.warning("Invalid SSL certificate: %s" % e)
-            return False
-        else:
-            return True
-
     def _handle_read(self) -> None:
         if self._ssl_accepting:
             self._do_ssl_handshake()

lib/tornado/locale.py

@@ -333,7 +333,7 @@ class Locale(object):
         shorter: bool = False,
         full_format: bool = False,
     ) -> str:
-        """Formats the given date (which should be GMT).
+        """Formats the given date.
 
         By default, we return a relative time (e.g., "2 minutes ago"). You
         can return an absolute date string with ``relative=False``.
@@ -343,10 +343,16 @@ class Locale(object):
 
         This method is primarily intended for dates in the past.
         For dates in the future, we fall back to full format.
+
+        .. versionchanged:: 6.4
+           Aware `datetime.datetime` objects are now supported (naive
+           datetimes are still assumed to be UTC).
         """
         if isinstance(date, (int, float)):
-            date = datetime.datetime.utcfromtimestamp(date)
+            date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc)
-        now = datetime.datetime.utcnow()
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        now = datetime.datetime.now(datetime.timezone.utc)
         if date > now:
             if relative and (date - now).seconds < 60:
                 # Due to click skew, things are some things slightly

lib/tornado/netutil.py

@@ -594,7 +594,7 @@ def ssl_options_to_context(
     `~ssl.SSLContext` object.
 
     The ``ssl_options`` dictionary contains keywords to be passed to
-    `ssl.wrap_socket`.  In Python 2.7.9+, `ssl.SSLContext` objects can
+    ``ssl.SSLContext.wrap_socket``.  In Python 2.7.9+, `ssl.SSLContext` objects can
     be used instead.  This function converts the dict form to its
     `~ssl.SSLContext` equivalent, and may be used when a component which
     accepts both forms needs to upgrade to the `~ssl.SSLContext` version
@@ -652,9 +652,7 @@ def ssl_wrap_socket(
 
     ``ssl_options`` may be either an `ssl.SSLContext` object or a
     dictionary (as accepted by `ssl_options_to_context`).  Additional
-    keyword arguments are passed to ``wrap_socket`` (either the
+    keyword arguments are passed to `ssl.SSLContext.wrap_socket`.
-    `~ssl.SSLContext` method or the `ssl` module function as
-    appropriate).
 
     .. versionchanged:: 6.2
 
@@ -664,14 +662,10 @@ def ssl_wrap_socket(
     context = ssl_options_to_context(ssl_options, server_side=server_side)
     if server_side is None:
         server_side = False
-    if ssl.HAS_SNI:
+    assert ssl.HAS_SNI
-        # In python 3.4, wrap_socket only accepts the server_hostname
+    # TODO: add a unittest for hostname validation (python added server-side SNI support in 3.4)
-        # argument if HAS_SNI is true.
+    # In the meantime it can be manually tested with
-        # TODO: add a unittest (python added server-side SNI support in 3.4)
+    # python3 -m tornado.httpclient https://sni.velox.ch
-        # In the meantime it can be manually tested with
+    return context.wrap_socket(
-        # python3 -m tornado.httpclient https://sni.velox.ch
+        socket, server_hostname=server_hostname, server_side=server_side, **kwargs
-        return context.wrap_socket(
+    )
-            socket, server_hostname=server_hostname, server_side=server_side, **kwargs
-        )
-    else:
-        return context.wrap_socket(socket, server_side=server_side, **kwargs)

lib/tornado/platform/asyncio.py

@@ -36,23 +36,32 @@ import warnings
 from tornado.gen import convert_yielded
 from tornado.ioloop import IOLoop, _Selectable
 
-from typing import Any, TypeVar, Awaitable, Callable, Union, Optional, List, Dict
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Protocol,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
-if typing.TYPE_CHECKING:
-    from typing import Set, Tuple  # noqa: F401
-    from typing_extensions import Protocol
 
-    class _HasFileno(Protocol):
+class _HasFileno(Protocol):
-        def fileno(self) -> int:
+    def fileno(self) -> int:
-            pass
+        pass
 
-    _FileDescriptorLike = Union[int, _HasFileno]
+
+_FileDescriptorLike = Union[int, _HasFileno]
 
 _T = TypeVar("_T")
 
 
 # Collection of selector thread event loops to shut down on exit.
-_selector_loops = set()  # type: Set[AddThreadSelectorEventLoop]
+_selector_loops: Set["SelectorThread"] = set()
 
 
 def _atexit_callback() -> None:
@@ -64,11 +73,12 @@ def _atexit_callback() -> None:
             loop._waker_w.send(b"a")
         except BlockingIOError:
             pass
-        # If we don't join our (daemon) thread here, we may get a deadlock
+        if loop._thread is not None:
-        # during interpreter shutdown. I don't really understand why. This
+            # If we don't join our (daemon) thread here, we may get a deadlock
-        # deadlock happens every time in CI (both travis and appveyor) but
+            # during interpreter shutdown. I don't really understand why. This
-        # I've never been able to reproduce locally.
+            # deadlock happens every time in CI (both travis and appveyor) but
-        loop._thread.join()
+            # I've never been able to reproduce locally.
+            loop._thread.join()
     _selector_loops.clear()
 
 
@@ -87,16 +97,16 @@ class BaseAsyncIOLoop(IOLoop):
         # as windows where the default event loop does not implement these methods.
         self.selector_loop = asyncio_loop
         if hasattr(asyncio, "ProactorEventLoop") and isinstance(
-            asyncio_loop, asyncio.ProactorEventLoop  # type: ignore
+            asyncio_loop, asyncio.ProactorEventLoop
         ):
             # Ignore this line for mypy because the abstract method checker
             # doesn't understand dynamic proxies.
             self.selector_loop = AddThreadSelectorEventLoop(asyncio_loop)  # type: ignore
         # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler)
-        self.handlers = {}  # type: Dict[int, Tuple[Union[int, _Selectable], Callable]]
+        self.handlers: Dict[int, Tuple[Union[int, _Selectable], Callable]] = {}
         # Set of fds listening for reads/writes
-        self.readers = set()  # type: Set[int]
+        self.readers: Set[int] = set()
-        self.writers = set()  # type: Set[int]
+        self.writers: Set[int] = set()
         self.closing = False
         # If an asyncio loop was closed through an asyncio interface
         # instead of IOLoop.close(), we'd never hear about it and may
@@ -239,6 +249,7 @@ class BaseAsyncIOLoop(IOLoop):
     def add_callback_from_signal(
         self, callback: Callable, *args: Any, **kwargs: Any
     ) -> None:
+        warnings.warn("add_callback_from_signal is deprecated", DeprecationWarning)
         try:
             self.asyncio_loop.call_soon_threadsafe(
                 self._run_callback, functools.partial(callback, *args, **kwargs)
@@ -251,7 +262,7 @@ class BaseAsyncIOLoop(IOLoop):
         executor: Optional[concurrent.futures.Executor],
         func: Callable[..., _T],
         *args: Any,
-    ) -> Awaitable[_T]:
+    ) -> "asyncio.Future[_T]":
         return self.asyncio_loop.run_in_executor(executor, func, *args)
 
     def set_default_executor(self, executor: concurrent.futures.Executor) -> None:
@@ -417,87 +428,51 @@ class AnyThreadEventLoopPolicy(_BasePolicy):  # type: ignore
     def get_event_loop(self) -> asyncio.AbstractEventLoop:
         try:
             return super().get_event_loop()
-        except (RuntimeError, AssertionError):
+        except RuntimeError:
-            # This was an AssertionError in Python 3.4.2 (which ships with Debian Jessie)
-            # and changed to a RuntimeError in 3.4.3.
             # "There is no current event loop in thread %r"
             loop = self.new_event_loop()
             self.set_event_loop(loop)
             return loop
 
 
-class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
+class SelectorThread:
-    """Wrap an event loop to add implementations of the ``add_reader`` method family.
+    """Define ``add_reader`` methods to be called in a background select thread.
 
     Instances of this class start a second thread to run a selector.
-    This thread is completely hidden from the user; all callbacks are
+    This thread is completely hidden from the user;
-    run on the wrapped event loop's thread.
+    all callbacks are run on the wrapped event loop's thread.
-
-    This class is used automatically by Tornado; applications should not need
-    to refer to it directly.
-
-    It is safe to wrap any event loop with this class, although it only makes sense
-    for event loops that do not implement the ``add_reader`` family of methods
-    themselves (i.e. ``WindowsProactorEventLoop``)
-
-    Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop.
 
+    Typically used via ``AddThreadSelectorEventLoop``,
+    but can be attached to a running asyncio loop.
     """
 
-    # This class is a __getattribute__-based proxy. All attributes other than those
+    _closed = False
-    # in this set are proxied through to the underlying loop.
-    MY_ATTRIBUTES = {
-        "_consume_waker",
-        "_select_cond",
-        "_select_args",
-        "_closing_selector",
-        "_thread",
-        "_handle_event",
-        "_readers",
-        "_real_loop",
-        "_start_select",
-        "_run_select",
-        "_handle_select",
-        "_wake_selector",
-        "_waker_r",
-        "_waker_w",
-        "_writers",
-        "add_reader",
-        "add_writer",
-        "close",
-        "remove_reader",
-        "remove_writer",
-    }
-
-    def __getattribute__(self, name: str) -> Any:
-        if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES:
-            return super().__getattribute__(name)
-        return getattr(self._real_loop, name)
 
     def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None:
         self._real_loop = real_loop
 
-        # Create a thread to run the select system call. We manage this thread
-        # manually so we can trigger a clean shutdown from an atexit hook. Note
-        # that due to the order of operations at shutdown, only daemon threads
-        # can be shut down in this way (non-daemon threads would require the
-        # introduction of a new hook: https://bugs.python.org/issue41962)
         self._select_cond = threading.Condition()
-        self._select_args = (
+        self._select_args: Optional[
-            None
+            Tuple[List[_FileDescriptorLike], List[_FileDescriptorLike]]
-        )  # type: Optional[Tuple[List[_FileDescriptorLike], List[_FileDescriptorLike]]]
+        ] = None
         self._closing_selector = False
-        self._thread = threading.Thread(
+        self._thread: Optional[threading.Thread] = None
-            name="Tornado selector",
+        self._thread_manager_handle = self._thread_manager()
-            daemon=True,
-            target=self._run_select,
-        )
-        self._thread.start()
-        # Start the select loop once the loop is started.
-        self._real_loop.call_soon(self._start_select)
 
-        self._readers = {}  # type: Dict[_FileDescriptorLike, Callable]
+        async def thread_manager_anext() -> None:
-        self._writers = {}  # type: Dict[_FileDescriptorLike, Callable]
+            # the anext builtin wasn't added until 3.10. We just need to iterate
+            # this generator one step.
+            await self._thread_manager_handle.__anext__()
+
+        # When the loop starts, start the thread. Not too soon because we can't
+        # clean up if we get to this point but the event loop is closed without
+        # starting.
+        self._real_loop.call_soon(
+            lambda: self._real_loop.create_task(thread_manager_anext())
+        )
+
+        self._readers: Dict[_FileDescriptorLike, Callable] = {}
+        self._writers: Dict[_FileDescriptorLike, Callable] = {}
 
         # Writing to _waker_w will wake up the selector thread, which
         # watches for _waker_r to be readable.
@@ -507,28 +482,49 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
         _selector_loops.add(self)
         self.add_reader(self._waker_r, self._consume_waker)
 
-    def __del__(self) -> None:
-        # If the top-level application code uses asyncio interfaces to
-        # start and stop the event loop, no objects created in Tornado
-        # can get a clean shutdown notification. If we're just left to
-        # be GC'd, we must explicitly close our sockets to avoid
-        # logging warnings.
-        _selector_loops.discard(self)
-        self._waker_r.close()
-        self._waker_w.close()
-
     def close(self) -> None:
+        if self._closed:
+            return
         with self._select_cond:
             self._closing_selector = True
             self._select_cond.notify()
         self._wake_selector()
-        self._thread.join()
+        if self._thread is not None:
+            self._thread.join()
         _selector_loops.discard(self)
+        self.remove_reader(self._waker_r)
         self._waker_r.close()
         self._waker_w.close()
-        self._real_loop.close()
+        self._closed = True
+
+    async def _thread_manager(self) -> typing.AsyncGenerator[None, None]:
+        # Create a thread to run the select system call. We manage this thread
+        # manually so we can trigger a clean shutdown from an atexit hook. Note
+        # that due to the order of operations at shutdown, only daemon threads
+        # can be shut down in this way (non-daemon threads would require the
+        # introduction of a new hook: https://bugs.python.org/issue41962)
+        self._thread = threading.Thread(
+            name="Tornado selector",
+            daemon=True,
+            target=self._run_select,
+        )
+        self._thread.start()
+        self._start_select()
+        try:
+            # The presense of this yield statement means that this coroutine
+            # is actually an asynchronous generator, which has a special
+            # shutdown protocol. We wait at this yield point until the
+            # event loop's shutdown_asyncgens method is called, at which point
+            # we will get a GeneratorExit exception and can shut down the
+            # selector thread.
+            yield
+        except GeneratorExit:
+            self.close()
+            raise
 
     def _wake_selector(self) -> None:
+        if self._closed:
+            return
         try:
             self._waker_w.send(b"a")
         except BlockingIOError:
@@ -614,7 +610,7 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
                 pass
 
     def _handle_select(
-        self, rs: List["_FileDescriptorLike"], ws: List["_FileDescriptorLike"]
+        self, rs: List[_FileDescriptorLike], ws: List[_FileDescriptorLike]
     ) -> None:
         for r in rs:
             self._handle_event(r, self._readers)
@@ -624,8 +620,8 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
 
     def _handle_event(
         self,
-        fd: "_FileDescriptorLike",
+        fd: _FileDescriptorLike,
-        cb_map: Dict["_FileDescriptorLike", Callable],
+        cb_map: Dict[_FileDescriptorLike, Callable],
     ) -> None:
         try:
             callback = cb_map[fd]
@@ -634,18 +630,18 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
         callback()
 
     def add_reader(
-        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+        self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any
     ) -> None:
         self._readers[fd] = functools.partial(callback, *args)
         self._wake_selector()
 
     def add_writer(
-        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+        self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any
     ) -> None:
         self._writers[fd] = functools.partial(callback, *args)
         self._wake_selector()
 
-    def remove_reader(self, fd: "_FileDescriptorLike") -> bool:
+    def remove_reader(self, fd: _FileDescriptorLike) -> bool:
         try:
             del self._readers[fd]
         except KeyError:
@@ -653,10 +649,70 @@ class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
         self._wake_selector()
         return True
 
-    def remove_writer(self, fd: "_FileDescriptorLike") -> bool:
+    def remove_writer(self, fd: _FileDescriptorLike) -> bool:
         try:
             del self._writers[fd]
         except KeyError:
             return False
         self._wake_selector()
         return True
+
+
+class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
+    """Wrap an event loop to add implementations of the ``add_reader`` method family.
+
+    Instances of this class start a second thread to run a selector.
+    This thread is completely hidden from the user; all callbacks are
+    run on the wrapped event loop's thread.
+
+    This class is used automatically by Tornado; applications should not need
+    to refer to it directly.
+
+    It is safe to wrap any event loop with this class, although it only makes sense
+    for event loops that do not implement the ``add_reader`` family of methods
+    themselves (i.e. ``WindowsProactorEventLoop``)
+
+    Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop.
+
+    """
+
+    # This class is a __getattribute__-based proxy. All attributes other than those
+    # in this set are proxied through to the underlying loop.
+    MY_ATTRIBUTES = {
+        "_real_loop",
+        "_selector",
+        "add_reader",
+        "add_writer",
+        "close",
+        "remove_reader",
+        "remove_writer",
+    }
+
+    def __getattribute__(self, name: str) -> Any:
+        if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES:
+            return super().__getattribute__(name)
+        return getattr(self._real_loop, name)
+
+    def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None:
+        self._real_loop = real_loop
+        self._selector = SelectorThread(real_loop)
+
+    def close(self) -> None:
+        self._selector.close()
+        self._real_loop.close()
+
+    def add_reader(
+        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+    ) -> None:
+        return self._selector.add_reader(fd, callback, *args)
+
+    def add_writer(
+        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+    ) -> None:
+        return self._selector.add_writer(fd, callback, *args)
+
+    def remove_reader(self, fd: "_FileDescriptorLike") -> bool:
+        return self._selector.remove_reader(fd)
+
+    def remove_writer(self, fd: "_FileDescriptorLike") -> bool:
+        return self._selector.remove_writer(fd)

lib/tornado/process.py

@@ -17,6 +17,7 @@
 the server into multiple processes and managing subprocesses.
 """
 
+import asyncio
 import os
 import multiprocessing
 import signal
@@ -210,7 +211,6 @@ class Subprocess(object):
 
     _initialized = False
     _waiting = {}  # type: ignore
-    _old_sigchld = None
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         self.io_loop = ioloop.IOLoop.current()
@@ -322,11 +322,8 @@ class Subprocess(object):
         """
         if cls._initialized:
             return
-        io_loop = ioloop.IOLoop.current()
+        loop = asyncio.get_event_loop()
-        cls._old_sigchld = signal.signal(
+        loop.add_signal_handler(signal.SIGCHLD, cls._cleanup)
-            signal.SIGCHLD,
-            lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup),
-        )
         cls._initialized = True
 
     @classmethod
@@ -334,7 +331,8 @@ class Subprocess(object):
         """Removes the ``SIGCHLD`` handler."""
         if not cls._initialized:
             return
-        signal.signal(signal.SIGCHLD, cls._old_sigchld)
+        loop = asyncio.get_event_loop()
+        loop.remove_signal_handler(signal.SIGCHLD)
         cls._initialized = False
 
     @classmethod
@@ -352,7 +350,7 @@ class Subprocess(object):
             return
         assert ret_pid == pid
         subproc = cls._waiting.pop(pid)
-        subproc.io_loop.add_callback_from_signal(subproc._set_returncode, status)
+        subproc.io_loop.add_callback(subproc._set_returncode, status)
 
     def _set_returncode(self, status: int) -> None:
         if sys.platform == "win32":

lib/tornado/tcpserver.py

@@ -61,7 +61,7 @@ class TCPServer(object):
     To make this server serve SSL traffic, send the ``ssl_options`` keyword
     argument with an `ssl.SSLContext` object. For compatibility with older
     versions of Python ``ssl_options`` may also be a dictionary of keyword
-    arguments for the `ssl.wrap_socket` method.::
+    arguments for the `ssl.SSLContext.wrap_socket` method.::
 
        ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
        ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"),

lib/tornado/testing.py

@@ -206,10 +206,7 @@ class AsyncTestCase(unittest.TestCase):
         # this always happens in tests, so cancel any tasks that are
         # still pending by the time we get here.
         asyncio_loop = self.io_loop.asyncio_loop  # type: ignore
-        if hasattr(asyncio, "all_tasks"):  # py37
+        tasks = asyncio.all_tasks(asyncio_loop)
-            tasks = asyncio.all_tasks(asyncio_loop)  # type: ignore
-        else:
-            tasks = asyncio.Task.all_tasks(asyncio_loop)
         # Tasks that are done may still appear here and may contain
         # non-cancellation exceptions, so filter them out.
         tasks = [t for t in tasks if not t.done()]  # type: ignore
@@ -520,7 +517,9 @@ class AsyncHTTPSTestCase(AsyncHTTPTestCase):
     def default_ssl_options() -> Dict[str, Any]:
         # Testing keys were generated with:
         # openssl req -new -keyout tornado/test/test.key \
-        #                     -out tornado/test/test.crt -nodes -days 3650 -x509
+        #     -out tornado/test/test.crt \
+        #     -nodes -days 3650 -x509 \
+        #     -subj "/CN=foo.example.com" -addext "subjectAltName = DNS:foo.example.com"
         module_dir = os.path.dirname(__file__)
         return dict(
             certfile=os.path.join(module_dir, "test", "test.crt"),

lib/tornado/web.py

@@ -647,7 +647,9 @@ class RequestHandler(object):
         if domain:
             morsel["domain"] = domain
         if expires_days is not None and not expires:
-            expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
+            expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+                days=expires_days
+            )
         if expires:
             morsel["expires"] = httputil.format_timestamp(expires)
         if path:
@@ -698,7 +700,9 @@ class RequestHandler(object):
                 raise TypeError(
                     f"clear_cookie() got an unexpected keyword argument '{excluded_arg}'"
                 )
-        expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
+        expires = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+            days=365
+        )
         self.set_cookie(name, value="", expires=expires, **kwargs)
 
     def clear_all_cookies(self, **kwargs: Any) -> None:
@@ -2793,7 +2797,8 @@ class StaticFileHandler(RequestHandler):
         if cache_time > 0:
             self.set_header(
                 "Expires",
-                datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time),
+                datetime.datetime.now(datetime.timezone.utc)
+                + datetime.timedelta(seconds=cache_time),
             )
             self.set_header("Cache-Control", "max-age=" + str(cache_time))
 
@@ -2812,12 +2817,12 @@ class StaticFileHandler(RequestHandler):
         # content has not been modified
         ims_value = self.request.headers.get("If-Modified-Since")
         if ims_value is not None:
-            date_tuple = email.utils.parsedate(ims_value)
+            if_since = email.utils.parsedate_to_datetime(ims_value)
-            if date_tuple is not None:
+            if if_since.tzinfo is None:
-                if_since = datetime.datetime(*date_tuple[:6])
+                if_since = if_since.replace(tzinfo=datetime.timezone.utc)
-                assert self.modified is not None
+            assert self.modified is not None
-                if if_since >= self.modified:
+            if if_since >= self.modified:
-                    return True
+                return True
 
         return False
 
@@ -2981,6 +2986,10 @@ class StaticFileHandler(RequestHandler):
         object or None.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 6.4
+           Now returns an aware datetime object instead of a naive one.
+           Subclasses that override this method may return either kind.
         """
         stat_result = self._stat()
         # NOTE: Historically, this used stat_result[stat.ST_MTIME],
@@ -2991,7 +3000,9 @@ class StaticFileHandler(RequestHandler):
         # consistency with the past (and because we have a unit test
         # that relies on this), we truncate the float here, although
         # I'm not sure that's the right thing to do.
-        modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime))
+        modified = datetime.datetime.fromtimestamp(
+            int(stat_result.st_mtime), datetime.timezone.utc
+        )
         return modified
 
     def get_content_type(self) -> str:
@@ -3125,7 +3136,7 @@ class FallbackHandler(RequestHandler):
             django.core.handlers.wsgi.WSGIHandler())
         application = tornado.web.Application([
             (r"/foo", FooHandler),
-            (r".*", FallbackHandler, dict(fallback=wsgi_app),
+            (r".*", FallbackHandler, dict(fallback=wsgi_app)),
         ])
     """
 

lib/tornado/websocket.py

@@ -20,6 +20,7 @@ import sys
 import struct
 import tornado
 from urllib.parse import urlparse
+import warnings
 import zlib
 
 from tornado.concurrent import Future, future_set_result_unless_cancelled
@@ -1356,7 +1357,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
         ping_interval: Optional[float] = None,
         ping_timeout: Optional[float] = None,
         max_message_size: int = _default_max_message_size,
-        subprotocols: Optional[List[str]] = [],
+        subprotocols: Optional[List[str]] = None,
         resolver: Optional[Resolver] = None,
     ) -> None:
         self.connect_future = Future()  # type: Future[WebSocketClientConnection]
@@ -1410,6 +1411,15 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             104857600,
         )
 
+    def __del__(self) -> None:
+        if self.protocol is not None:
+            # Unclosed client connections can sometimes log "task was destroyed but
+            # was pending" warnings if shutdown strikes at the wrong time (such as
+            # while a ping is being processed due to ping_interval). Log our own
+            # warning to make it a little more deterministic (although it's still
+            # dependent on GC timing).
+            warnings.warn("Unclosed WebSocketClientConnection", ResourceWarning)
+
     def close(self, code: Optional[int] = None, reason: Optional[str] = None) -> None:
         """Closes the websocket connection.