Merge branch 'feature/UpdateTornado' into develop

CHANGES.md

@@ -25,12 +25,13 @@
 * Update scandir 1.3 to 1.6 (c3592ee)
 * Update SimpleJSON library 3.10.0 (c52efea) to 3.13.2 (6ffddbe)
 * Update Six compatibility library 1.10.0 (r433) to 1.11.0 (68112f3)
-* Update Tornado Web Server 4.5.1 (79b2683) to 5.0.1 (35a538f)
+* Update Tornado Web Server 5.0.1 (35a538f) to 5.1.dev1 (415f453)
 * Update unidecode library 0.04.21 (e99b0e3) to 1.0.22 (81f938d)
 * Update webencodings 0.5 (3970651) to 0.5.1 (fa2cb5d)
 * Update xmltodict library 0.10.2 (375d3a6) to 0.11.0 (79ac9a4)
 
 [develop changelog]
+* Update Tornado Web Server 4.5.1 (79b2683) to 5.0.1 (35a538f)
 * Change pick up the stragglers late to the more security party
 * Change remove redundant xsrf handling for POSTs that don't use web and API
 * Change add xsrf protection support to media processing scripts

lib/tornado/auth.py

@@ -202,9 +202,9 @@ class OpenIdMixin(object):
         url = self._OPENID_ENDPOINT
         if http_client is None:
             http_client = self.get_auth_http_client()
-        http_client.fetch(url, functools.partial(
+        fut = http_client.fetch(url, method="POST", body=urllib_parse.urlencode(args))
-            self._on_authentication_verified, callback),
+        fut.add_done_callback(functools.partial(
-            method="POST", body=urllib_parse.urlencode(args))
+            self._on_authentication_verified, callback))
 
     def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None):
         url = urlparse.urljoin(self.request.full_url(), callback_uri)
@@ -254,11 +254,16 @@ class OpenIdMixin(object):
             })
         return args
 
-    def _on_authentication_verified(self, future, response):
+    def _on_authentication_verified(self, future, response_fut):
-        if response.error or b"is_valid:true" not in response.body:
+        try:
+            response = response_fut.result()
+        except Exception as e:
             future.set_exception(AuthError(
-                "Invalid OpenID response: %s" % (response.error or
+                "Error response %s" % e))
-                                                 response.body)))
+            return
+        if b"is_valid:true" not in response.body:
+            future.set_exception(AuthError(
+                "Invalid OpenID response: %s" % response.body))
             return
 
         # Make sure we got back at least an email from attribute exchange
@@ -374,17 +379,17 @@ class OAuthMixin(object):
         if http_client is None:
             http_client = self.get_auth_http_client()
         if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a":
-            http_client.fetch(
+            fut = http_client.fetch(
                 self._oauth_request_token_url(callback_uri=callback_uri,
-                                              extra_params=extra_params),
+                                              extra_params=extra_params))
-                functools.partial(
+            fut.add_done_callback(functools.partial(
-                    self._on_request_token,
+                self._on_request_token,
-                    self._OAUTH_AUTHORIZE_URL,
+                self._OAUTH_AUTHORIZE_URL,
-                    callback_uri,
+                callback_uri,
-                    callback))
+                callback))
         else:
-            http_client.fetch(
+            fut = http_client.fetch(self._oauth_request_token_url())
-                self._oauth_request_token_url(),
+            fut.add_done_callback(
                 functools.partial(
                     self._on_request_token, self._OAUTH_AUTHORIZE_URL,
                     callback_uri,
@@ -427,8 +432,8 @@ class OAuthMixin(object):
             token["verifier"] = oauth_verifier
         if http_client is None:
             http_client = self.get_auth_http_client()
-        http_client.fetch(self._oauth_access_token_url(token),
+        fut = http_client.fetch(self._oauth_access_token_url(token))
-                          functools.partial(self._on_access_token, callback))
+        fut.add_done_callback(functools.partial(self._on_access_token, callback))
 
     def _oauth_request_token_url(self, callback_uri=None, extra_params=None):
         consumer_token = self._oauth_consumer_token()
@@ -456,9 +461,11 @@ class OAuthMixin(object):
         return url + "?" + urllib_parse.urlencode(args)
 
     def _on_request_token(self, authorize_url, callback_uri, callback,
-                          response):
+                          response_fut):
-        if response.error:
+        try:
-            raise Exception("Could not get request token: %s" % response.error)
+            response = response_fut.result()
+        except Exception as e:
+            raise Exception("Could not get request token: %s" % e)
         request_token = _oauth_parse_response(response.body)
         data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" +
                 base64.b64encode(escape.utf8(request_token["secret"])))
@@ -498,8 +505,10 @@ class OAuthMixin(object):
         args["oauth_signature"] = signature
         return url + "?" + urllib_parse.urlencode(args)
 
-    def _on_access_token(self, future, response):
+    def _on_access_token(self, future, response_fut):
-        if response.error:
+        try:
+            response = response_fut.result()
+        except Exception:
             future.set_exception(AuthError("Could not fetch access token"))
             return
 
@@ -707,15 +716,16 @@ class OAuth2Mixin(object):
         callback = functools.partial(self._on_oauth2_request, callback)
         http = self.get_auth_http_client()
         if post_args is not None:
-            http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
+            fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args))
-                       callback=callback)
         else:
-            http.fetch(url, callback=callback)
+            fut = http.fetch(url)
+        fut.add_done_callback(callback)
 
-    def _on_oauth2_request(self, future, response):
+    def _on_oauth2_request(self, future, response_fut):
-        if response.error:
+        try:
-            future.set_exception(AuthError("Error response %s fetching %s" %
+            response = response_fut.result()
-                                           (response.error, response.request.url)))
+        except Exception as e:
+            future.set_exception(AuthError("Error response %s" % e))
             return
 
         future_set_result_unless_cancelled(future, escape.json_decode(response.body))
@@ -857,18 +867,19 @@ class TwitterMixin(OAuthMixin):
         if args:
             url += "?" + urllib_parse.urlencode(args)
         http = self.get_auth_http_client()
-        http_callback = functools.partial(self._on_twitter_request, callback)
+        http_callback = functools.partial(self._on_twitter_request, callback, url)
         if post_args is not None:
-            http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
+            fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args))
-                       callback=http_callback)
         else:
-            http.fetch(url, callback=http_callback)
+            fut = http.fetch(url)
+        fut.add_done_callback(http_callback)
 
-    def _on_twitter_request(self, future, response):
+    def _on_twitter_request(self, future, url, response_fut):
-        if response.error:
+        try:
+            response = response_fut.result()
+        except Exception as e:
             future.set_exception(AuthError(
-                "Error response %s fetching %s" % (response.error,
+                "Error response %s fetching %s" % (e, url)))
-                                                   response.request.url)))
             return
         future_set_result_unless_cancelled(future, escape.json_decode(response.body))
 
@@ -967,16 +978,18 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
             "grant_type": "authorization_code",
         })
 
-        http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
+        fut = http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
-                   functools.partial(self._on_access_token, callback),
+                         method="POST",
-                   method="POST",
+                         headers={'Content-Type': 'application/x-www-form-urlencoded'},
-                   headers={'Content-Type': 'application/x-www-form-urlencoded'},
+                         body=body)
-                   body=body)
+        fut.add_done_callback(functools.partial(self._on_access_token, callback))
 
-    def _on_access_token(self, future, response):
+    def _on_access_token(self, future, response_fut):
         """Callback function for the exchange to the access token."""
-        if response.error:
+        try:
-            future.set_exception(AuthError('Google auth error: %s' % str(response)))
+            response = response_fut.result()
+        except Exception as e:
+            future.set_exception(AuthError('Google auth error: %s' % str(e)))
             return
 
         args = escape.json_decode(response.body)
@@ -1053,15 +1066,17 @@ class FacebookGraphMixin(OAuth2Mixin):
         if extra_fields:
             fields.update(extra_fields)
 
-        http.fetch(self._oauth_request_token_url(**args),
+        fut = http.fetch(self._oauth_request_token_url(**args))
-                   functools.partial(self._on_access_token, redirect_uri, client_id,
+        fut.add_done_callback(functools.partial(self._on_access_token, redirect_uri, client_id,
-                                     client_secret, callback, fields))
+                                                client_secret, callback, fields))
 
     @gen.coroutine
     def _on_access_token(self, redirect_uri, client_id, client_secret,
-                         future, fields, response):
+                         future, fields, response_fut):
-        if response.error:
+        try:
-            future.set_exception(AuthError('Facebook auth error: %s' % str(response)))
+            response = response_fut.result()
+        except Exception as e:
+            future.set_exception(AuthError('Facebook auth error: %s' % str(e)))
             return
 
         args = escape.json_decode(response.body)

lib/tornado/httpclient.py

@@ -42,6 +42,7 @@ from __future__ import absolute_import, division, print_function
 
 import functools
 import time
+import warnings
 import weakref
 
 from tornado.concurrent import Future, future_set_result_unless_cancelled
@@ -238,6 +239,18 @@ class AsyncHTTPClient(Configurable):
         In the callback interface, `HTTPError` is not automatically raised.
         Instead, you must check the response's ``error`` attribute or
         call its `~HTTPResponse.rethrow` method.
+
+        .. deprecated:: 5.1
+
+           The ``callback`` argument is deprecated and will be removed
+           in 6.0. Use the returned `.Future` instead.
+
+           The ``raise_error=False`` argument currently suppresses
+           *all* errors, encapsulating them in `HTTPResponse` objects
+           with a 599 response code. This will change in Tornado 6.0:
+           ``raise_error=False`` will only affect the `HTTPError`
+           raised when a non-200 response code is used.
+
         """
         if self._closed:
             raise RuntimeError("fetch() called on closed AsyncHTTPClient")
@@ -253,6 +266,8 @@ class AsyncHTTPClient(Configurable):
         request = _RequestProxy(request, self.defaults)
         future = Future()
         if callback is not None:
+            warnings.warn("callback arguments are deprecated, use the returned Future instead",
+                          DeprecationWarning)
             callback = stack_context.wrap(callback)
 
             def handle_future(future):
@@ -270,8 +285,13 @@ class AsyncHTTPClient(Configurable):
 
         def handle_response(response):
             if raise_error and response.error:
+                if isinstance(response.error, HTTPError):
+                    response.error.response = response
                 future.set_exception(response.error)
             else:
+                if response.error and not response._error_is_response_code:
+                    warnings.warn("raise_error=False will allow '%s' to be raised in the future" %
+                                  response.error, DeprecationWarning)
                 future_set_result_unless_cancelled(future, response)
         self.fetch_impl(request, handle_response)
         return future
@@ -585,8 +605,10 @@ class HTTPResponse(object):
             self.effective_url = request.url
         else:
             self.effective_url = effective_url
+        self._error_is_response_code = False
         if error is None:
             if self.code < 200 or self.code >= 300:
+                self._error_is_response_code = True
                 self.error = HTTPError(self.code, message=self.reason,
                                        response=self)
             else:
@@ -615,7 +637,7 @@ class HTTPResponse(object):
         return "%s(%s)" % (self.__class__.__name__, args)
 
 
-class HTTPError(Exception):
+class HTTPClientError(Exception):
     """Exception thrown for an unsuccessful HTTP request.
 
     Attributes:
@@ -628,12 +650,18 @@ class HTTPError(Exception):
     Note that if ``follow_redirects`` is False, redirects become HTTPErrors,
     and you can look at ``error.response.headers['Location']`` to see the
     destination of the redirect.
+
+    .. versionchanged:: 5.1
+
+       Renamed from ``HTTPError`` to ``HTTPClientError`` to avoid collisions with
+       `tornado.web.HTTPError`. The name ``tornado.httpclient.HTTPError`` remains
+       as an alias.
     """
     def __init__(self, code, message=None, response=None):
         self.code = code
         self.message = message or httputil.responses.get(code, "Unknown")
         self.response = response
-        super(HTTPError, self).__init__(code, message, response)
+        super(HTTPClientError, self).__init__(code, message, response)
 
     def __str__(self):
         return "HTTP %d: %s" % (self.code, self.message)
@@ -645,6 +673,9 @@ class HTTPError(Exception):
     __repr__ = __str__
 
 
+HTTPError = HTTPClientError
+
+
 class _RequestProxy(object):
     """Combines an object with a dictionary of defaults.
 

lib/tornado/ioloop.py

@@ -1213,11 +1213,31 @@ class PeriodicCallback(object):
 
     def _schedule_next(self):
         if self._running:
-            current_time = self.io_loop.time()
+            self._update_next(self.io_loop.time())
-
-            if self._next_timeout <= current_time:
-                callback_time_sec = self.callback_time / 1000.0
-                self._next_timeout += (math.floor((current_time - self._next_timeout) /
-                                                  callback_time_sec) + 1) * callback_time_sec
-
             self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run)
+
+    def _update_next(self, current_time):
+        callback_time_sec = self.callback_time / 1000.0
+        if self._next_timeout <= current_time:
+            # The period should be measured from the start of one call
+            # to the start of the next. If one call takes too long,
+            # skip cycles to get back to a multiple of the original
+            # schedule.
+            self._next_timeout += (math.floor((current_time - self._next_timeout) /
+                                              callback_time_sec) + 1) * callback_time_sec
+        else:
+            # If the clock moved backwards, ensure we advance the next
+            # timeout instead of recomputing the same value again.
+            # This may result in long gaps between callbacks if the
+            # clock jumps backwards by a lot, but the far more common
+            # scenario is a small NTP adjustment that should just be
+            # ignored.
+            #
+            # Note that on some systems if time.time() runs slower
+            # than time.monotonic() (most common on windows), we
+            # effectively experience a small backwards time jump on
+            # every iteration because PeriodicCallback uses
+            # time.time() while asyncio schedules callbacks using
+            # time.monotonic().
+            # https://github.com/tornadoweb/tornado/issues/2333
+            self._next_timeout += callback_time_sec

lib/tornado/platform/caresresolver.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function
 import pycares  # type: ignore
 import socket
 
+from tornado.concurrent import Future
 from tornado import gen
 from tornado.ioloop import IOLoop
 from tornado.netutil import Resolver, is_valid_ip
@@ -55,11 +56,10 @@ class CaresResolver(Resolver):
             addresses = [host]
         else:
             # gethostbyname doesn't take callback as a kwarg
-            self.channel.gethostbyname(host, family, (yield gen.Callback(1)))
+            fut = Future()
-            callback_args = yield gen.Wait(1)
+            self.channel.gethostbyname(host, family,
-            assert isinstance(callback_args, gen.Arguments)
+                                       lambda result, error: fut.set_result((result, error)))
-            assert not callback_args.kwargs
+            result, error = yield fut
-            result, error = callback_args.args
             if error:
                 raise IOError('C-Ares returned error %s: %s while resolving %s' %
                               (error, pycares.errno.strerror(error), host))

lib/tornado/simple_httpclient.py

@@ -35,6 +35,39 @@ except ImportError:
     ssl = None
 
 
+class HTTPTimeoutError(HTTPError):
+    """Error raised by SimpleAsyncHTTPClient on timeout.
+
+    For historical reasons, this is a subclass of `.HTTPClientError`
+    which simulates a response code of 599.
+
+    .. versionadded:: 5.1
+    """
+    def __init__(self, message):
+        super(HTTPTimeoutError, self).__init__(599, message=message)
+
+    def __str__(self):
+        return self.message
+
+
+class HTTPStreamClosedError(HTTPError):
+    """Error raised by SimpleAsyncHTTPClient when the underlying stream is closed.
+
+    When a more specific exception is available (such as `ConnectionResetError`),
+    it may be raised instead of this one.
+
+    For historical reasons, this is a subclass of `.HTTPClientError`
+    which simulates a response code of 599.
+
+    .. versionadded:: 5.1
+    """
+    def __init__(self, message):
+        super(HTTPStreamClosedError, self).__init__(599, message=message)
+
+    def __str__(self):
+        return self.message
+
+
 class SimpleAsyncHTTPClient(AsyncHTTPClient):
     """Non-blocking HTTP client with no external dependencies.
 
@@ -168,7 +201,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
 
         error_message = "Timeout {0}".format(info) if info else "Timeout"
         timeout_response = HTTPResponse(
-            request, 599, error=HTTPError(599, error_message),
+            request, 599, error=HTTPTimeoutError(error_message),
             request_time=self.io_loop.time() - request.start_time)
         self.io_loop.add_callback(callback, timeout_response)
         del self.waiting[key]
@@ -261,14 +294,14 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
     def _on_timeout(self, info=None):
         """Timeout callback of _HTTPConnection instance.
 
-        Raise a timeout HTTPError when a timeout occurs.
+        Raise a `HTTPTimeoutError` when a timeout occurs.
 
         :info string key: More detailed timeout information.
         """
         self._timeout = None
         error_message = "Timeout {0}".format(info) if info else "Timeout"
         if self.final_callback is not None:
-            raise HTTPError(599, error_message)
+            raise HTTPTimeoutError(error_message)
 
     def _remove_timeout(self):
         if self._timeout is not None:
@@ -413,7 +446,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             self._remove_timeout()
             if isinstance(value, StreamClosedError):
                 if value.real_error is None:
-                    value = HTTPError(599, "Stream closed")
+                    value = HTTPStreamClosedError("Stream closed")
                 else:
                     value = value.real_error
             self._run_callback(HTTPResponse(self.request, 599, error=value,
@@ -439,8 +472,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             if self.stream.error:
                 raise self.stream.error
             try:
-                raise HTTPError(599, message)
+                raise HTTPStreamClosedError(message)
-            except HTTPError:
+            except HTTPStreamClosedError:
                 self._handle_exception(*sys.exc_info())
 
     def headers_received(self, first_line, headers):
@@ -498,7 +531,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             final_callback = self.final_callback
             self.final_callback = None
             self._release()
-            self.client.fetch(new_request, final_callback)
+            fut = self.client.fetch(new_request, raise_error=False)
+            fut.add_done_callback(lambda f: final_callback(f.result()))
             self._on_end_request()
             return
         if self.request.streaming_callback:

lib/tornado/testing.py

@@ -80,6 +80,7 @@ else:
     import tornado.platform.asyncio
     _NON_OWNED_IOLOOPS = tornado.platform.asyncio.AsyncIOMainLoop
 
+
 def bind_unused_port(reuse_port=False):
     """Binds a server socket to an available port on localhost.
 
@@ -386,7 +387,7 @@ class AsyncHTTPTestCase(AsyncTestCase):
         """
         raise NotImplementedError()
 
-    def fetch(self, path, **kwargs):
+    def fetch(self, path, raise_error=False, **kwargs):
         """Convenience method to synchronously fetch a URL.
 
         The given path will be appended to the local server's host and
@@ -397,14 +398,36 @@ class AsyncHTTPTestCase(AsyncTestCase):
         If the path begins with http:// or https://, it will be treated as a
         full URL and will be fetched as-is.
 
+        If ``raise_error`` is True, a `tornado.httpclient.HTTPError` will
+        be raised if the response code is not 200. This is the same behavior
+        as the ``raise_error`` argument to `.AsyncHTTPClient.fetch`, but
+        the default is False here (it's True in `.AsyncHTTPClient`) because
+        tests often need to deal with non-200 response codes.
+
         .. versionchanged:: 5.0
            Added support for absolute URLs.
+
+        .. versionchanged:: 5.1
+
+           Added the ``raise_error`` argument.
+
+        .. deprecated:: 5.1
+
+           This method currently turns any exception into an
+           `.HTTPResponse` with status code 599. In Tornado 6.0,
+           errors other than `tornado.httpclient.HTTPError` will be
+           passed through, and ``raise_error=False`` will only
+           suppress errors that would be raised due to non-200
+           response codes.
+
         """
         if path.lower().startswith(('http://', 'https://')):
-            self.http_client.fetch(path, self.stop, **kwargs)
+            url = path
         else:
-            self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
+            url = self.get_url(path)
-        return self.wait()
+        return self.io_loop.run_sync(
+            lambda: self.http_client.fetch(url, raise_error=raise_error, **kwargs),
+            timeout=get_async_test_timeout())
 
     def get_httpserver_options(self):
         """May be overridden by subclasses to return additional

sickbeard/webserveInit.py

@@ -70,7 +70,7 @@ class WebServer(threading.Thread):
 
         # Load the app
         self.app = Application([],
-                               debug=False,
+                               debug=True,
                                serve_traceback=True,
                                autoreload=False,
                                compress_response=True,