diff --git a/CHANGES.md b/CHANGES.md index a524b257..34972c1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,12 +25,13 @@ * Update scandir 1.3 to 1.6 (c3592ee) * Update SimpleJSON library 3.10.0 (c52efea) to 3.13.2 (6ffddbe) * Update Six compatibility library 1.10.0 (r433) to 1.11.0 (68112f3) -* Update Tornado Web Server 4.5.1 (79b2683) to 5.0.1 (35a538f) +* Update Tornado Web Server 5.0.1 (35a538f) to 5.1.dev1 (415f453) * Update unidecode library 0.04.21 (e99b0e3) to 1.0.22 (81f938d) * Update webencodings 0.5 (3970651) to 0.5.1 (fa2cb5d) * Update xmltodict library 0.10.2 (375d3a6) to 0.11.0 (79ac9a4) [develop changelog] +* Update Tornado Web Server 4.5.1 (79b2683) to 5.0.1 (35a538f) * Change pick up the stragglers late to the more security party * Change remove redundant xsrf handling for POSTs that don't use web and API * Change add xsrf protection support to media processing scripts diff --git a/lib/tornado/auth.py b/lib/tornado/auth.py index beee6453..0069efcb 100644 --- a/lib/tornado/auth.py +++ b/lib/tornado/auth.py @@ -202,9 +202,9 @@ class OpenIdMixin(object): url = self._OPENID_ENDPOINT if http_client is None: http_client = self.get_auth_http_client() - http_client.fetch(url, functools.partial( - self._on_authentication_verified, callback), - method="POST", body=urllib_parse.urlencode(args)) + fut = http_client.fetch(url, method="POST", body=urllib_parse.urlencode(args)) + fut.add_done_callback(functools.partial( + self._on_authentication_verified, callback)) def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): url = urlparse.urljoin(self.request.full_url(), callback_uri) @@ -254,11 +254,16 @@ class OpenIdMixin(object): }) return args - def _on_authentication_verified(self, future, response): - if response.error or b"is_valid:true" not in response.body: + def _on_authentication_verified(self, future, response_fut): + try: + response = response_fut.result() + except Exception as e: future.set_exception(AuthError( - "Invalid OpenID response: %s" % (response.error or - response.body))) + "Error response %s" % e)) + return + if b"is_valid:true" not in response.body: + future.set_exception(AuthError( + "Invalid OpenID response: %s" % response.body)) return # Make sure we got back at least an email from attribute exchange @@ -374,17 +379,17 @@ class OAuthMixin(object): if http_client is None: http_client = self.get_auth_http_client() if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - http_client.fetch( + fut = http_client.fetch( self._oauth_request_token_url(callback_uri=callback_uri, - extra_params=extra_params), - functools.partial( - self._on_request_token, - self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback)) + extra_params=extra_params)) + fut.add_done_callback(functools.partial( + self._on_request_token, + self._OAUTH_AUTHORIZE_URL, + callback_uri, + callback)) else: - http_client.fetch( - self._oauth_request_token_url(), + fut = http_client.fetch(self._oauth_request_token_url()) + fut.add_done_callback( functools.partial( self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri, @@ -427,8 +432,8 @@ class OAuthMixin(object): token["verifier"] = oauth_verifier if http_client is None: http_client = self.get_auth_http_client() - http_client.fetch(self._oauth_access_token_url(token), - functools.partial(self._on_access_token, callback)) + fut = http_client.fetch(self._oauth_access_token_url(token)) + fut.add_done_callback(functools.partial(self._on_access_token, callback)) def _oauth_request_token_url(self, callback_uri=None, extra_params=None): consumer_token = self._oauth_consumer_token() @@ -456,9 +461,11 @@ class OAuthMixin(object): return url + "?" + urllib_parse.urlencode(args) def _on_request_token(self, authorize_url, callback_uri, callback, - response): - if response.error: - raise Exception("Could not get request token: %s" % response.error) + response_fut): + try: + response = response_fut.result() + except Exception as e: + raise Exception("Could not get request token: %s" % e) request_token = _oauth_parse_response(response.body) data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + base64.b64encode(escape.utf8(request_token["secret"]))) @@ -498,8 +505,10 @@ class OAuthMixin(object): args["oauth_signature"] = signature return url + "?" + urllib_parse.urlencode(args) - def _on_access_token(self, future, response): - if response.error: + def _on_access_token(self, future, response_fut): + try: + response = response_fut.result() + except Exception: future.set_exception(AuthError("Could not fetch access token")) return @@ -707,15 +716,16 @@ class OAuth2Mixin(object): callback = functools.partial(self._on_oauth2_request, callback) http = self.get_auth_http_client() if post_args is not None: - http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), - callback=callback) + fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) else: - http.fetch(url, callback=callback) + fut = http.fetch(url) + fut.add_done_callback(callback) - def _on_oauth2_request(self, future, response): - if response.error: - future.set_exception(AuthError("Error response %s fetching %s" % - (response.error, response.request.url))) + def _on_oauth2_request(self, future, response_fut): + try: + response = response_fut.result() + except Exception as e: + future.set_exception(AuthError("Error response %s" % e)) return future_set_result_unless_cancelled(future, escape.json_decode(response.body)) @@ -857,18 +867,19 @@ class TwitterMixin(OAuthMixin): if args: url += "?" + urllib_parse.urlencode(args) http = self.get_auth_http_client() - http_callback = functools.partial(self._on_twitter_request, callback) + http_callback = functools.partial(self._on_twitter_request, callback, url) if post_args is not None: - http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), - callback=http_callback) + fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) else: - http.fetch(url, callback=http_callback) + fut = http.fetch(url) + fut.add_done_callback(http_callback) - def _on_twitter_request(self, future, response): - if response.error: + def _on_twitter_request(self, future, url, response_fut): + try: + response = response_fut.result() + except Exception as e: future.set_exception(AuthError( - "Error response %s fetching %s" % (response.error, - response.request.url))) + "Error response %s fetching %s" % (e, url))) return future_set_result_unless_cancelled(future, escape.json_decode(response.body)) @@ -967,16 +978,18 @@ class GoogleOAuth2Mixin(OAuth2Mixin): "grant_type": "authorization_code", }) - http.fetch(self._OAUTH_ACCESS_TOKEN_URL, - functools.partial(self._on_access_token, callback), - method="POST", - headers={'Content-Type': 'application/x-www-form-urlencoded'}, - body=body) + fut = http.fetch(self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={'Content-Type': 'application/x-www-form-urlencoded'}, + body=body) + fut.add_done_callback(functools.partial(self._on_access_token, callback)) - def _on_access_token(self, future, response): + def _on_access_token(self, future, response_fut): """Callback function for the exchange to the access token.""" - if response.error: - future.set_exception(AuthError('Google auth error: %s' % str(response))) + try: + response = response_fut.result() + except Exception as e: + future.set_exception(AuthError('Google auth error: %s' % str(e))) return args = escape.json_decode(response.body) @@ -1053,15 +1066,17 @@ class FacebookGraphMixin(OAuth2Mixin): if extra_fields: fields.update(extra_fields) - http.fetch(self._oauth_request_token_url(**args), - functools.partial(self._on_access_token, redirect_uri, client_id, - client_secret, callback, fields)) + fut = http.fetch(self._oauth_request_token_url(**args)) + fut.add_done_callback(functools.partial(self._on_access_token, redirect_uri, client_id, + client_secret, callback, fields)) @gen.coroutine def _on_access_token(self, redirect_uri, client_id, client_secret, - future, fields, response): - if response.error: - future.set_exception(AuthError('Facebook auth error: %s' % str(response))) + future, fields, response_fut): + try: + response = response_fut.result() + except Exception as e: + future.set_exception(AuthError('Facebook auth error: %s' % str(e))) return args = escape.json_decode(response.body) diff --git a/lib/tornado/httpclient.py b/lib/tornado/httpclient.py index 9c438d15..f0a2df88 100644 --- a/lib/tornado/httpclient.py +++ b/lib/tornado/httpclient.py @@ -42,6 +42,7 @@ from __future__ import absolute_import, division, print_function import functools import time +import warnings import weakref from tornado.concurrent import Future, future_set_result_unless_cancelled @@ -238,6 +239,18 @@ class AsyncHTTPClient(Configurable): In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. + + .. deprecated:: 5.1 + + The ``callback`` argument is deprecated and will be removed + in 6.0. Use the returned `.Future` instead. + + The ``raise_error=False`` argument currently suppresses + *all* errors, encapsulating them in `HTTPResponse` objects + with a 599 response code. This will change in Tornado 6.0: + ``raise_error=False`` will only affect the `HTTPError` + raised when a non-200 response code is used. + """ if self._closed: raise RuntimeError("fetch() called on closed AsyncHTTPClient") @@ -253,6 +266,8 @@ class AsyncHTTPClient(Configurable): request = _RequestProxy(request, self.defaults) future = Future() if callback is not None: + warnings.warn("callback arguments are deprecated, use the returned Future instead", + DeprecationWarning) callback = stack_context.wrap(callback) def handle_future(future): @@ -270,8 +285,13 @@ class AsyncHTTPClient(Configurable): def handle_response(response): if raise_error and response.error: + if isinstance(response.error, HTTPError): + response.error.response = response future.set_exception(response.error) else: + if response.error and not response._error_is_response_code: + warnings.warn("raise_error=False will allow '%s' to be raised in the future" % + response.error, DeprecationWarning) future_set_result_unless_cancelled(future, response) self.fetch_impl(request, handle_response) return future @@ -585,8 +605,10 @@ class HTTPResponse(object): self.effective_url = request.url else: self.effective_url = effective_url + self._error_is_response_code = False if error is None: if self.code < 200 or self.code >= 300: + self._error_is_response_code = True self.error = HTTPError(self.code, message=self.reason, response=self) else: @@ -615,7 +637,7 @@ class HTTPResponse(object): return "%s(%s)" % (self.__class__.__name__, args) -class HTTPError(Exception): +class HTTPClientError(Exception): """Exception thrown for an unsuccessful HTTP request. Attributes: @@ -628,12 +650,18 @@ class HTTPError(Exception): Note that if ``follow_redirects`` is False, redirects become HTTPErrors, and you can look at ``error.response.headers['Location']`` to see the destination of the redirect. + + .. versionchanged:: 5.1 + + Renamed from ``HTTPError`` to ``HTTPClientError`` to avoid collisions with + `tornado.web.HTTPError`. The name ``tornado.httpclient.HTTPError`` remains + as an alias. """ def __init__(self, code, message=None, response=None): self.code = code self.message = message or httputil.responses.get(code, "Unknown") self.response = response - super(HTTPError, self).__init__(code, message, response) + super(HTTPClientError, self).__init__(code, message, response) def __str__(self): return "HTTP %d: %s" % (self.code, self.message) @@ -645,6 +673,9 @@ class HTTPError(Exception): __repr__ = __str__ +HTTPError = HTTPClientError + + class _RequestProxy(object): """Combines an object with a dictionary of defaults. diff --git a/lib/tornado/ioloop.py b/lib/tornado/ioloop.py index f6ec177b..48700139 100644 --- a/lib/tornado/ioloop.py +++ b/lib/tornado/ioloop.py @@ -1213,11 +1213,31 @@ class PeriodicCallback(object): def _schedule_next(self): if self._running: - current_time = self.io_loop.time() - - if self._next_timeout <= current_time: - callback_time_sec = self.callback_time / 1000.0 - self._next_timeout += (math.floor((current_time - self._next_timeout) / - callback_time_sec) + 1) * callback_time_sec - + self._update_next(self.io_loop.time()) self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) + + def _update_next(self, current_time): + callback_time_sec = self.callback_time / 1000.0 + if self._next_timeout <= current_time: + # The period should be measured from the start of one call + # to the start of the next. If one call takes too long, + # skip cycles to get back to a multiple of the original + # schedule. + self._next_timeout += (math.floor((current_time - self._next_timeout) / + callback_time_sec) + 1) * callback_time_sec + else: + # If the clock moved backwards, ensure we advance the next + # timeout instead of recomputing the same value again. + # This may result in long gaps between callbacks if the + # clock jumps backwards by a lot, but the far more common + # scenario is a small NTP adjustment that should just be + # ignored. + # + # Note that on some systems if time.time() runs slower + # than time.monotonic() (most common on windows), we + # effectively experience a small backwards time jump on + # every iteration because PeriodicCallback uses + # time.time() while asyncio schedules callbacks using + # time.monotonic(). + # https://github.com/tornadoweb/tornado/issues/2333 + self._next_timeout += callback_time_sec diff --git a/lib/tornado/platform/caresresolver.py b/lib/tornado/platform/caresresolver.py index 38247f7e..768cb624 100644 --- a/lib/tornado/platform/caresresolver.py +++ b/lib/tornado/platform/caresresolver.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function import pycares # type: ignore import socket +from tornado.concurrent import Future from tornado import gen from tornado.ioloop import IOLoop from tornado.netutil import Resolver, is_valid_ip @@ -55,11 +56,10 @@ class CaresResolver(Resolver): addresses = [host] else: # gethostbyname doesn't take callback as a kwarg - self.channel.gethostbyname(host, family, (yield gen.Callback(1))) - callback_args = yield gen.Wait(1) - assert isinstance(callback_args, gen.Arguments) - assert not callback_args.kwargs - result, error = callback_args.args + fut = Future() + self.channel.gethostbyname(host, family, + lambda result, error: fut.set_result((result, error))) + result, error = yield fut if error: raise IOError('C-Ares returned error %s: %s while resolving %s' % (error, pycares.errno.strerror(error), host)) diff --git a/lib/tornado/simple_httpclient.py b/lib/tornado/simple_httpclient.py index 1fc3e707..7696dd18 100644 --- a/lib/tornado/simple_httpclient.py +++ b/lib/tornado/simple_httpclient.py @@ -35,6 +35,39 @@ except ImportError: ssl = None +class HTTPTimeoutError(HTTPError): + """Error raised by SimpleAsyncHTTPClient on timeout. + + For historical reasons, this is a subclass of `.HTTPClientError` + which simulates a response code of 599. + + .. versionadded:: 5.1 + """ + def __init__(self, message): + super(HTTPTimeoutError, self).__init__(599, message=message) + + def __str__(self): + return self.message + + +class HTTPStreamClosedError(HTTPError): + """Error raised by SimpleAsyncHTTPClient when the underlying stream is closed. + + When a more specific exception is available (such as `ConnectionResetError`), + it may be raised instead of this one. + + For historical reasons, this is a subclass of `.HTTPClientError` + which simulates a response code of 599. + + .. versionadded:: 5.1 + """ + def __init__(self, message): + super(HTTPStreamClosedError, self).__init__(599, message=message) + + def __str__(self): + return self.message + + class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. @@ -168,7 +201,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): error_message = "Timeout {0}".format(info) if info else "Timeout" timeout_response = HTTPResponse( - request, 599, error=HTTPError(599, error_message), + request, 599, error=HTTPTimeoutError(error_message), request_time=self.io_loop.time() - request.start_time) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key] @@ -261,14 +294,14 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): def _on_timeout(self, info=None): """Timeout callback of _HTTPConnection instance. - Raise a timeout HTTPError when a timeout occurs. + Raise a `HTTPTimeoutError` when a timeout occurs. :info string key: More detailed timeout information. """ self._timeout = None error_message = "Timeout {0}".format(info) if info else "Timeout" if self.final_callback is not None: - raise HTTPError(599, error_message) + raise HTTPTimeoutError(error_message) def _remove_timeout(self): if self._timeout is not None: @@ -413,7 +446,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): self._remove_timeout() if isinstance(value, StreamClosedError): if value.real_error is None: - value = HTTPError(599, "Stream closed") + value = HTTPStreamClosedError("Stream closed") else: value = value.real_error self._run_callback(HTTPResponse(self.request, 599, error=value, @@ -439,8 +472,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): if self.stream.error: raise self.stream.error try: - raise HTTPError(599, message) - except HTTPError: + raise HTTPStreamClosedError(message) + except HTTPStreamClosedError: self._handle_exception(*sys.exc_info()) def headers_received(self, first_line, headers): @@ -498,7 +531,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): final_callback = self.final_callback self.final_callback = None self._release() - self.client.fetch(new_request, final_callback) + fut = self.client.fetch(new_request, raise_error=False) + fut.add_done_callback(lambda f: final_callback(f.result())) self._on_end_request() return if self.request.streaming_callback: diff --git a/lib/tornado/testing.py b/lib/tornado/testing.py index 19bc5a94..04ea3816 100644 --- a/lib/tornado/testing.py +++ b/lib/tornado/testing.py @@ -80,6 +80,7 @@ else: import tornado.platform.asyncio _NON_OWNED_IOLOOPS = tornado.platform.asyncio.AsyncIOMainLoop + def bind_unused_port(reuse_port=False): """Binds a server socket to an available port on localhost. @@ -386,7 +387,7 @@ class AsyncHTTPTestCase(AsyncTestCase): """ raise NotImplementedError() - def fetch(self, path, **kwargs): + def fetch(self, path, raise_error=False, **kwargs): """Convenience method to synchronously fetch a URL. The given path will be appended to the local server's host and @@ -397,14 +398,36 @@ class AsyncHTTPTestCase(AsyncTestCase): If the path begins with http:// or https://, it will be treated as a full URL and will be fetched as-is. + If ``raise_error`` is True, a `tornado.httpclient.HTTPError` will + be raised if the response code is not 200. This is the same behavior + as the ``raise_error`` argument to `.AsyncHTTPClient.fetch`, but + the default is False here (it's True in `.AsyncHTTPClient`) because + tests often need to deal with non-200 response codes. + .. versionchanged:: 5.0 Added support for absolute URLs. + + .. versionchanged:: 5.1 + + Added the ``raise_error`` argument. + + .. deprecated:: 5.1 + + This method currently turns any exception into an + `.HTTPResponse` with status code 599. In Tornado 6.0, + errors other than `tornado.httpclient.HTTPError` will be + passed through, and ``raise_error=False`` will only + suppress errors that would be raised due to non-200 + response codes. + """ if path.lower().startswith(('http://', 'https://')): - self.http_client.fetch(path, self.stop, **kwargs) + url = path else: - self.http_client.fetch(self.get_url(path), self.stop, **kwargs) - return self.wait() + url = self.get_url(path) + return self.io_loop.run_sync( + lambda: self.http_client.fetch(url, raise_error=raise_error, **kwargs), + timeout=get_async_test_timeout()) def get_httpserver_options(self): """May be overridden by subclasses to return additional diff --git a/sickbeard/webserveInit.py b/sickbeard/webserveInit.py index aebe4f1a..ff44fa64 100644 --- a/sickbeard/webserveInit.py +++ b/sickbeard/webserveInit.py @@ -70,7 +70,7 @@ class WebServer(threading.Thread): # Load the app self.app = Application([], - debug=False, + debug=True, serve_traceback=True, autoreload=False, compress_response=True,