diff --git a/CHANGES.md b/CHANGES.md index 03c03786..6df6d7b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * Update Tornado webserver to 4.2b1 (61a16c9) * Update change to suppress reporting of Tornado exception error 1 to updated package (ref:hacks.txt) * Update fix for API response header for JSON content type and the return of JSONP data to updated package (ref:hacks.txt) +* Update Requests library 2.6.2 to 2.7.0 (ab1f493) ### 0.9.0 (2015-05-18 14:33:00 UTC) diff --git a/lib/requests/LICENSE b/lib/requests/LICENSE new file mode 100644 index 00000000..a103fc91 --- /dev/null +++ b/lib/requests/LICENSE @@ -0,0 +1,13 @@ +Copyright 2015 Kenneth Reitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/requests/NOTICE b/lib/requests/NOTICE new file mode 100644 index 00000000..f583e47a --- /dev/null +++ b/lib/requests/NOTICE @@ -0,0 +1,54 @@ +Requests includes some vendorized python libraries to ease installation. + +Urllib3 License +=============== + +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt), +Modifications copyright 2012 Kenneth Reitz. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Chardet License +=============== + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +02110-1301 USA + + +CA Bundle License +================= + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/lib/requests/__init__.py b/lib/requests/__init__.py index 46116bea..d2471284 100644 --- a/lib/requests/__init__.py +++ b/lib/requests/__init__.py @@ -6,7 +6,7 @@ # / """ -requests HTTP library +Requests HTTP library ~~~~~~~~~~~~~~~~~~~~~ Requests is an HTTP library, written in Python, for human beings. Basic GET @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.6.2' -__build__ = 0x020602 +__version__ = '2.7.0' +__build__ = 0x020700 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' diff --git a/lib/requests/api.py b/lib/requests/api.py index 98c92298..d40fa380 100644 --- a/lib/requests/api.py +++ b/lib/requests/api.py @@ -55,17 +55,18 @@ def request(method, url, **kwargs): return response -def get(url, **kwargs): +def get(url, params=None, **kwargs): """Sends a GET request. :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) - return request('get', url, **kwargs) + return request('get', url, params=params, **kwargs) def options(url, **kwargs): diff --git a/lib/requests/auth.py b/lib/requests/auth.py index 0ff9c298..03c3302a 100644 --- a/lib/requests/auth.py +++ b/lib/requests/auth.py @@ -179,7 +179,7 @@ class HTTPDigestAuth(AuthBase): # Consume content and release the original connection # to allow our new request to reuse the same one. r.content - r.raw.release_conn() + r.close() prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) diff --git a/lib/requests/cookies.py b/lib/requests/cookies.py index 1fbc934c..88b478c7 100644 --- a/lib/requests/cookies.py +++ b/lib/requests/cookies.py @@ -415,11 +415,14 @@ def morsel_to_cookie(morsel): expires = None if morsel['max-age']: - expires = time.time() + morsel['max-age'] + try: + expires = int(time.time() + int(morsel['max-age'])) + except ValueError: + raise TypeError('max-age: %s must be integer' % morsel['max-age']) elif morsel['expires']: time_template = '%a, %d-%b-%Y %H:%M:%S GMT' - expires = time.mktime( - time.strptime(morsel['expires'], time_template)) - time.timezone + expires = int(time.mktime( + time.strptime(morsel['expires'], time_template)) - time.timezone) return create_cookie( comment=morsel['comment'], comment_url=bool(morsel['comment']), diff --git a/lib/requests/models.py b/lib/requests/models.py index 45b3ea96..ccaf5968 100644 --- a/lib/requests/models.py +++ b/lib/requests/models.py @@ -30,7 +30,8 @@ from .utils import ( iter_slices, guess_json_utf, super_len, to_native_string) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, - is_py2, chardet, json, builtin_str, basestring) + is_py2, chardet, builtin_str, basestring) +from .compat import json as complexjson from .status_codes import codes #: The set of HTTP status codes that indicate an automatically @@ -42,12 +43,11 @@ REDIRECT_STATI = ( codes.temporary_redirect, # 307 codes.permanent_redirect, # 308 ) + DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 -json_dumps = json.dumps - class RequestEncodingMixin(object): @property @@ -149,8 +149,7 @@ class RequestEncodingMixin(object): else: fdata = fp.read() - rf = RequestField(name=k, data=fdata, - filename=fn, headers=fh) + rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) new_fields.append(rf) @@ -207,17 +206,8 @@ class Request(RequestHooksMixin): """ - def __init__(self, - method=None, - url=None, - headers=None, - files=None, - data=None, - params=None, - auth=None, - cookies=None, - hooks=None, - json=None): + def __init__(self, method=None, url=None, headers=None, files=None, + data=None, params=None, auth=None, cookies=None, hooks=None, json=None): # Default empty dicts for dict params. data = [] if data is None else data @@ -296,8 +286,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.hooks = default_hooks() def prepare(self, method=None, url=None, headers=None, files=None, - data=None, params=None, auth=None, cookies=None, hooks=None, - json=None): + data=None, params=None, auth=None, cookies=None, hooks=None, json=None): """Prepares the entire request with the given parameters.""" self.prepare_method(method) @@ -306,6 +295,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.prepare_cookies(cookies) self.prepare_body(data, files, json) self.prepare_auth(auth, url) + # Note that prepare_auth must be last to enable authentication schemes # such as OAuth to work on a fully prepared request. @@ -357,9 +347,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): raise InvalidURL(*e.args) if not scheme: - raise MissingSchema("Invalid URL {0!r}: No schema supplied. " - "Perhaps you meant http://{0}?".format( - to_native_string(url, 'utf8'))) + error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + error = error.format(to_native_string(url, 'utf8')) + + raise MissingSchema(error) if not host: raise InvalidURL("Invalid URL %r: No host supplied" % url) @@ -425,7 +416,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if json is not None: content_type = 'application/json' - body = json_dumps(json) + body = complexjson.dumps(json) is_stream = all([ hasattr(data, '__iter__'), @@ -537,16 +528,8 @@ class Response(object): """ __attrs__ = [ - '_content', - 'status_code', - 'headers', - 'url', - 'history', - 'encoding', - 'reason', - 'cookies', - 'elapsed', - 'request', + '_content', 'status_code', 'headers', 'url', 'history', + 'encoding', 'reason', 'cookies', 'elapsed', 'request' ] def __init__(self): @@ -666,9 +649,10 @@ class Response(object): If decode_unicode is True, content will be decoded using the best available encoding based on the response. """ + def generate(): - try: - # Special case for urllib3. + # Special case for urllib3. + if hasattr(self.raw, 'stream'): try: for chunk in self.raw.stream(chunk_size, decode_content=True): yield chunk @@ -678,7 +662,7 @@ class Response(object): raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) - except AttributeError: + else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) @@ -809,14 +793,16 @@ class Response(object): encoding = guess_json_utf(self.content) if encoding is not None: try: - return json.loads(self.content.decode(encoding), **kwargs) + return complexjson.loads( + self.content.decode(encoding), **kwargs + ) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass - return json.loads(self.text, **kwargs) + return complexjson.loads(self.text, **kwargs) @property def links(self): diff --git a/lib/requests/packages/urllib3/__init__.py b/lib/requests/packages/urllib3/__init__.py index 333060c2..f48ac4af 100644 --- a/lib/requests/packages/urllib3/__init__.py +++ b/lib/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.10.3' +__version__ = '1.10.4' from .connectionpool import ( @@ -57,9 +57,10 @@ del NullHandler import warnings # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning) +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning) +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/lib/requests/packages/urllib3/connectionpool.py b/lib/requests/packages/urllib3/connectionpool.py index 43c67967..117269ac 100644 --- a/lib/requests/packages/urllib3/connectionpool.py +++ b/lib/requests/packages/urllib3/connectionpool.py @@ -760,14 +760,12 @@ class HTTPSConnectionPool(HTTPConnectionPool): if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` conn.connect() - """ if not conn.is_verified: warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' 'https://urllib3.readthedocs.org/en/latest/security.html'), InsecureRequestWarning) - """ def connection_from_url(url, **kw): diff --git a/lib/requests/packages/urllib3/response.py b/lib/requests/packages/urllib3/response.py index f1ea9bb5..24140c4c 100644 --- a/lib/requests/packages/urllib3/response.py +++ b/lib/requests/packages/urllib3/response.py @@ -126,14 +126,15 @@ class HTTPResponse(io.IOBase): # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '') - if tr_enc.lower() == "chunked": + tr_enc = self.headers.get('transfer-encoding', '').lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: self.chunked = True # We certainly don't want to preload content when the response is chunked. - if not self.chunked: - if preload_content and not self._body: - self._body = self.read(decode_content=decode_content) + if not self.chunked and preload_content and not self._body: + self._body = self.read(decode_content=decode_content) def get_redirect_location(self): """ @@ -179,9 +180,8 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ @@ -299,10 +299,9 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - self._init_decoder() if self.chunked: - for line in self.read_chunked(amt): - yield self._decode(line, decode_content, True) + for line in self.read_chunked(amt, decode_content=decode_content): + yield line else: while not is_fp_closed(self._fp): data = self.read(amt=amt, decode_content=decode_content) @@ -387,48 +386,70 @@ class HTTPResponse(io.IOBase): b[:len(temp)] = temp return len(temp) - def read_chunked(self, amt=None): - # FIXME: Rewrite this method and make it a class with - # a better structured logic. + def _update_chunk_length(self): + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is not None: + return + line = self._fp.fp.readline() + line = line.split(b';', 1)[0] + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(line) + + def _handle_chunk(self, amt): + returned_chunk = None + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + returned_chunk = chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + returned_chunk = value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + returned_chunk = value + else: # amt > self.chunk_left + returned_chunk = self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + return returned_chunk + + def read_chunked(self, amt=None, decode_content=None): + """ + Similar to :meth:`HTTPResponse.read`, but with an additional + parameter: ``decode_content``. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + self._init_decoder() + # FIXME: Rewrite this method and make it a class with a better structured logic. if not self.chunked: raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") + + if self._original_response and self._original_response._method.upper() == 'HEAD': + # Don't bother reading the body of a HEAD request. + # FIXME: Can we do this somehow without accessing private httplib _method? + self._original_response.close() + return + while True: - # First, we'll figure out length of a chunk and then - # we'll try to read it from socket. - if self.chunk_left is None: - line = self._fp.fp.readline() - line = line.decode() - # See RFC 7230: Chunked Transfer Coding. - i = line.find(';') - if i >= 0: - line = line[:i] # Strip chunk-extensions. - try: - self.chunk_left = int(line, 16) - except ValueError: - # Invalid chunked protocol response, abort. - self.close() - raise httplib.IncompleteRead(''.join(line)) - if self.chunk_left == 0: - break - if amt is None: - chunk = self._fp._safe_read(self.chunk_left) - yield chunk - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - elif amt < self.chunk_left: - value = self._fp._safe_read(amt) - self.chunk_left = self.chunk_left - amt - yield value - elif amt == self.chunk_left: - value = self._fp._safe_read(amt) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - yield value - else: # amt > self.chunk_left - yield self._fp._safe_read(self.chunk_left) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) + yield self._decode(chunk, decode_content=decode_content, + flush_decoder=True) # Chunk content ends with \r\n: discard it. while True: @@ -440,5 +461,6 @@ class HTTPResponse(io.IOBase): break # We read everything; close the "file". + if self._original_response: + self._original_response.close() self.release_conn() - diff --git a/lib/requests/packages/urllib3/util/ssl_.py b/lib/requests/packages/urllib3/util/ssl_.py index f9573af2..b846d42c 100644 --- a/lib/requests/packages/urllib3/util/ssl_.py +++ b/lib/requests/packages/urllib3/util/ssl_.py @@ -81,7 +81,6 @@ except ImportError: self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None): - """ warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' @@ -90,7 +89,6 @@ except ImportError: '#insecureplatformwarning.', InsecurePlatformWarning ) - """ kwargs = { 'keyfile': self.keyfile, 'certfile': self.certfile, diff --git a/lib/requests/packages/urllib3/util/url.py b/lib/requests/packages/urllib3/util/url.py index b2ec834f..e58050cd 100644 --- a/lib/requests/packages/urllib3/util/url.py +++ b/lib/requests/packages/urllib3/util/url.py @@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)): def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): + if path and not path.startswith('/'): + path = '/' + path return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment)