Fix issue fetching data in a rare case.

This commit is contained in:
JackDandy 2018-01-15 17:35:27 +00:00
parent b57660a681
commit cf40431bfb
2 changed files with 8 additions and 2 deletions

View file

@ -1,4 +1,9 @@
### 0.13.10 (2018-01-08 17:20:00 UTC)
### 0.13.11 (2018-01-15 17:35:00 UTC)
* Fix issue fetching data in a rare case
### 0.13.10 (2018-01-08 17:20:00 UTC)
* Fix "Upgrade once" for wanted qualities

View file

@ -1119,6 +1119,7 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
# reuse or instantiate request session
if None is session:
session = CloudflareScraper.create_scraper()
session.headers.update({'User-Agent': USER_AGENT})
# download and save file or simply fetch url
savename = None
@ -1135,7 +1136,7 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
# session master headers
req_headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip,deflate', 'User-Agent': USER_AGENT}
'Accept-Encoding': 'gzip,deflate'}
if headers:
req_headers.update(headers)
if hasattr(session, 'reserved') and 'headers' in session.reserved: