diff --git a/CHANGES.md b/CHANGES.md index 2fd52d10..7c475834 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,9 @@ -### 0.15.3 (2018-03-28 16:55:00 UTC) +### 0.15.4 (2018-04-03 16:10:00 UTC) + +* Fix Torrentleech provider + + +### 0.15.3 (2018-03-28 16:55:00 UTC) * Fix clicking next and previous show buttons on macOS Safari diff --git a/lib/cfscrape.py b/lib/cfscrape.py index cd737fc7..b15f9783 100644 --- a/lib/cfscrape.py +++ b/lib/cfscrape.py @@ -9,36 +9,41 @@ from copy import deepcopy try: from urlparse import urlparse except ImportError: + # noinspection PyCompatibility from urllib.parse import urlparse DEFAULT_USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0" + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko)' + ' Chrome/41.0.2228.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko)' + ' Chrome/50.0.2661.102 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)' + ' Chrome/52.0.2743.116 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0)' + ' Gecko/20100101 Firefox/46.0', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0)' + ' Gecko/20100101 Firefox/41.0' ] DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS) class CloudflareScraper(Session): - def __init__(self, *args, **kwargs): - super(CloudflareScraper, self).__init__(*args, **kwargs) + def __init__(self): + super(CloudflareScraper, self).__init__() - if "requests" in self.headers["User-Agent"]: + if 'requests' in self.headers['User-Agent']: # Spoof Firefox on Linux if no custom User-Agent has been set - self.headers["User-Agent"] = DEFAULT_USER_AGENT + self.headers['User-Agent'] = DEFAULT_USER_AGENT def request(self, method, url, *args, **kwargs): resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) # Check if Cloudflare anti-bot is on - if ( resp.status_code == 503 - and resp.headers.get("Server") == "cloudflare-nginx" - and b"jschl_vc" in resp.content - and b"jschl_answer" in resp.content - ): + if (503 == resp.status_code + and re.search('(?i)cloudflare', resp.headers.get('Server')) + and b'jschl_vc' in resp.content + and b'jschl_answer' in resp.content): return self.solve_cf_challenge(resp, **kwargs) # Otherwise, no Cloudflare anti-bot detected @@ -50,16 +55,16 @@ class CloudflareScraper(Session): body = resp.text parsed_url = urlparse(resp.url) domain = urlparse(resp.url).netloc - submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain) + submit_url = '%s://%s/cdn-cgi/l/chk_jschl' % (parsed_url.scheme, domain) - cloudflare_kwargs = deepcopy(original_kwargs) - params = cloudflare_kwargs.setdefault("params", {}) - headers = cloudflare_kwargs.setdefault("headers", {}) - headers["Referer"] = resp.url + cloudflare_kwargs = {k: v for k, v in original_kwargs.items() if k not in ['hooks']} + params = cloudflare_kwargs.setdefault('params', {}) + headers = cloudflare_kwargs.setdefault('headers', {}) + headers['Referer'] = resp.url try: - params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1) - params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1) + params['jschl_vc'] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1) + params['pass'] = re.search(r'name="pass" value="(.+?)"', body).group(1) # Extract the arithmetic operation js = self.extract_js(body) @@ -69,45 +74,46 @@ class CloudflareScraper(Session): # This may indicate Cloudflare has changed their anti-bot # technique. If you see this and are running the latest version, # please open a GitHub issue so I can update the code accordingly. - logging.error("[!] Unable to parse Cloudflare anti-bots page. " - "Try upgrading cloudflare-scrape, or submit a bug report " - "if you are running the latest version. Please read " - "https://github.com/Anorov/cloudflare-scrape#updates " - "before submitting a bug report.") + logging.error('[!] Unable to parse Cloudflare anti-bots page. ' + 'Try upgrading cloudflare-scrape, or submit a bug report ' + 'if you are running the latest version. Please read ' + 'https://github.com/Anorov/cloudflare-scrape#updates ' + 'before submitting a bug report.') raise # Safely evaluate the Javascript expression - params["jschl_answer"] = str(int(js2py.eval_js(js)) + len(domain)) + params['jschl_answer'] = str(int(js2py.eval_js(js)) + len(domain)) # Requests transforms any request into a GET after a redirect, # so the redirect has to be handled manually here to allow for # performing other types of requests even as the first request. method = resp.request.method - cloudflare_kwargs["allow_redirects"] = False + cloudflare_kwargs['allow_redirects'] = False redirect = self.request(method, submit_url, **cloudflare_kwargs) - return self.request(method, redirect.headers["Location"], **original_kwargs) + return self.request(method, redirect.headers['Location'], **original_kwargs) - def extract_js(self, body): - js = re.search(r"setTimeout\(function\(\){\s+(var " - "s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n", body).group(1) - js = re.sub(r"a\.value = (parseInt\(.+?\)).+", r"\1", js) - js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js) + @staticmethod + def extract_js(body): + js = re.search(r'setTimeout\(function\(\){\s+(var ' + 's,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n', body).group(1) + js = re.sub(r'a\.value = (parseInt\(.+?\)).+', r'\1', js) + js = re.sub(r'\s{3,}[a-z](?: = |\.).+', '', js) # Strip characters that could be used to exit the string context # These characters are not currently used in Cloudflare's arithmetic snippet - js = re.sub(r"[\n\\']", "", js) + js = re.sub(r'[\n\\"]', '', js) return js @classmethod - def create_scraper(cls, sess=None, **kwargs): + def create_scraper(cls, sess=None): """ Convenience function for creating a ready-to-go requests.Session (subclass) object. """ scraper = cls() if sess: - attrs = ["auth", "cert", "cookies", "headers", "hooks", "params", "proxies", "data"] + attrs = ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data'] for attr in attrs: val = getattr(sess, attr, None) if val: @@ -115,46 +121,46 @@ class CloudflareScraper(Session): return scraper - - ## Functions for integrating cloudflare-scrape with other applications and scripts + # Functions for integrating cloudflare-scrape with other applications and scripts @classmethod - def get_tokens(cls, url, user_agent=None, **kwargs): + def get_tokens(cls, url, user_agent=None): scraper = cls.create_scraper() if user_agent: - scraper.headers["User-Agent"] = user_agent + scraper.headers['User-Agent'] = user_agent + # noinspection PyUnusedLocal try: resp = scraper.get(url) resp.raise_for_status() except Exception as e: - logging.error("'%s' returned an error. Could not collect tokens." % url) + logging.error('[%s] returned an error. Could not collect tokens.' % url) raise domain = urlparse(resp.url).netloc - cookie_domain = None + # cookie_domain = None for d in scraper.cookies.list_domains(): - if d.startswith(".") and d in ("." + domain): + if d.startswith('.') and d in ('.' + domain): cookie_domain = d break else: - raise ValueError("Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM (\"I'm Under Attack Mode\") enabled?") + raise ValueError('Unable to find Cloudflare cookies.' + ' Does the site actually have Cloudflare IUAM (\'I\'m Under Attack Mode\') enabled?') - return ({ - "__cfduid": scraper.cookies.get("__cfduid", "", domain=cookie_domain), - "cf_clearance": scraper.cookies.get("cf_clearance", "", domain=cookie_domain) - }, - scraper.headers["User-Agent"] - ) + return ({'__cfduid': scraper.cookies.get('__cfduid', '', domain=cookie_domain), + 'cf_clearance': scraper.cookies.get('cf_clearance', '', domain=cookie_domain) + }, + scraper.headers['User-Agent']) @classmethod - def get_cookie_string(cls, url, user_agent=None, **kwargs): + def get_cookie_string(cls, url, user_agent=None): """ Convenience function for building a Cookie HTTP header value. """ tokens, user_agent = cls.get_tokens(url, user_agent=user_agent) - return "; ".join("=".join(pair) for pair in tokens.items()), user_agent + return '; '.join('='.join(pair) for pair in tokens.items()), user_agent + create_scraper = CloudflareScraper.create_scraper get_tokens = CloudflareScraper.get_tokens diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py index 2c1adb65..4dd401b7 100644 --- a/sickbeard/providers/torrentleech.py +++ b/sickbeard/providers/torrentleech.py @@ -29,7 +29,7 @@ class TorrentLeechProvider(generic.TorrentProvider): def __init__(self): generic.TorrentProvider.__init__(self, 'TorrentLeech', cache_update_freq=20) - self.url_base = 'https://torrentleech.org/' + self.url_base = 'https://v4.torrentleech.org/' self.urls = {'config_provider_home_uri': self.url_base, 'login_action': self.url_base, 'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s',