monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2025-01-07 10:33:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix Torrentleech provider.

Browse source
This commit is contained in:
JackDandy 2018-04-03 16:14:54 +01:00
parent 7a85df2d0e
commit a342acea5a
3 changed files with 66 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGES.md
View file

@ -1,4 +1,9 @@
### 0.15.3 (2018-03-28 16:55:00 UTC) ### 0.15.4 (2018-04-03 16:10:00 UTC)
* Fix Torrentleech provider
### 0.15.3 (2018-03-28 16:55:00 UTC)
* Fix clicking next and previous show buttons on macOS Safari * Fix clicking next and previous show buttons on macOS Safari

112
lib/cfscrape.py
View file

@ -9,36 +9,41 @@ from copy import deepcopy
try: try:
from urlparse import urlparse from urlparse import urlparse
except ImportError: except ImportError:
# noinspection PyCompatibility
from urllib.parse import urlparse from urllib.parse import urlparse
DEFAULT_USER_AGENTS = [ DEFAULT_USER_AGENTS = [
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko)'
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", ' Chrome/41.0.2228.0 Safari/537.36',
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36", 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko)'
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0", ' Chrome/50.0.2661.102 Safari/537.36',
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0" 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0)'
' Gecko/20100101 Firefox/46.0',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0)'
' Gecko/20100101 Firefox/41.0'
] ]
DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS) DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS)
class CloudflareScraper(Session): class CloudflareScraper(Session):
def __init__(self, *args, **kwargs): def __init__(self):
super(CloudflareScraper, self).__init__(*args, **kwargs) super(CloudflareScraper, self).__init__()
if "requests" in self.headers["User-Agent"]: if 'requests' in self.headers['User-Agent']:
# Spoof Firefox on Linux if no custom User-Agent has been set # Spoof Firefox on Linux if no custom User-Agent has been set
self.headers["User-Agent"] = DEFAULT_USER_AGENT self.headers['User-Agent'] = DEFAULT_USER_AGENT
def request(self, method, url, *args, **kwargs): def request(self, method, url, *args, **kwargs):
resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
# Check if Cloudflare anti-bot is on # Check if Cloudflare anti-bot is on
if ( resp.status_code == 503 if (503 == resp.status_code
and resp.headers.get("Server") == "cloudflare-nginx" and re.search('(?i)cloudflare', resp.headers.get('Server'))
and b"jschl_vc" in resp.content and b'jschl_vc' in resp.content
and b"jschl_answer" in resp.content and b'jschl_answer' in resp.content):
):
return self.solve_cf_challenge(resp, **kwargs) return self.solve_cf_challenge(resp, **kwargs)
# Otherwise, no Cloudflare anti-bot detected # Otherwise, no Cloudflare anti-bot detected
@ -50,16 +55,16 @@ class CloudflareScraper(Session):
body = resp.text body = resp.text
parsed_url = urlparse(resp.url) parsed_url = urlparse(resp.url)
domain = urlparse(resp.url).netloc domain = urlparse(resp.url).netloc
submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain) submit_url = '%s://%s/cdn-cgi/l/chk_jschl' % (parsed_url.scheme, domain)
cloudflare_kwargs = deepcopy(original_kwargs) cloudflare_kwargs = {k: v for k, v in original_kwargs.items() if k not in ['hooks']}
params = cloudflare_kwargs.setdefault("params", {}) params = cloudflare_kwargs.setdefault('params', {})
headers = cloudflare_kwargs.setdefault("headers", {}) headers = cloudflare_kwargs.setdefault('headers', {})
headers["Referer"] = resp.url headers['Referer'] = resp.url
try: try:
params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1) params['jschl_vc'] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1)
params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1) params['pass'] = re.search(r'name="pass" value="(.+?)"', body).group(1)
# Extract the arithmetic operation # Extract the arithmetic operation
js = self.extract_js(body) js = self.extract_js(body)
@ -69,45 +74,46 @@ class CloudflareScraper(Session):
# This may indicate Cloudflare has changed their anti-bot # This may indicate Cloudflare has changed their anti-bot
# technique. If you see this and are running the latest version, # technique. If you see this and are running the latest version,
# please open a GitHub issue so I can update the code accordingly. # please open a GitHub issue so I can update the code accordingly.
logging.error("[!] Unable to parse Cloudflare anti-bots page. " logging.error('[!] Unable to parse Cloudflare anti-bots page. '
"Try upgrading cloudflare-scrape, or submit a bug report " 'Try upgrading cloudflare-scrape, or submit a bug report '
"if you are running the latest version. Please read " 'if you are running the latest version. Please read '
"https://github.com/Anorov/cloudflare-scrape#updates " 'https://github.com/Anorov/cloudflare-scrape#updates '
"before submitting a bug report.") 'before submitting a bug report.')
raise raise
# Safely evaluate the Javascript expression # Safely evaluate the Javascript expression
params["jschl_answer"] = str(int(js2py.eval_js(js)) + len(domain)) params['jschl_answer'] = str(int(js2py.eval_js(js)) + len(domain))
# Requests transforms any request into a GET after a redirect, # Requests transforms any request into a GET after a redirect,
# so the redirect has to be handled manually here to allow for # so the redirect has to be handled manually here to allow for
# performing other types of requests even as the first request. # performing other types of requests even as the first request.
method = resp.request.method method = resp.request.method
cloudflare_kwargs["allow_redirects"] = False cloudflare_kwargs['allow_redirects'] = False
redirect = self.request(method, submit_url, **cloudflare_kwargs) redirect = self.request(method, submit_url, **cloudflare_kwargs)
return self.request(method, redirect.headers["Location"], **original_kwargs) return self.request(method, redirect.headers['Location'], **original_kwargs)
def extract_js(self, body): @staticmethod
js = re.search(r"setTimeout\(function\(\){\s+(var " def extract_js(body):
"s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n", body).group(1) js = re.search(r'setTimeout\(function\(\){\s+(var '
js = re.sub(r"a\.value = (parseInt\(.+?\)).+", r"\1", js) 's,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n', body).group(1)
js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js) js = re.sub(r'a\.value = (parseInt\(.+?\)).+', r'\1', js)
js = re.sub(r'\s{3,}[a-z](?: = |\.).+', '', js)
# Strip characters that could be used to exit the string context # Strip characters that could be used to exit the string context
# These characters are not currently used in Cloudflare's arithmetic snippet # These characters are not currently used in Cloudflare's arithmetic snippet
js = re.sub(r"[\n\\']", "", js) js = re.sub(r'[\n\\"]', '', js)
return js return js
@classmethod @classmethod
def create_scraper(cls, sess=None, **kwargs): def create_scraper(cls, sess=None):
""" """
Convenience function for creating a ready-to-go requests.Session (subclass) object. Convenience function for creating a ready-to-go requests.Session (subclass) object.
""" """
scraper = cls() scraper = cls()
if sess: if sess:
attrs = ["auth", "cert", "cookies", "headers", "hooks", "params", "proxies", "data"] attrs = ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']
for attr in attrs: for attr in attrs:
val = getattr(sess, attr, None) val = getattr(sess, attr, None)
if val: if val:
@ -115,46 +121,46 @@ class CloudflareScraper(Session):
return scraper return scraper
# Functions for integrating cloudflare-scrape with other applications and scripts
## Functions for integrating cloudflare-scrape with other applications and scripts
@classmethod @classmethod
def get_tokens(cls, url, user_agent=None, **kwargs): def get_tokens(cls, url, user_agent=None):
scraper = cls.create_scraper() scraper = cls.create_scraper()
if user_agent: if user_agent:
scraper.headers["User-Agent"] = user_agent scraper.headers['User-Agent'] = user_agent
# noinspection PyUnusedLocal
try: try:
resp = scraper.get(url) resp = scraper.get(url)
resp.raise_for_status() resp.raise_for_status()
except Exception as e: except Exception as e:
logging.error("'%s' returned an error. Could not collect tokens." % url) logging.error('[%s] returned an error. Could not collect tokens.' % url)
raise raise
domain = urlparse(resp.url).netloc domain = urlparse(resp.url).netloc
cookie_domain = None # cookie_domain = None
for d in scraper.cookies.list_domains(): for d in scraper.cookies.list_domains():
if d.startswith(".") and d in ("." + domain): if d.startswith('.') and d in ('.' + domain):
cookie_domain = d cookie_domain = d
break break
else: else:
raise ValueError("Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM (\"I'm Under Attack Mode\") enabled?") raise ValueError('Unable to find Cloudflare cookies.'
' Does the site actually have Cloudflare IUAM (\'I\'m Under Attack Mode\') enabled?')
return ({ return ({'__cfduid': scraper.cookies.get('__cfduid', '', domain=cookie_domain),
"__cfduid": scraper.cookies.get("__cfduid", "", domain=cookie_domain), 'cf_clearance': scraper.cookies.get('cf_clearance', '', domain=cookie_domain)
"cf_clearance": scraper.cookies.get("cf_clearance", "", domain=cookie_domain) },
}, scraper.headers['User-Agent'])
scraper.headers["User-Agent"]
)
@classmethod @classmethod
def get_cookie_string(cls, url, user_agent=None, **kwargs): def get_cookie_string(cls, url, user_agent=None):
""" """
Convenience function for building a Cookie HTTP header value. Convenience function for building a Cookie HTTP header value.
""" """
tokens, user_agent = cls.get_tokens(url, user_agent=user_agent) tokens, user_agent = cls.get_tokens(url, user_agent=user_agent)
return "; ".join("=".join(pair) for pair in tokens.items()), user_agent return '; '.join('='.join(pair) for pair in tokens.items()), user_agent
create_scraper = CloudflareScraper.create_scraper create_scraper = CloudflareScraper.create_scraper
get_tokens = CloudflareScraper.get_tokens get_tokens = CloudflareScraper.get_tokens

2
sickbeard/providers/torrentleech.py
View file

@ -29,7 +29,7 @@ class TorrentLeechProvider(generic.TorrentProvider):
def __init__(self): def __init__(self):
generic.TorrentProvider.__init__(self, 'TorrentLeech', cache_update_freq=20) generic.TorrentProvider.__init__(self, 'TorrentLeech', cache_update_freq=20)
self.url_base = 'https://torrentleech.org/' self.url_base = 'https://v4.torrentleech.org/'
self.urls = {'config_provider_home_uri': self.url_base, self.urls = {'config_provider_home_uri': self.url_base,
'login_action': self.url_base, 'login_action': self.url_base,
'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s', 'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s',