diff --git a/CHANGES.md b/CHANGES.md index cc242dca..8469ebaf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,13 @@ -### 0.13.6 (2017-12-13 01:50:00 UTC) +### 0.14.0 (2018-xx-xx xx:xx:xx UTC) + +* Add ETTV torrent provider +* Add PotUK torrent provider +* Fix BTScene and Lime + +[develop changelog] + + +### 0.13.6 (2017-12-13 01:50:00 UTC) * Change improve multi episode release search * Change improve usage of the optional regex library diff --git a/gui/slick/images/providers/ettv.png b/gui/slick/images/providers/ettv.png new file mode 100644 index 00000000..f4fddd48 Binary files /dev/null and b/gui/slick/images/providers/ettv.png differ diff --git a/gui/slick/images/providers/potuk.png b/gui/slick/images/providers/potuk.png new file mode 100644 index 00000000..22a785b5 Binary files /dev/null and b/gui/slick/images/providers/potuk.png differ diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py index 979ed703..ba8e982f 100755 --- a/sickbeard/providers/__init__.py +++ b/sickbeard/providers/__init__.py @@ -26,9 +26,9 @@ from sickbeard import logger, encodingKludge as ek # usenet from . import newznab, omgwtfnzbs # torrent -from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, \ +from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, ettv, \ fano, filelist, funfile, gftracker, grabtheinfo, hd4free, hdbits, hdspace, hdtorrents, \ - iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, pretome, privatehd, ptf, \ + iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, potuk, pretome, privatehd, ptf, \ rarbg, revtt, scenehd, scenetime, shazbat, skytorrents, speedcd, \ thepiratebay, torlock, torrentbytes, torrentday, torrenting, torrentleech, \ torrentvault, torrentz2, tvchaosuk, wop, zooqle @@ -51,6 +51,7 @@ __all__ = ['omgwtfnzbs', 'btscene', 'custom01', 'dh', + 'ettv', 'fano', 'filelist', 'funfile', @@ -68,6 +69,7 @@ __all__ = ['omgwtfnzbs', 'ncore', 'nyaa', 'pisexy', + 'potuk', 'pretome', 'privatehd', 'ptf', diff --git a/sickbeard/providers/btscene.py b/sickbeard/providers/btscene.py index 405af906..29169b9d 100644 --- a/sickbeard/providers/btscene.py +++ b/sickbeard/providers/btscene.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . +import base64 import re import traceback import urllib @@ -31,8 +32,13 @@ class BTSceneProvider(generic.TorrentProvider): def __init__(self): generic.TorrentProvider.__init__(self, 'BTScene') - self.url_home = ['http://btsone.cc/', 'http://diriri.xyz/', 'http://mytorrentz.tv/'] - + self.url_home = ['http://btsone.cc/', 'http://diriri.xyz/'] + \ + ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [ + [re.sub('[L\sT]+', '', x[::-1]) for x in [ + 'zTRnTY', 'uVT 2Y', '15LSTZ', 's JmLb', 'rTNL2b', 'uQW LZ', '=LLMmd']], + [re.sub('[j\sq]+', '', x[::-1]) for x in [ + 'zRn qY', 'l52j b', '1j5S M', 'sq Jmb', 'r Nq2b', 'ujQWqZ', 's9jGqb']], + ]]] self.url_vars = {'search': '?q=%s&category=series&order=1', 'browse': 'lastdaycat/type/Series/', 'get': 'torrentdownload.php?id=%s'} self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(vars)s', diff --git a/sickbeard/providers/ettv.py b/sickbeard/providers/ettv.py new file mode 100644 index 00000000..c83aa971 --- /dev/null +++ b/sickbeard/providers/ettv.py @@ -0,0 +1,129 @@ +# coding=utf-8 +# +# This file is part of SickGear. +# +# SickGear is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# SickGear is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with SickGear. If not, see . + +import re +import traceback + +from . import generic +from sickbeard import logger +from sickbeard.bs4_parser import BS4Parser +from sickbeard.helpers import tryInt +from lib.unidecode import unidecode + + +class ETTVProvider(generic.TorrentProvider): + + def __init__(self): + generic.TorrentProvider.__init__(self, 'ETTV') + + self.url_home = ['https://ettv.tv/'] + self.url_vars = {'search': 'torrents-search.php?%s&search=%s&sort=id&order=desc'} + self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s'} + + self.categories = {'Season': [7], 'Episode': [41, 5, 50]} + self.categories['Cache'] = self.categories['Season'] + self.categories['Episode'] + + self.minseed, self.minleech = 2 * [None] + + @staticmethod + def _has_signature(data=None): + return data and re.search(r'(?i)(?:ettv)', data) + + def _search_provider(self, search_params, **kwargs): + + results = [] + if not self.url: + return results + + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'torrent/'}.iteritems()) + + for mode in search_params.keys(): + for search_string in search_params[mode]: + + search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string + + search_url = self.urls['search'] % ( + self._categories_string(mode), ('%2B ', '')['Cache' == mode] + '.'.join(search_string.split())) + + html = self.get_url(search_url) + + cnt = len(items[mode]) + try: + if not html or self._has_no_results(html): + raise generic.HaltParseException + with BS4Parser(html, features=['html5lib', 'permissive']) as soup: + torrent_table = soup.find('table', class_='table') + torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') + + if not len(torrent_rows): + raise generic.HaltParseException + + head = None + for tr in torrent_rows[1:]: + cells = tr.find_all('td') + if 6 > len(cells): + continue + try: + head = head if None is not head else self._header_row( + tr, {'seed': r'seed', 'leech': r'leech', 'size': r'^size'}) + seeders, leechers, size = [tryInt(n, n) for n in [ + cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']] + if self._peers_fail(mode, seeders, leechers): + continue + + info = tr.find('a', href=rc['info']) + title = (info.attrs.get('title') or info.get_text()).strip() + download_url = self._link(info.get('href')) + except (AttributeError, TypeError, ValueError, IndexError): + continue + + if title and download_url: + items[mode].append((title, download_url, seeders, self._bytesizer(size))) + + except generic.HaltParseException: + pass + except (StandardError, Exception): + logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) + + self._log_search(mode, len(items[mode]) - cnt, search_url) + + results = self._sort_seeding(mode, results + items[mode]) + + return results + + def get_data(self, url): + result = None + html = self.get_url(url, timeout=90) + try: + result = re.findall('(?i)"(magnet:[^"]+?)">', html)[0] + except IndexError: + logger.log('Failed no magnet in response', logger.DEBUG) + return result + + def get_result(self, episodes, url): + result = None + + if url: + result = super(ETTVProvider, self).get_result(episodes, url) + result.get_data_func = self.get_data + + return result + + +provider = ETTVProvider() diff --git a/sickbeard/providers/limetorrents.py b/sickbeard/providers/limetorrents.py index 7d55dad7..61ce72e2 100644 --- a/sickbeard/providers/limetorrents.py +++ b/sickbeard/providers/limetorrents.py @@ -34,18 +34,8 @@ class LimeTorrentsProvider(generic.TorrentProvider): self.url_home = ['https://www.limetorrents.cc/'] + \ ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [ - [re.sub('[f\sX]+', '', x[::-1]) for x in [ - 'tlXGfb', '1X5SfZ', 'sfJfmb', 'rN 2Xb', 'u QfWZ', 's9G b']], - [re.sub('[ \ss]+', '', x[::-1]) for x in [ - 'Ztl Gsb', 'nc svRX', 'Rs nblJ', '5 JmLz', 'czsFsGc', 'nLskVs2', '0s N']], - [re.sub('[1\sF]+', '', x[::-1]) for x in [ - 'X Zt1lGb', 'l1Jn1cvR', 'mL11zRnb', 'uVXbtFFl', 'Hdp NWFa', '=1FQ3cuk']], - [re.sub('[y\sW]+', '', x[::-1]) for x in [ - 'XWZtlyGb', 'lJnyWcvR', 'nyLzRn b', 'vxmYWuWV', 'CWZlt2yY', '== Adyz5']], - [re.sub('[j\sy]+', '', x[::-1]) for x in [ - 'XyZtlG b', 'lJjnjcvR', 'njLz Rnb', 'vjxmYyuV', 'Gbyhjt2Y', 'n jJ3buw']], - [re.sub('[o\sg]+', '', x[::-1]) for x in [ - 'XZt lgGb', 'loJn cvR', 'ngLz Rnb', 'v xgmYuV', 'Gbh t2gY', '6l Heu w']], + [re.sub('[ \sF]+', '', x[::-1]) for x in [ + 'X ZtlFGb', 'lJnc vR', 'n LzR nb', 'vxmYuF V', 'CFZltF2Y', '==wYF2F5']], ]]] self.url_vars = {'search': 'search/tv/%s/', 'browse': 'browse-torrents/TV-shows/'} diff --git a/sickbeard/providers/potuk.py b/sickbeard/providers/potuk.py new file mode 100644 index 00000000..cbabbaf8 --- /dev/null +++ b/sickbeard/providers/potuk.py @@ -0,0 +1,157 @@ +# coding=utf-8 +# +# This file is part of SickGear. +# +# SickGear is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# SickGear is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with SickGear. If not, see . + +import re +import traceback + +from . import generic +from sickbeard import logger +from sickbeard.bs4_parser import BS4Parser +from sickbeard.helpers import tryInt +from lib.unidecode import unidecode + + +class PotUKProvider(generic.TorrentProvider): + + def __init__(self): + generic.TorrentProvider.__init__(self, 'PotUK') + + self.url_base = 'http://www.potuk.com/newforum/' + self.urls = {'config_provider_home_uri': self.url_base, + 'login': self.url_base + 'search.php', + 'browse': self.url_base + 'search.php?do=getdaily&exclude=%s', + 'get_data': self.url_base + 'misc.php?do=showattachments&t=%s'} + + self.url = self.urls['config_provider_home_uri'] + + self.digest, self.resp = 2 * [None] + + def logged_in(self, resp): + try: + self.resp = re.findall('(?sim)
', resp)[0] + except (IndexError, TypeError): + return False + return self.has_all_cookies('bbsessionhash') + + def _authorised(self, **kwargs): + + return super(PotUKProvider, self)._authorised( + logged_in=(lambda y=None: self.logged_in(y)), + failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) + + def _search_provider(self, search_params, **kwargs): + + results = [] + if not self._authorised(): + return results + + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + opts = re.findall('(?sim)forumchoice\[\][^<]+(.*?)', self.resp)[0] + cat_opts = re.findall(r'(?mis)]*?value=[\'"](\d+)[^>]*>(.*?)', opts) + include = [] + tv = False + for c in cat_opts: + if not tv and 'TV Shows' in c[1]: + tv = True + elif tv: + if 3 > len(re.findall(' ', c[1])): + break + elif not filter(lambda v: v in c[1], ('Requests', 'Offer', 'Discussion')): + include += [c[0]] + exclude = ','.join(list(filter(lambda v: v not in include, map(lambda x: x[0], cat_opts)))) + + for mode in search_params.keys(): + for search_string in search_params[mode]: + search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string + + params = {} + if 'Cache' == mode: + search_url = self.urls['browse'] % exclude + else: + search_url = self._link(re.findall('(?i)action="([^"]+?)"', self.resp)[0]) + params = {'query': search_string, 'showposts': 0, 'titleonly': 1, 'prefixchoice': '', + 'replyless': 0, 'searchdate': 0, 'beforeafter': 'after', 'sortby': 'threadstart', + 'order': 'descending', 'starteronly': 0, 'forumchoice': include} + tags = re.findall(r'(?is)(]*?name=[\'"][^\'"]+[^>]*)', self.resp) + attrs = [[(re.findall(r'(?is)%s=[\'"]([^\'"]+)' % attr, c) or [''])[0] + for attr in ['type', 'name', 'value']] for c in tags] + for itype, name, value in attrs: + params.setdefault(name, value) + del params['doprefs'] + html = self.get_url(search_url, post_data=params) + + cnt = len(items[mode]) + try: + if not html or self._has_no_results(html): + raise generic.HaltParseException + + with BS4Parser(html, features=['html5lib', 'permissive']) as soup: + torrent_table = soup.find('table', id='threadslist') + torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') + + if 2 > len(torrent_rows): + raise generic.HaltParseException + + for tr in torrent_rows[1:]: + if 6 > len(tr.find_all('td')) or not tr.select('img[alt*="ttach"]'): + continue + try: + link = tr.select('td[id^="td_threadtitle"]')[0].select('a[id*="title"]')[0] + title = link.get_text().strip() + download_url = self.urls['get_data'] % re.findall('t=(\d+)', link['href'])[0] + except (AttributeError, TypeError, ValueError, IndexError): + continue + + if title and download_url: + items[mode].append((title, download_url, '', '')) + + except generic.HaltParseException: + pass + except (StandardError, Exception): + logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) + + self._log_search( + mode, len(items[mode]) - cnt, ('search_param: ' + search_string, search_url)['Cache' == mode]) + + results = self._sort_seeding(mode, results + items[mode]) + + return results + + def get_data(self, url): + result = None + html = self.get_url(url, timeout=90) + try: + result = self._link(re.findall('(?i)"(attachment\.php[^"]+?)"', html)[0]) + except IndexError: + logger.log('Failed no torrent in response', logger.DEBUG) + return result + + def get_result(self, episodes, url): + result = None + + if url: + result = super(PotUKProvider, self).get_result(episodes, url) + result.get_data_func = self.get_data + + return result + + def ui_string(self, key): + return ('%s_digest' % self.get_id()) == key and 'use... \'bbuserid=xx; bbpassword=yy\'' or '' + + +provider = PotUKProvider() diff --git a/sickbeard/search.py b/sickbeard/search.py index 4aca07ff..ce6a3615 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -130,6 +130,11 @@ def snatch_episode(result, end_status=SNATCHED): # TORRENTs can be sent to clients or saved to disk elif 'torrent' == result.resultType: + if not result.url.startswith('magnet') and None is not result.get_data_func: + result.url = result.get_data_func(result.url) + result.get_data_func = None # consume only once + if not result.url: + return False # torrents are saved to disk when blackhole mode if 'blackhole' == sickbeard.TORRENT_METHOD: dl_result = _download_result(result) @@ -739,6 +744,11 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr # filter out possible bad torrents from providers if 'torrent' == best_result.resultType: + if not best_result.url.startswith('magnet') and None is not best_result.get_data_func: + best_result.url = best_result.get_data_func(best_result.url) + best_result.get_data_func = None # consume only once + if not best_result.url: + continue if best_result.url.startswith('magnet'): if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = None