From 8d97f2664a63a5bf2949d97b40a4572ca39ac5af Mon Sep 17 00:00:00 2001 From: JackDandy Date: Sat, 16 Dec 2017 02:52:08 +0000 Subject: [PATCH 1/2] Add ETTV and PotUK providers, fix BTScene and Lime. --- CHANGES.md | 11 +- gui/slick/images/providers/ettv.png | Bin 0 -> 268 bytes gui/slick/images/providers/potuk.png | Bin 0 -> 237 bytes sickbeard/providers/__init__.py | 6 +- sickbeard/providers/btscene.py | 10 +- sickbeard/providers/ettv.py | 129 ++++++++++++++++++++++ sickbeard/providers/limetorrents.py | 14 +-- sickbeard/providers/potuk.py | 157 +++++++++++++++++++++++++++ sickbeard/search.py | 10 ++ 9 files changed, 320 insertions(+), 17 deletions(-) create mode 100644 gui/slick/images/providers/ettv.png create mode 100644 gui/slick/images/providers/potuk.png create mode 100644 sickbeard/providers/ettv.py create mode 100644 sickbeard/providers/potuk.py diff --git a/CHANGES.md b/CHANGES.md index cc242dca..8469ebaf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,13 @@ -### 0.13.6 (2017-12-13 01:50:00 UTC) +### 0.14.0 (2018-xx-xx xx:xx:xx UTC) + +* Add ETTV torrent provider +* Add PotUK torrent provider +* Fix BTScene and Lime + +[develop changelog] + + +### 0.13.6 (2017-12-13 01:50:00 UTC) * Change improve multi episode release search * Change improve usage of the optional regex library diff --git a/gui/slick/images/providers/ettv.png b/gui/slick/images/providers/ettv.png new file mode 100644 index 0000000000000000000000000000000000000000..f4fddd483feb4491370fecf4654963e58295686b GIT binary patch literal 268 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!73?$#)eFPFP2=EDU4QFAQRNrvn)ad{%tu%lC zQZ}{?OP2on{MA)lyec#6@SeTT9z1GdXHO6l%i`czG;z|mD90N>y_^Lek;M!Q+(IDC zc%hbCuir!_x4#HE+~tNin^s_ub{klPgQy4A<4GI+fQvb zIUmOu@_Ap)XJ27<1?3Gjn}4%}87%gSeZVs#aIfsC8;|TB%1sdD54p4F6Yt@f=}%R8 z{k#~gm%UQ1?%c14e6Nhjmp0$s!4 M>FVdQ&MBb@0EN$F)&Kwi literal 0 HcmV?d00001 diff --git a/gui/slick/images/providers/potuk.png b/gui/slick/images/providers/potuk.png new file mode 100644 index 0000000000000000000000000000000000000000..22a785b516c3997e3d7edbe8434bf2e2bd1abfb1 GIT binary patch literal 237 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|SkfJR9T^xl z_H+M9WCij$3p^r=85sBufiR<}hF1enu*uWKF~q`uZI2^ggMt9_&;QOpncTOW))Uig zlu%!JaG~nn73~wR1YgLV%)Z}>eYV+O1HKJ^|1(aX=3x0CZ`${C4Q=)h_3mEY=WZVA zD>V0V>zz@-utTcx{+-f@TagF9*S(vevouHELH(Qrn;P?<)dugZRy*-K*Cl9aEowi= erMFT*Gj`U6yytCedNhEpVeoYIb6Mw<&;$VElTyF{ literal 0 HcmV?d00001 diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py index 979ed703..ba8e982f 100755 --- a/sickbeard/providers/__init__.py +++ b/sickbeard/providers/__init__.py @@ -26,9 +26,9 @@ from sickbeard import logger, encodingKludge as ek # usenet from . import newznab, omgwtfnzbs # torrent -from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, \ +from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, ettv, \ fano, filelist, funfile, gftracker, grabtheinfo, hd4free, hdbits, hdspace, hdtorrents, \ - iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, pretome, privatehd, ptf, \ + iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, potuk, pretome, privatehd, ptf, \ rarbg, revtt, scenehd, scenetime, shazbat, skytorrents, speedcd, \ thepiratebay, torlock, torrentbytes, torrentday, torrenting, torrentleech, \ torrentvault, torrentz2, tvchaosuk, wop, zooqle @@ -51,6 +51,7 @@ __all__ = ['omgwtfnzbs', 'btscene', 'custom01', 'dh', + 'ettv', 'fano', 'filelist', 'funfile', @@ -68,6 +69,7 @@ __all__ = ['omgwtfnzbs', 'ncore', 'nyaa', 'pisexy', + 'potuk', 'pretome', 'privatehd', 'ptf', diff --git a/sickbeard/providers/btscene.py b/sickbeard/providers/btscene.py index 405af906..29169b9d 100644 --- a/sickbeard/providers/btscene.py +++ b/sickbeard/providers/btscene.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . +import base64 import re import traceback import urllib @@ -31,8 +32,13 @@ class BTSceneProvider(generic.TorrentProvider): def __init__(self): generic.TorrentProvider.__init__(self, 'BTScene') - self.url_home = ['http://btsone.cc/', 'http://diriri.xyz/', 'http://mytorrentz.tv/'] - + self.url_home = ['http://btsone.cc/', 'http://diriri.xyz/'] + \ + ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [ + [re.sub('[L\sT]+', '', x[::-1]) for x in [ + 'zTRnTY', 'uVT 2Y', '15LSTZ', 's JmLb', 'rTNL2b', 'uQW LZ', '=LLMmd']], + [re.sub('[j\sq]+', '', x[::-1]) for x in [ + 'zRn qY', 'l52j b', '1j5S M', 'sq Jmb', 'r Nq2b', 'ujQWqZ', 's9jGqb']], + ]]] self.url_vars = {'search': '?q=%s&category=series&order=1', 'browse': 'lastdaycat/type/Series/', 'get': 'torrentdownload.php?id=%s'} self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(vars)s', diff --git a/sickbeard/providers/ettv.py b/sickbeard/providers/ettv.py new file mode 100644 index 00000000..c83aa971 --- /dev/null +++ b/sickbeard/providers/ettv.py @@ -0,0 +1,129 @@ +# coding=utf-8 +# +# This file is part of SickGear. +# +# SickGear is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# SickGear is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with SickGear. If not, see . + +import re +import traceback + +from . import generic +from sickbeard import logger +from sickbeard.bs4_parser import BS4Parser +from sickbeard.helpers import tryInt +from lib.unidecode import unidecode + + +class ETTVProvider(generic.TorrentProvider): + + def __init__(self): + generic.TorrentProvider.__init__(self, 'ETTV') + + self.url_home = ['https://ettv.tv/'] + self.url_vars = {'search': 'torrents-search.php?%s&search=%s&sort=id&order=desc'} + self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s'} + + self.categories = {'Season': [7], 'Episode': [41, 5, 50]} + self.categories['Cache'] = self.categories['Season'] + self.categories['Episode'] + + self.minseed, self.minleech = 2 * [None] + + @staticmethod + def _has_signature(data=None): + return data and re.search(r'(?i)(?:ettv)', data) + + def _search_provider(self, search_params, **kwargs): + + results = [] + if not self.url: + return results + + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'torrent/'}.iteritems()) + + for mode in search_params.keys(): + for search_string in search_params[mode]: + + search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string + + search_url = self.urls['search'] % ( + self._categories_string(mode), ('%2B ', '')['Cache' == mode] + '.'.join(search_string.split())) + + html = self.get_url(search_url) + + cnt = len(items[mode]) + try: + if not html or self._has_no_results(html): + raise generic.HaltParseException + with BS4Parser(html, features=['html5lib', 'permissive']) as soup: + torrent_table = soup.find('table', class_='table') + torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') + + if not len(torrent_rows): + raise generic.HaltParseException + + head = None + for tr in torrent_rows[1:]: + cells = tr.find_all('td') + if 6 > len(cells): + continue + try: + head = head if None is not head else self._header_row( + tr, {'seed': r'seed', 'leech': r'leech', 'size': r'^size'}) + seeders, leechers, size = [tryInt(n, n) for n in [ + cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']] + if self._peers_fail(mode, seeders, leechers): + continue + + info = tr.find('a', href=rc['info']) + title = (info.attrs.get('title') or info.get_text()).strip() + download_url = self._link(info.get('href')) + except (AttributeError, TypeError, ValueError, IndexError): + continue + + if title and download_url: + items[mode].append((title, download_url, seeders, self._bytesizer(size))) + + except generic.HaltParseException: + pass + except (StandardError, Exception): + logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) + + self._log_search(mode, len(items[mode]) - cnt, search_url) + + results = self._sort_seeding(mode, results + items[mode]) + + return results + + def get_data(self, url): + result = None + html = self.get_url(url, timeout=90) + try: + result = re.findall('(?i)"(magnet:[^"]+?)">', html)[0] + except IndexError: + logger.log('Failed no magnet in response', logger.DEBUG) + return result + + def get_result(self, episodes, url): + result = None + + if url: + result = super(ETTVProvider, self).get_result(episodes, url) + result.get_data_func = self.get_data + + return result + + +provider = ETTVProvider() diff --git a/sickbeard/providers/limetorrents.py b/sickbeard/providers/limetorrents.py index 7d55dad7..61ce72e2 100644 --- a/sickbeard/providers/limetorrents.py +++ b/sickbeard/providers/limetorrents.py @@ -34,18 +34,8 @@ class LimeTorrentsProvider(generic.TorrentProvider): self.url_home = ['https://www.limetorrents.cc/'] + \ ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [ - [re.sub('[f\sX]+', '', x[::-1]) for x in [ - 'tlXGfb', '1X5SfZ', 'sfJfmb', 'rN 2Xb', 'u QfWZ', 's9G b']], - [re.sub('[ \ss]+', '', x[::-1]) for x in [ - 'Ztl Gsb', 'nc svRX', 'Rs nblJ', '5 JmLz', 'czsFsGc', 'nLskVs2', '0s N']], - [re.sub('[1\sF]+', '', x[::-1]) for x in [ - 'X Zt1lGb', 'l1Jn1cvR', 'mL11zRnb', 'uVXbtFFl', 'Hdp NWFa', '=1FQ3cuk']], - [re.sub('[y\sW]+', '', x[::-1]) for x in [ - 'XWZtlyGb', 'lJnyWcvR', 'nyLzRn b', 'vxmYWuWV', 'CWZlt2yY', '== Adyz5']], - [re.sub('[j\sy]+', '', x[::-1]) for x in [ - 'XyZtlG b', 'lJjnjcvR', 'njLz Rnb', 'vjxmYyuV', 'Gbyhjt2Y', 'n jJ3buw']], - [re.sub('[o\sg]+', '', x[::-1]) for x in [ - 'XZt lgGb', 'loJn cvR', 'ngLz Rnb', 'v xgmYuV', 'Gbh t2gY', '6l Heu w']], + [re.sub('[ \sF]+', '', x[::-1]) for x in [ + 'X ZtlFGb', 'lJnc vR', 'n LzR nb', 'vxmYuF V', 'CFZltF2Y', '==wYF2F5']], ]]] self.url_vars = {'search': 'search/tv/%s/', 'browse': 'browse-torrents/TV-shows/'} diff --git a/sickbeard/providers/potuk.py b/sickbeard/providers/potuk.py new file mode 100644 index 00000000..cbabbaf8 --- /dev/null +++ b/sickbeard/providers/potuk.py @@ -0,0 +1,157 @@ +# coding=utf-8 +# +# This file is part of SickGear. +# +# SickGear is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# SickGear is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with SickGear. If not, see . + +import re +import traceback + +from . import generic +from sickbeard import logger +from sickbeard.bs4_parser import BS4Parser +from sickbeard.helpers import tryInt +from lib.unidecode import unidecode + + +class PotUKProvider(generic.TorrentProvider): + + def __init__(self): + generic.TorrentProvider.__init__(self, 'PotUK') + + self.url_base = 'http://www.potuk.com/newforum/' + self.urls = {'config_provider_home_uri': self.url_base, + 'login': self.url_base + 'search.php', + 'browse': self.url_base + 'search.php?do=getdaily&exclude=%s', + 'get_data': self.url_base + 'misc.php?do=showattachments&t=%s'} + + self.url = self.urls['config_provider_home_uri'] + + self.digest, self.resp = 2 * [None] + + def logged_in(self, resp): + try: + self.resp = re.findall('(?sim)
', resp)[0] + except (IndexError, TypeError): + return False + return self.has_all_cookies('bbsessionhash') + + def _authorised(self, **kwargs): + + return super(PotUKProvider, self)._authorised( + logged_in=(lambda y=None: self.logged_in(y)), + failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) + + def _search_provider(self, search_params, **kwargs): + + results = [] + if not self._authorised(): + return results + + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + opts = re.findall('(?sim)forumchoice\[\][^<]+(.*?)', self.resp)[0] + cat_opts = re.findall(r'(?mis)]*?value=[\'"](\d+)[^>]*>(.*?)', opts) + include = [] + tv = False + for c in cat_opts: + if not tv and 'TV Shows' in c[1]: + tv = True + elif tv: + if 3 > len(re.findall(' ', c[1])): + break + elif not filter(lambda v: v in c[1], ('Requests', 'Offer', 'Discussion')): + include += [c[0]] + exclude = ','.join(list(filter(lambda v: v not in include, map(lambda x: x[0], cat_opts)))) + + for mode in search_params.keys(): + for search_string in search_params[mode]: + search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string + + params = {} + if 'Cache' == mode: + search_url = self.urls['browse'] % exclude + else: + search_url = self._link(re.findall('(?i)action="([^"]+?)"', self.resp)[0]) + params = {'query': search_string, 'showposts': 0, 'titleonly': 1, 'prefixchoice': '', + 'replyless': 0, 'searchdate': 0, 'beforeafter': 'after', 'sortby': 'threadstart', + 'order': 'descending', 'starteronly': 0, 'forumchoice': include} + tags = re.findall(r'(?is)(]*?name=[\'"][^\'"]+[^>]*)', self.resp) + attrs = [[(re.findall(r'(?is)%s=[\'"]([^\'"]+)' % attr, c) or [''])[0] + for attr in ['type', 'name', 'value']] for c in tags] + for itype, name, value in attrs: + params.setdefault(name, value) + del params['doprefs'] + html = self.get_url(search_url, post_data=params) + + cnt = len(items[mode]) + try: + if not html or self._has_no_results(html): + raise generic.HaltParseException + + with BS4Parser(html, features=['html5lib', 'permissive']) as soup: + torrent_table = soup.find('table', id='threadslist') + torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') + + if 2 > len(torrent_rows): + raise generic.HaltParseException + + for tr in torrent_rows[1:]: + if 6 > len(tr.find_all('td')) or not tr.select('img[alt*="ttach"]'): + continue + try: + link = tr.select('td[id^="td_threadtitle"]')[0].select('a[id*="title"]')[0] + title = link.get_text().strip() + download_url = self.urls['get_data'] % re.findall('t=(\d+)', link['href'])[0] + except (AttributeError, TypeError, ValueError, IndexError): + continue + + if title and download_url: + items[mode].append((title, download_url, '', '')) + + except generic.HaltParseException: + pass + except (StandardError, Exception): + logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) + + self._log_search( + mode, len(items[mode]) - cnt, ('search_param: ' + search_string, search_url)['Cache' == mode]) + + results = self._sort_seeding(mode, results + items[mode]) + + return results + + def get_data(self, url): + result = None + html = self.get_url(url, timeout=90) + try: + result = self._link(re.findall('(?i)"(attachment\.php[^"]+?)"', html)[0]) + except IndexError: + logger.log('Failed no torrent in response', logger.DEBUG) + return result + + def get_result(self, episodes, url): + result = None + + if url: + result = super(PotUKProvider, self).get_result(episodes, url) + result.get_data_func = self.get_data + + return result + + def ui_string(self, key): + return ('%s_digest' % self.get_id()) == key and 'use... \'bbuserid=xx; bbpassword=yy\'' or '' + + +provider = PotUKProvider() diff --git a/sickbeard/search.py b/sickbeard/search.py index 4aca07ff..ce6a3615 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -130,6 +130,11 @@ def snatch_episode(result, end_status=SNATCHED): # TORRENTs can be sent to clients or saved to disk elif 'torrent' == result.resultType: + if not result.url.startswith('magnet') and None is not result.get_data_func: + result.url = result.get_data_func(result.url) + result.get_data_func = None # consume only once + if not result.url: + return False # torrents are saved to disk when blackhole mode if 'blackhole' == sickbeard.TORRENT_METHOD: dl_result = _download_result(result) @@ -739,6 +744,11 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr # filter out possible bad torrents from providers if 'torrent' == best_result.resultType: + if not best_result.url.startswith('magnet') and None is not best_result.get_data_func: + best_result.url = best_result.get_data_func(best_result.url) + best_result.get_data_func = None # consume only once + if not best_result.url: + continue if best_result.url.startswith('magnet'): if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = None From 6da32a5ed053e1a42202860b74f76b20c5e33c5c Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Wed, 27 Dec 2017 03:14:20 +0000 Subject: [PATCH 2/2] Add log message for not found on indexer when adding a new show. Fix upgrade once ARCHIVED setting by postProcessor. Fix determination of is_first_best_match. Change improve smart selection of categories in manual and failed search modes. Change refactor wantedQuality into own function that can be used in multiple places. Change improve error resistance in neededQualities class. Add log warning message if wantedQuality or eps_aired_in_season is missing for search. Add check backlogitem for wantedQuality and add if missing. Add use wantedQuality list in wantEpisode. Change don't use wantedQualities for multipart. --- CHANGES.md | 9 +- gui/slick/interfaces/default/displayShow.tmpl | 2 +- gui/slick/interfaces/default/editShow.tmpl | 4 +- .../interfaces/default/manage_massEdit.tmpl | 4 +- sickbeard/__init__.py | 6 +- sickbeard/common.py | 56 ++++++++ sickbeard/postProcessor.py | 7 +- sickbeard/providers/btscene.py | 3 + sickbeard/providers/newznab.py | 36 ++++- sickbeard/search.py | 133 +++++++++--------- sickbeard/search_queue.py | 19 ++- sickbeard/show_queue.py | 17 ++- sickbeard/tv.py | 65 ++++++--- sickbeard/webserve.py | 38 ++--- tests/test_lib.py | 2 +- 15 files changed, 270 insertions(+), 131 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8469ebaf..cd67f27a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,11 @@ -### 0.14.0 (2018-xx-xx xx:xx:xx UTC) +### 0.13.7 (2017-12-27 03:00:00 UTC) +* Add log message for not found on indexer when adding a new show +* Fix upgrade once ARCHIVED setting by postProcessor +* Fix determination of is_first_best_match +* Fix BTScene and Lime * Add ETTV torrent provider * Add PotUK torrent provider -* Fix BTScene and Lime - -[develop changelog] ### 0.13.6 (2017-12-13 01:50:00 UTC) diff --git a/gui/slick/interfaces/default/displayShow.tmpl b/gui/slick/interfaces/default/displayShow.tmpl index ff0e044d..acf25b33 100644 --- a/gui/slick/interfaces/default/displayShow.tmpl +++ b/gui/slick/interfaces/default/displayShow.tmpl @@ -340,7 +340,7 @@ #if $show.paused Paused #end if -#if ($anyQualities + $bestQualities) and int($show.archive_firstmatch) +#if ($anyQualities + $bestQualities) and int($show.upgrade_once) Upgrade once #end if #if $show.exceptions diff --git a/gui/slick/interfaces/default/editShow.tmpl b/gui/slick/interfaces/default/editShow.tmpl index fde593ae..22317152 100644 --- a/gui/slick/interfaces/default/editShow.tmpl +++ b/gui/slick/interfaces/default/editShow.tmpl @@ -152,10 +152,10 @@ #if $anyQualities + $bestQualities