diff --git a/CHANGES.md b/CHANGES.md index f6af4e15..bfa1e324 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -87,14 +87,16 @@ * Add DigitalHive torrent provider * Add RevTT torrent provider * Add PTF torrent provider -* Add ILT torrent provider * Add Fano torrent provider * Add BTScene torrent provider * Add Extratorrent provider * Add Limetorrents provider +* Add HD-Torrents provider * Add nCore torrent provider * Add TorLock provider * Add Torrentz2 provider +* Add freeleech options to fano, freshon, hdspace, phd, ptf providers +* Change SceneTime to cookie auth * Remove Usenet-Crawler provider * Change CPU throttling on General Config/Advanced to "Disabled" by default for new installs * Change provider OMGWTFNZBS api url and auto reject nuked releases @@ -180,6 +182,7 @@ * Add handler for when rar files can not be opened during post processing * Fix join clean up * Fix add custom torrent RSS +* Remove ILT torrent provider ### 0.11.15 (2016-09-13 19:50:00 UTC) diff --git a/gui/slick/css/style.css b/gui/slick/css/style.css index 4c43ece1..174ed9c2 100644 --- a/gui/slick/css/style.css +++ b/gui/slick/css/style.css @@ -2290,7 +2290,7 @@ config*.tmpl } #config label.space-right{ - margin-right:20px + margin-right:16px } #config .metadataDiv{ diff --git a/gui/slick/images/providers/hdtorrents.png b/gui/slick/images/providers/hdtorrents.png new file mode 100644 index 00000000..a19defc1 Binary files /dev/null and b/gui/slick/images/providers/hdtorrents.png differ diff --git a/gui/slick/images/providers/ilovetorrents.png b/gui/slick/images/providers/ilovetorrents.png deleted file mode 100644 index 8d17f2b5..00000000 Binary files a/gui/slick/images/providers/ilovetorrents.png and /dev/null differ diff --git a/gui/slick/interfaces/default/config_providers.tmpl b/gui/slick/interfaces/default/config_providers.tmpl index 6d2b3290..0466eceb 100644 --- a/gui/slick/interfaces/default/config_providers.tmpl +++ b/gui/slick/interfaces/default/config_providers.tmpl @@ -520,6 +520,23 @@ name = '' if not client else get_client_instance(sickbeard.TORRENT_METHOD)().nam #end if + #if $hasattr($cur_torrent_provider, 'may_filter'): +
+ Allow releases that are + + #for $cur_fval, $filter in $cur_torrent_provider.may_filter.iteritems() + #set $cur_fname, $cur_is_default = $filter[0], $filter[1] + #set $filter_id = '%s_filter_%s' % ($cur_torrent_provider.get_id(), $cur_fval) + + #end for + (see site for meaning) +

nothing selected allows everything (no filter, default)

+ +
+ #end if #if $hasattr($cur_torrent_provider, 'reject_m2ts'):
', '', html) + html = re.sub('(?im)href=([^\\"][^>]+)>', r'href="\1">', html) + html = (html.replace('"/>', '" />') + .replace('"title="', '" title="') + .replace('', '')) + html = re.sub('(?im)]+)', r'\1. - -import re -import traceback - -from . import generic -from sickbeard import logger -from sickbeard.bs4_parser import BS4Parser -from sickbeard.helpers import tryInt -from lib.unidecode import unidecode - - -class ILTProvider(generic.TorrentProvider): - - def __init__(self): - generic.TorrentProvider.__init__(self, 'ILoveTorrents') - - self.url_base = 'https://www.ilovetorrents.me/' - self.urls = {'config_provider_home_uri': self.url_base, - 'login_action': self.url_base + 'login.php', - 'search': self.url_base + 'browse.php?search=%s&%s&incldead=0&blah=0', - 'get': self.url_base + '%s'} - - self.categories = {'Season': [5], 'Episode': [7, 8, 40], 'anime': [23]} - self.categories['Cache'] = self.categories['Season'] + self.categories['Episode'] - - self.url = self.urls['config_provider_home_uri'] - - self.username, self.password, self.minseed, self.minleech = 4 * [None] - - def _authorised(self, **kwargs): - - return super(ILTProvider, self)._authorised() - - def _search_provider(self, search_params, **kwargs): - - results = [] - if not self._authorised(): - return results - - items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} - - rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'details', 'get': 'download'}.items()) - for mode in search_params.keys(): - rc['cats'] = re.compile('(?i)cat=(?:%s)' % self._categories_string(mode, template='', delimiter='|')) - for search_string in search_params[mode]: - search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string - - html = self.get_url(self.urls['search'] % ('+'.join(search_string.split()), - self._categories_string(mode))) - - cnt = len(items[mode]) - try: - if not html or self._has_no_results(html): - raise generic.HaltParseException - - with BS4Parser(html, features=['html5lib', 'permissive']) as soup: - torrent_table = soup.find('table', 'koptekst') - torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') - - if 2 > len(torrent_rows): - raise generic.HaltParseException - - for tr in torrent_rows[1:]: - try: - seeders, leechers, size = [tryInt(n, n) for n in [ - tr.find_all('td')[x].get_text().strip() for x in -3, -2, -5]] - if self._peers_fail(mode, seeders, leechers) or not tr.find('a', href=rc['cats']): - continue - - title = tr.find('a', href=rc['info']).get_text().strip() - download_url = self._link(tr.find('a', href=rc['get'])['href']) - except (AttributeError, TypeError, ValueError, IndexError): - continue - - if title and download_url: - items[mode].append((title, download_url, seeders, self._bytesizer(size))) - - except generic.HaltParseException: - pass - except (StandardError, Exception): - logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - - self._log_search(mode, len(items[mode]) - cnt, self.session.response.get('url')) - - results = self._sort_seeding(mode, results + items[mode]) - - return results - - -provider = ILTProvider() diff --git a/sickbeard/providers/ncore.py b/sickbeard/providers/ncore.py index 0ac76f09..0316fce0 100644 --- a/sickbeard/providers/ncore.py +++ b/sickbeard/providers/ncore.py @@ -49,7 +49,7 @@ class NcoreProvider(generic.TorrentProvider): return super(NcoreProvider, self)._authorised( logged_in=(lambda y='': all([bool(y), 'action="login' not in y, self.has_all_cookies('PHPSESSID')])), - post_params={'nev': self.username, 'pass': self.password, 'form_tmpl': 'name=[\'"]login[\'"]'}) + post_params={'nev': self.username, 'form_tmpl': 'name=[\'"]login[\'"]'}) def _search_provider(self, search_params, **kwargs): diff --git a/sickbeard/providers/pisexy.py b/sickbeard/providers/pisexy.py index c3fbb882..439aa856 100644 --- a/sickbeard/providers/pisexy.py +++ b/sickbeard/providers/pisexy.py @@ -36,7 +36,7 @@ class PiSexyProvider(generic.TorrentProvider): self.url = self.urls['config_provider_home_uri'] - self.username, self.password, self.minseed, self.minleech = 4 * [None] + self.username, self.password, self.freeleech, self.minseed, self.minleech = 5 * [None] def _authorised(self, **kwargs): @@ -51,9 +51,9 @@ class PiSexyProvider(generic.TorrentProvider): items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} - rc = dict((k, re.compile('(?i)' + v)) - for (k, v) in {'info': 'download', 'get': 'download', 'valid_cat': 'cat=(?:0|50[12])', - 'title': r'Download\s([^\s]+).*', 'seeders': r'(^\d+)', 'leechers': r'(\d+)$'}.items()) + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in { + 'info': 'download', 'get': 'download', 'valid_cat': 'cat=(?:0|50[12])', 'filter': 'free', + 'title': r'Download\s([^\s]+).*', 'seeders': r'(^\d+)', 'leechers': r'(\d+)$'}.items()) for mode in search_params.keys(): for search_string in search_params[mode]: search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string @@ -78,7 +78,8 @@ class PiSexyProvider(generic.TorrentProvider): seeders, leechers = 2 * [tr.find_all('td')[-4].get_text().strip()] seeders, leechers = [tryInt(n) for n in [ rc['seeders'].findall(seeders)[0], rc['leechers'].findall(leechers)[0]]] - if self._peers_fail(mode, seeders, leechers) or not tr.find('a', href=rc['valid_cat']): + if self._peers_fail(mode, seeders, leechers) or not tr.find('a', href=rc['valid_cat']) \ + or (self.freeleech and not tr.find('img', src=rc['filter'])): continue info = tr.find('a', href=rc['info']) diff --git a/sickbeard/providers/privatehd.py b/sickbeard/providers/privatehd.py index d316135b..e0ba5b17 100644 --- a/sickbeard/providers/privatehd.py +++ b/sickbeard/providers/privatehd.py @@ -15,6 +15,10 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . +try: + from collections import OrderedDict +except ImportError: + from requests.compat import OrderedDict import re import traceback @@ -35,13 +39,18 @@ class PrivateHDProvider(generic.TorrentProvider): 'login_action': self.url_base + 'auth/login', 'search': self.url_base + 'torrents?%s' % '&'.join( ['in=1', 'tags=', 'type=2', 'language=0', 'subtitle=0', 'rip_type=0', - 'video_quality=0', 'uploader=', 'search=%s', 'tv_type[]=%s', 'discount[]=%s'])} + 'video_quality=0', 'uploader=', 'search=%s', 'tv_type[]=%s'])} self.categories = {'Season': [2], 'Episode': [1], 'Cache': [0]} self.url = self.urls['config_provider_home_uri'] - self.username, self.password, self.freeleech, self.minseed, self.minleech = 5 * [None] + self.filter = [] + self.may_filter = OrderedDict([ + ('f0', ('not marked', False)), ('free', ('free', True)), + ('half', ('50% down', True)), ('double', ('2x up', True))]) + self.username, self.password, self.minseed, self.minleech = 4 * [None] + self.confirmed = False def _authorised(self, **kwargs): @@ -59,6 +68,21 @@ class PrivateHDProvider(generic.TorrentProvider): rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': '.*?details\s*-\s*', 'get': 'download'}.items()) + log = '' + if self.filter: + non_marked = 'f0' in self.filter + # if search_any, use unselected to exclude, else use selected to keep + filters = ([f for f in self.may_filter if f in self.filter], + [f for f in self.may_filter if f not in self.filter])[non_marked] + filters += (((all([x in filters for x in 'free', 'double']) and ['freedouble'] or []) + + (all([x in filters for x in 'half', 'double']) and ['halfdouble'] or [])), + ((not all([x not in filters for x in 'free', 'double']) and ['freedouble'] or []) + + (not all([x not in filters for x in 'half', 'double']) and ['halfdouble'] or [])) + )[non_marked] + rc['filter'] = re.compile('(?i)^(%s)$' % '|'.join( + ['%s' % f for f in filters if (f in self.may_filter and self.may_filter[f][1]) or f])) + log = '%sing (%s) ' % (('keep', 'skipp')[non_marked], ', '.join( + [f in self.may_filter and self.may_filter[f][0] or f for f in filters])) for mode in search_params.keys(): if mode in ['Season', 'Episode']: show_type = self.show.air_by_date and 'Air By Date' \ @@ -70,7 +94,7 @@ class PrivateHDProvider(generic.TorrentProvider): for search_string in search_params[mode]: search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string search_url = self.urls['search'] % ( - '+'.join(search_string.split()), self._categories_string(mode, ''), (1, 0)[not self.freeleech]) + '+'.join(search_string.split()), self._categories_string(mode, '')) html = self.get_url(search_url) @@ -87,6 +111,15 @@ class PrivateHDProvider(generic.TorrentProvider): raise generic.HaltParseException for tr in torrent_rows[1:]: + if self.confirmed and tr.find('i', title=re.compile('(?i)unverified')): + continue + if any(self.filter): + marked = ','.join([x.attrs.get('title', '').lower() for x in tr.find_all( + 'i', attrs={'class': ['fa-star', 'fa-diamond', 'fa-star-half-o']})]) + munged = ''.join(filter(marked.__contains__, ['free', 'half', 'double'])) + if ((non_marked and rc['filter'].search(munged)) or + (not non_marked and not rc['filter'].search(munged))): + continue try: seeders, leechers, size = [tryInt(n, n) for n in [ tr.find_all('td')[x].get_text().strip() for x in -3, -2, -4]] @@ -106,7 +139,7 @@ class PrivateHDProvider(generic.TorrentProvider): except (StandardError, Exception): logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - self._log_search(mode, len(items[mode]) - cnt, search_url) + self._log_search(mode, len(items[mode]) - cnt, log + search_url) results = self._sort_seeding(mode, results + items[mode]) diff --git a/sickbeard/providers/ptf.py b/sickbeard/providers/ptf.py index f48e6f05..6d51ac8d 100644 --- a/sickbeard/providers/ptf.py +++ b/sickbeard/providers/ptf.py @@ -15,6 +15,10 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . +try: + from collections import OrderedDict +except ImportError: + from requests.compat import OrderedDict import re import time import traceback @@ -22,7 +26,7 @@ import traceback from . import generic from sickbeard import logger from sickbeard.bs4_parser import BS4Parser -from sickbeard.helpers import tryInt +from sickbeard.helpers import tryInt, anon_url from lib.unidecode import unidecode @@ -33,9 +37,8 @@ class PTFProvider(generic.TorrentProvider): self.url_base = 'https://ptfiles.net/' self.urls = {'config_provider_home_uri': self.url_base, - 'login_action': self.url_base + 'loginproc/', - 'login_base': self.url_base + 'loginproc/', - 'search': self.url_base + 'browse.php?search=%s&%s&incldead=0&title=0%s', + 'login': self.url_base + 'panel.php?tool=links', + 'search': self.url_base + 'browse.php?search=%s&%s&incldead=0&title=0', 'get': self.url_base + '%s'} self.categories = {'Season': [39], 'Episode': [7, 33, 42], 'anime': [23]} @@ -43,12 +46,19 @@ class PTFProvider(generic.TorrentProvider): self.url = self.urls['config_provider_home_uri'] - self.username, self.password, self.freeleech, self.minseed, self.minleech = 5 * [None] + self.filter = [] + self.may_filter = OrderedDict([ + ('f0', ('not marked', False, '')), ('free', ('free', True, '^free$')), + ('freeday', ('free day', True, '^free[^!]+day')), ('freeweek', ('free week', True, '^free[^!]+week'))]) + self.digest, self.minseed, self.minleech = 3 * [None] def _authorised(self, **kwargs): - return super(PTFProvider, self)._authorised(logged_in=(lambda y=None: self.has_all_cookies('session_key')), - post_params={'force_ssl': 'on', 'ssl': '', 'form_tmpl': True}) + return super(PTFProvider, self)._authorised( + logged_in=(lambda y='': all( + ['RSS Feed' in y, self.has_all_cookies('session_key')] + + [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in ['session_key']])), + failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) def _search_provider(self, search_params, **kwargs): @@ -60,13 +70,21 @@ class PTFProvider(generic.TorrentProvider): rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'details', 'get': 'dl.php', 'snatch': 'snatches', 'seeders': r'(^\d+)', 'leechers': r'(\d+)$'}.items()) + log = '' + if self.filter: + non_marked = 'f0' in self.filter + # if search_any, use unselected to exclude, else use selected to keep + filters = ([f for f in self.may_filter if f in self.filter], + [f for f in self.may_filter if f not in self.filter])[non_marked] + rc['filter'] = re.compile('(?i)(%s)' % '|'.join( + [self.may_filter[f][2] for f in filters if self.may_filter[f][1]])) + log = '%sing (%s) ' % (('keep', 'skipp')[non_marked], ', '.join([self.may_filter[f][0] for f in filters])) for mode in search_params.keys(): rc['cats'] = re.compile('(?i)cat=(?:%s)' % self._categories_string(mode, template='', delimiter='|')) for search_string in search_params[mode]: search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string - search_url = self.urls['search'] % ('+'.join(search_string.split()), self._categories_string(mode), - ('&free=1', '')[not self.freeleech]) + search_url = self.urls['search'] % ('+'.join(search_string.split()), self._categories_string(mode)) html = self.get_url(search_url) time.sleep(2) if not self.has_all_cookies(['session_key']): @@ -87,6 +105,15 @@ class PTFProvider(generic.TorrentProvider): raise generic.HaltParseException for tr in torrent_rows[1:]: + if any(self.filter): + marker = '' + try: + marker = tr.select('a[href^="browse"] .tip')[0].get_text().strip() + except (StandardError, Exception): + pass + if ((non_marked and rc['filter'].search(marker)) or + (not non_marked and not rc['filter'].search(marker))): + continue try: seeders, leechers = 2 * [tr.find_all('td')[-2].get_text().strip()] seeders, leechers = [tryInt(n) for n in [ @@ -110,11 +137,19 @@ class PTFProvider(generic.TorrentProvider): except (StandardError, Exception): logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - self._log_search(mode, len(items[mode]) - cnt, self.session.response.get('url')) + self._log_search(mode, len(items[mode]) - cnt, log + self.session.response.get('url')) results = self._sort_seeding(mode, results + items[mode]) return results + def ui_string(self, key): + if 'ptfiles_digest' == key and self._valid_home(): + current_url = getattr(self, 'urls', {}).get('config_provider_home_uri') + return ('use... \'session_key=xx\'' + + (current_url and (' from a session logged in at %s' % + (anon_url(current_url), current_url.strip('/'))) or '')) + return '' + provider = PTFProvider() diff --git a/sickbeard/providers/scenetime.py b/sickbeard/providers/scenetime.py index b202e822..82f6a6e7 100644 --- a/sickbeard/providers/scenetime.py +++ b/sickbeard/providers/scenetime.py @@ -22,7 +22,7 @@ import traceback from . import generic from sickbeard import logger from sickbeard.bs4_parser import BS4Parser -from sickbeard.helpers import tryInt +from sickbeard.helpers import tryInt, anon_url from lib.unidecode import unidecode @@ -31,22 +31,27 @@ class SceneTimeProvider(generic.TorrentProvider): def __init__(self): generic.TorrentProvider.__init__(self, 'SceneTime', cache_update_freq=15) - self.url_base = 'https://www.scenetime.com/' - self.urls = {'config_provider_home_uri': self.url_base, - 'login_action': self.url_base + 'login.php', - 'browse': self.url_base + 'browse_API.php', - 'params': {'sec': 'jax', 'cata': 'yes'}, - 'get': self.url_base + 'download.php/%(id)s/%(title)s.torrent'} + self.url_home = ['https://%s.scenetime.com/' % u for u in 'www', 'uk'] + + self.url_vars = {'login': 'support.php', 'browse': 'browse_API.php', 'get': 'download.php/%s.torrent'} + self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s', + 'browse': '%(home)s%(vars)s', 'get': '%(home)s%(vars)s'} self.categories = {'shows': [2, 43, 9, 63, 77, 79, 83]} - self.url = self.urls['config_provider_home_uri'] - - self.username, self.password, self.freeleech, self.minseed, self.minleech = 5 * [None] + self.digest, self.freeleech, self.minseed, self.minleech = 4 * [None] def _authorised(self, **kwargs): - return super(SceneTimeProvider, self)._authorised(post_params={'form_tmpl': True}) + return super(SceneTimeProvider, self)._authorised( + logged_in=(lambda y='': all( + ['staff-support' in y, self.has_all_cookies()] + + [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in 'uid', 'pass'])), + failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) + + @staticmethod + def _has_signature(data=None): + return generic.TorrentProvider._has_signature(data) or (data and re.search(r'(?i)%s' % + (anon_url(current_url), current_url.strip('/'))) or '')) + return '' + provider = SceneTimeProvider() diff --git a/sickbeard/providers/shazbat.py b/sickbeard/providers/shazbat.py index c8b6c433..bcdf1876 100644 --- a/sickbeard/providers/shazbat.py +++ b/sickbeard/providers/shazbat.py @@ -51,8 +51,7 @@ class ShazbatProvider(generic.TorrentProvider): return super(ShazbatProvider, self)._authorised( logged_in=(lambda y=None: '