Add ETTV and PotUK providers, fix BTScene and Lime.

CHANGES.md

@@ -1,4 +1,13 @@
-### 0.13.6 (2017-12-13 01:50:00 UTC)
+### 0.14.0 (2018-xx-xx xx:xx:xx UTC)
+
+* Add ETTV torrent provider
+* Add PotUK torrent provider
+* Fix BTScene and Lime
+
+[develop changelog]
+
+
+### 0.13.6 (2017-12-13 01:50:00 UTC)
 
 * Change improve multi episode release search
 * Change improve usage of the optional regex library

sickbeard/providers/__init__.py

@@ -26,9 +26,9 @@ from sickbeard import logger, encodingKludge as ek
 # usenet
 from . import newznab, omgwtfnzbs
 # torrent
-from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, \
+from . import alpharatio, beyondhd, bithdtv, bitmetv, blutopia, btn, btscene, dh, ettv, \
     fano, filelist, funfile, gftracker, grabtheinfo, hd4free, hdbits, hdspace, hdtorrents, \
-    iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, pretome, privatehd, ptf, \
+    iptorrents, limetorrents, magnetdl, morethan, nebulance, ncore, nyaa, pisexy, potuk, pretome, privatehd, ptf, \
     rarbg, revtt, scenehd, scenetime, shazbat, skytorrents, speedcd, \
     thepiratebay, torlock, torrentbytes, torrentday, torrenting, torrentleech, \
     torrentvault, torrentz2, tvchaosuk, wop, zooqle
@@ -51,6 +51,7 @@ __all__ = ['omgwtfnzbs',
            'btscene',
            'custom01',
            'dh',
+           'ettv',
            'fano',
            'filelist',
            'funfile',
@@ -68,6 +69,7 @@ __all__ = ['omgwtfnzbs',
            'ncore',
            'nyaa',
            'pisexy',
+           'potuk',
            'pretome',
            'privatehd',
            'ptf',

sickbeard/providers/btscene.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with SickGear.  If not, see <http://www.gnu.org/licenses/>.
 
+import base64
 import re
 import traceback
 import urllib
@@ -31,8 +32,13 @@ class BTSceneProvider(generic.TorrentProvider):
     def __init__(self):
         generic.TorrentProvider.__init__(self, 'BTScene')
 
-        self.url_home = ['http://btsone.cc/',  'http://diriri.xyz/', 'http://mytorrentz.tv/']
-
+        self.url_home = ['http://btsone.cc/',  'http://diriri.xyz/'] + \
+                        ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [
+                            [re.sub('[L\sT]+', '', x[::-1]) for x in [
+                                'zTRnTY', 'uVT 2Y', '15LSTZ', 's JmLb', 'rTNL2b', 'uQW LZ', '=LLMmd']],
+                            [re.sub('[j\sq]+', '', x[::-1]) for x in [
+                                'zRn qY', 'l52j b', '1j5S M', 'sq Jmb', 'r Nq2b', 'ujQWqZ', 's9jGqb']],
+                        ]]]
         self.url_vars = {'search': '?q=%s&category=series&order=1', 'browse': 'lastdaycat/type/Series/',
                          'get': 'torrentdownload.php?id=%s'}
         self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(vars)s',

sickbeard/providers/ettv.py

@@ -0,0 +1,129 @@
+# coding=utf-8
+#
+# This file is part of SickGear.
+#
+# SickGear is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# SickGear is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with SickGear.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import traceback
+
+from . import generic
+from sickbeard import logger
+from sickbeard.bs4_parser import BS4Parser
+from sickbeard.helpers import tryInt
+from lib.unidecode import unidecode
+
+
+class ETTVProvider(generic.TorrentProvider):
+
+    def __init__(self):
+        generic.TorrentProvider.__init__(self, 'ETTV')
+
+        self.url_home = ['https://ettv.tv/']
+        self.url_vars = {'search': 'torrents-search.php?%s&search=%s&sort=id&order=desc'}
+        self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s'}
+
+        self.categories = {'Season': [7], 'Episode': [41, 5, 50]}
+        self.categories['Cache'] = self.categories['Season'] + self.categories['Episode']
+
+        self.minseed, self.minleech = 2 * [None]
+
+    @staticmethod
+    def _has_signature(data=None):
+        return data and re.search(r'(?i)(?:ettv)', data)
+
+    def _search_provider(self, search_params, **kwargs):
+
+        results = []
+        if not self.url:
+            return results
+
+        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+
+        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'torrent/'}.iteritems())
+
+        for mode in search_params.keys():
+            for search_string in search_params[mode]:
+
+                search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
+
+                search_url = self.urls['search'] % (
+                    self._categories_string(mode), ('%2B ', '')['Cache' == mode] + '.'.join(search_string.split()))
+
+                html = self.get_url(search_url)
+
+                cnt = len(items[mode])
+                try:
+                    if not html or self._has_no_results(html):
+                        raise generic.HaltParseException
+                    with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
+                        torrent_table = soup.find('table', class_='table')
+                        torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
+
+                        if not len(torrent_rows):
+                            raise generic.HaltParseException
+
+                        head = None
+                        for tr in torrent_rows[1:]:
+                            cells = tr.find_all('td')
+                            if 6 > len(cells):
+                                continue
+                            try:
+                                head = head if None is not head else self._header_row(
+                                    tr, {'seed': r'seed', 'leech': r'leech', 'size': r'^size'})
+                                seeders, leechers, size = [tryInt(n, n) for n in [
+                                    cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']]
+                                if self._peers_fail(mode, seeders, leechers):
+                                    continue
+
+                                info = tr.find('a', href=rc['info'])
+                                title = (info.attrs.get('title') or info.get_text()).strip()
+                                download_url = self._link(info.get('href'))
+                            except (AttributeError, TypeError, ValueError, IndexError):
+                                continue
+
+                            if title and download_url:
+                                items[mode].append((title, download_url, seeders, self._bytesizer(size)))
+
+                except generic.HaltParseException:
+                    pass
+                except (StandardError, Exception):
+                    logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
+
+                self._log_search(mode, len(items[mode]) - cnt, search_url)
+
+            results = self._sort_seeding(mode, results + items[mode])
+
+        return results
+
+    def get_data(self, url):
+        result = None
+        html = self.get_url(url, timeout=90)
+        try:
+            result = re.findall('(?i)"(magnet:[^"]+?)">', html)[0]
+        except IndexError:
+            logger.log('Failed no magnet in response', logger.DEBUG)
+        return result
+
+    def get_result(self, episodes, url):
+        result = None
+
+        if url:
+            result = super(ETTVProvider, self).get_result(episodes, url)
+            result.get_data_func = self.get_data
+
+        return result
+
+
+provider = ETTVProvider()

sickbeard/providers/limetorrents.py

@@ -34,18 +34,8 @@ class LimeTorrentsProvider(generic.TorrentProvider):
 
         self.url_home = ['https://www.limetorrents.cc/'] + \
                         ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [
-                            [re.sub('[f\sX]+', '', x[::-1]) for x in [
-                                'tlXGfb', '1X5SfZ', 'sfJfmb', 'rN 2Xb', 'u QfWZ', 's9G  b']],
-                            [re.sub('[ \ss]+', '', x[::-1]) for x in [
-                                'Ztl Gsb', 'nc svRX', 'Rs nblJ', '5  JmLz', 'czsFsGc', 'nLskVs2', '0s N']],
-                            [re.sub('[1\sF]+', '', x[::-1]) for x in [
-                                'X Zt1lGb', 'l1Jn1cvR', 'mL11zRnb', 'uVXbtFFl', 'Hdp NWFa', '=1FQ3cuk']],
-                            [re.sub('[y\sW]+', '', x[::-1]) for x in [
-                                'XWZtlyGb', 'lJnyWcvR', 'nyLzRn b', 'vxmYWuWV', 'CWZlt2yY', '== Adyz5']],
-                            [re.sub('[j\sy]+', '', x[::-1]) for x in [
-                                'XyZtlG b', 'lJjnjcvR', 'njLz Rnb', 'vjxmYyuV', 'Gbyhjt2Y', 'n jJ3buw']],
-                            [re.sub('[o\sg]+', '', x[::-1]) for x in [
-                                'XZt lgGb', 'loJn cvR', 'ngLz Rnb', 'v xgmYuV', 'Gbh t2gY', '6l Heu w']],
+                            [re.sub('[ \sF]+', '', x[::-1]) for x in [
+                                'X ZtlFGb', 'lJnc  vR', 'n LzR nb', 'vxmYuF V', 'CFZltF2Y', '==wYF2F5']],
                         ]]]
 
         self.url_vars = {'search': 'search/tv/%s/', 'browse': 'browse-torrents/TV-shows/'}

sickbeard/providers/potuk.py

@@ -0,0 +1,157 @@
+# coding=utf-8
+#
+# This file is part of SickGear.
+#
+# SickGear is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# SickGear is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with SickGear.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import traceback
+
+from . import generic
+from sickbeard import logger
+from sickbeard.bs4_parser import BS4Parser
+from sickbeard.helpers import tryInt
+from lib.unidecode import unidecode
+
+
+class PotUKProvider(generic.TorrentProvider):
+
+    def __init__(self):
+        generic.TorrentProvider.__init__(self, 'PotUK')
+
+        self.url_base = 'http://www.potuk.com/newforum/'
+        self.urls = {'config_provider_home_uri': self.url_base,
+                     'login': self.url_base + 'search.php',
+                     'browse': self.url_base + 'search.php?do=getdaily&exclude=%s',
+                     'get_data': self.url_base + 'misc.php?do=showattachments&t=%s'}
+
+        self.url = self.urls['config_provider_home_uri']
+
+        self.digest, self.resp = 2 * [None]
+
+    def logged_in(self, resp):
+        try:
+            self.resp = re.findall('(?sim)<form .*?search.php.*?</form>', resp)[0]
+        except (IndexError, TypeError):
+            return False
+        return self.has_all_cookies('bbsessionhash')
+
+    def _authorised(self, **kwargs):
+
+        return super(PotUKProvider, self)._authorised(
+            logged_in=(lambda y=None: self.logged_in(y)),
+            failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings'))
+
+    def _search_provider(self, search_params, **kwargs):
+
+        results = []
+        if not self._authorised():
+            return results
+
+        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+
+        opts = re.findall('(?sim)forumchoice\[\][^<]+(.*?)</select>', self.resp)[0]
+        cat_opts = re.findall(r'(?mis)<option[^>]*?value=[\'"](\d+)[^>]*>(.*?)</option>', opts)
+        include = []
+        tv = False
+        for c in cat_opts:
+            if not tv and 'TV Shows' in c[1]:
+                tv = True
+            elif tv:
+                if 3 > len(re.findall('&nbsp;', c[1])):
+                    break
+                elif not filter(lambda v: v in c[1], ('Requests', 'Offer', 'Discussion')):
+                    include += [c[0]]
+        exclude = ','.join(list(filter(lambda v: v not in include, map(lambda x: x[0], cat_opts))))
+
+        for mode in search_params.keys():
+            for search_string in search_params[mode]:
+                search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
+
+                params = {}
+                if 'Cache' == mode:
+                    search_url = self.urls['browse'] % exclude
+                else:
+                    search_url = self._link(re.findall('(?i)action="([^"]+?)"', self.resp)[0])
+                    params = {'query': search_string, 'showposts': 0, 'titleonly': 1, 'prefixchoice': '',
+                              'replyless': 0, 'searchdate': 0, 'beforeafter': 'after', 'sortby': 'threadstart',
+                              'order': 'descending', 'starteronly': 0, 'forumchoice': include}
+                    tags = re.findall(r'(?is)(<input[^>]*?name=[\'"][^\'"]+[^>]*)', self.resp)
+                    attrs = [[(re.findall(r'(?is)%s=[\'"]([^\'"]+)' % attr, c) or [''])[0]
+                              for attr in ['type', 'name', 'value']] for c in tags]
+                    for itype, name, value in attrs:
+                        params.setdefault(name, value)
+                    del params['doprefs']
+                html = self.get_url(search_url, post_data=params)
+
+                cnt = len(items[mode])
+                try:
+                    if not html or self._has_no_results(html):
+                        raise generic.HaltParseException
+
+                    with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
+                        torrent_table = soup.find('table', id='threadslist')
+                        torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
+
+                        if 2 > len(torrent_rows):
+                            raise generic.HaltParseException
+
+                        for tr in torrent_rows[1:]:
+                            if 6 > len(tr.find_all('td')) or not tr.select('img[alt*="ttach"]'):
+                                continue
+                            try:
+                                link = tr.select('td[id^="td_threadtitle"]')[0].select('a[id*="title"]')[0]
+                                title = link.get_text().strip()
+                                download_url = self.urls['get_data'] % re.findall('t=(\d+)', link['href'])[0]
+                            except (AttributeError, TypeError, ValueError, IndexError):
+                                continue
+
+                            if title and download_url:
+                                items[mode].append((title, download_url, '', ''))
+
+                except generic.HaltParseException:
+                    pass
+                except (StandardError, Exception):
+                    logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
+
+                self._log_search(
+                    mode, len(items[mode]) - cnt, ('search_param: ' + search_string, search_url)['Cache' == mode])
+
+            results = self._sort_seeding(mode, results + items[mode])
+
+        return results
+
+    def get_data(self, url):
+        result = None
+        html = self.get_url(url, timeout=90)
+        try:
+            result = self._link(re.findall('(?i)"(attachment\.php[^"]+?)"', html)[0])
+        except IndexError:
+            logger.log('Failed no torrent in response', logger.DEBUG)
+        return result
+
+    def get_result(self, episodes, url):
+        result = None
+
+        if url:
+            result = super(PotUKProvider, self).get_result(episodes, url)
+            result.get_data_func = self.get_data
+
+        return result
+
+    def ui_string(self, key):
+        return ('%s_digest' % self.get_id()) == key and 'use... \'bbuserid=xx; bbpassword=yy\'' or ''
+
+
+provider = PotUKProvider()

sickbeard/search.py

@@ -130,6 +130,11 @@ def snatch_episode(result, end_status=SNATCHED):
 
     # TORRENTs can be sent to clients or saved to disk
     elif 'torrent' == result.resultType:
+        if not result.url.startswith('magnet') and None is not result.get_data_func:
+            result.url = result.get_data_func(result.url)
+            result.get_data_func = None  # consume only once
+            if not result.url:
+                return False
         # torrents are saved to disk when blackhole mode
         if 'blackhole' == sickbeard.TORRENT_METHOD:
             dl_result = _download_result(result)
@@ -739,6 +744,11 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
 
             # filter out possible bad torrents from providers
             if 'torrent' == best_result.resultType:
+                if not best_result.url.startswith('magnet') and None is not best_result.get_data_func:
+                    best_result.url = best_result.get_data_func(best_result.url)
+                    best_result.get_data_func = None  # consume only once
+                    if not best_result.url:
+                        continue
                 if best_result.url.startswith('magnet'):
                     if 'blackhole' != sickbeard.TORRENT_METHOD:
                         best_result.content = None