From 48032005fd67a580de9ee3acc7fa1ca92fca6991 Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Mon, 24 Oct 2016 02:28:22 +0200 Subject: [PATCH 1/2] Change image cache processing for browse Trakt page. Change logging for newznab response code 910 and add check for empty response data. --- sickbeard/providers/newznab.py | 10 +++- sickbeard/webserve.py | 94 +++++++++++++++++++++++++++++----- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index 82969e5f..137c2494 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -271,6 +271,7 @@ class NewznabProvider(generic.NZBProvider): if 'error' == data.tag: code = data.get('code', '') + description = data.get('description', '') if '100' == code: raise AuthException('Your API key for %s is incorrect, check your config.' % self.name) @@ -279,7 +280,8 @@ class NewznabProvider(generic.NZBProvider): elif '102' == code: raise AuthException('Your account isn\'t allowed to use the API on %s, contact the admin.' % self.name) elif '910' == code: - logger.log('%s currently has their API disabled, please check with provider.' % self.name, + logger.log('%s %s, please check with provider.' % + (self.name, ('currently has their API disabled', description)[description not in (None, '')]), logger.WARNING) else: logger.log('Unknown error given from %s: %s' % (self.name, data.get('description', '')), @@ -624,6 +626,10 @@ class NewznabProvider(generic.NZBProvider): data = helpers.getURL(search_url) + if not data: + logger.log('No Data returned from %s' % self.name, logger.DEBUG) + break + # hack this in until it's fixed server side if data and not data.startswith('%s' % data @@ -661,7 +667,7 @@ class NewznabProvider(generic.NZBProvider): hits = (total // self.limits + int(0 < (total % self.limits))) hits += int(0 == hits) offset = helpers.tryInt(parsed_xml.find('.//%sresponse' % n_spaces['newznab']).get('offset', 0)) - except AttributeError: + except (AttributeError, KeyError): break # No items found, prevent from doing another search diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 85fad43b..c6cc08c9 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -67,7 +67,7 @@ from lib.libtrakt import TraktAPI from lib.libtrakt.exceptions import TraktException, TraktAuthException from trakt_helpers import build_config, trakt_collection_remove_account from sickbeard.bs4_parser import BS4Parser - +from lib.tmdb_api import TMDB try: import json @@ -3052,10 +3052,11 @@ class NewHomeAddShows(Home): newest_dt = dt_ordinal newest = dt_string - img_uri = item.get('show', {}).get('images', {}).get('poster', {}).get('thumb', {}) or '' - if img_uri: - images = dict(poster=dict(thumb='imagecache?path=trakt/poster/thumb&source=%s' % img_uri)) - sickbeard.CACHE_IMAGE_URL_LIST.add_url(img_uri) + tmdbid = item.get('show', {}).get('ids', {}).get('tmdb', 0) + tvdbid = item.get('show', {}).get('ids', {}).get('tvdb', 0) + traktid = item.get('show', {}).get('ids', {}).get('trakt', 0) + images = dict(poster=dict(thumb='imagecache?path=trakt/poster/thumb&filename=%s&tmdbid=%s&tvdbid=%s' % + ('%s.jpg' % traktid, tmdbid, tvdbid))) filtered.append(dict( premiered=dt_ordinal, @@ -3068,7 +3069,7 @@ class NewHomeAddShows(Home): genres=('' if 'genres' not in item['show'] else ', '.join(['%s' % v for v in item['show']['genres']])), ids=item['show']['ids'], - images='' if not img_uri else images, + images=images, overview=('' if 'overview' not in item['show'] or None is item['show']['overview'] else self.encode_html(item['show']['overview'][:250:].strip())), rating=0 < item['show'].get('rating', 0) and @@ -5603,18 +5604,87 @@ class Cache(MainHandler): class CachedImages(MainHandler): - def index(self, path='', source=None, *args, **kwargs): + @staticmethod + def should_try_image(filename, source, days=1, minutes=0): + try: + dummy_file = '%s.%s.dummy' % (ek.ek(os.path.splitext, filename)[0], source) + if ek.ek(os.path.isfile, dummy_file): + if ek.ek(os.stat, dummy_file).st_mtime < time.mktime((datetime.datetime.now() - datetime.timedelta(days=days, minutes=minutes)).timetuple()): + CachedImages.delete_dummy_image(dummy_file) + return True + return False + except: + pass + return True + + @staticmethod + def create_dummy_image(filename, source): + dummy_file = '%s.%s.dummy' % (ek.ek(os.path.splitext, filename)[0], source) + CachedImages.delete_dummy_image(dummy_file) + try: + with open(dummy_file, 'w'): + pass + except: + pass + + @staticmethod + def delete_dummy_image(dummy_file): + try: + if ek.ek(os.path.isfile, dummy_file): + ek.ek(os.remove, dummy_file) + except: + pass + + @staticmethod + def delete_all_dummy_images(filename): + for f in ['tmdb', 'tvdb']: + CachedImages.delete_dummy_image('%s.%s.dummy' % (ek.ek(os.path.splitext, filename)[0], f)) + + def index(self, path='', source=None, filename=None, tmdbid=None, tvdbid=None, *args, **kwargs): path = path.strip('/') - file_name = ek.ek(os.path.basename, source) + file_name = '' + if None is not source: + file_name = ek.ek(os.path.basename, source) + elif filename not in [None, 0, '0']: + file_name = filename static_image_path = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images', path, file_name) static_image_path = ek.ek(os.path.abspath, static_image_path.replace('\\', '/')) - if not ek.ek(os.path.isfile, static_image_path) and source is not None and has_image_ext(file_name) \ - and source in sickbeard.CACHE_IMAGE_URL_LIST: + if not ek.ek(os.path.isfile, static_image_path) and has_image_ext(file_name): basepath = ek.ek(os.path.dirname, static_image_path) helpers.make_dirs(basepath) - if not helpers.download_file(source, static_image_path) and source.find('trakt.us'): - helpers.download_file(source.replace('trakt.us', 'trakt.tv'), static_image_path) + s = '' + tmdbimage = False + if source is not None and source in sickbeard.CACHE_IMAGE_URL_LIST: + s = source + if source is None and tmdbid not in [None, 0, '0'] and self.should_try_image(static_image_path, 'tmdb'): + tmdbimage = True + try: + tmdbapi = TMDB(sickbeard.TMDB_API_KEY) + tmdbconfig = tmdbapi.Configuration().info() + images = tmdbapi.TV(helpers.tryInt(tmdbid)).images() + s = '%s%s%s' % (tmdbconfig['images']['base_url'], tmdbconfig['images']['poster_sizes'][3], sorted(images['posters'], key=lambda x: x['vote_average'], reverse=True)[0]['file_path']) if len(images['posters']) > 0 else '' + except: + s = '' + if s and not helpers.download_file(s, static_image_path) and s.find('trakt.us'): + helpers.download_file(s.replace('trakt.us', 'trakt.tv'), static_image_path) + if tmdbimage and not ek.ek(os.path.isfile, static_image_path): + self.create_dummy_image(static_image_path, 'tmdb') + + if source is None and tvdbid not in [None, 0, '0'] and not ek.ek(os.path.isfile, static_image_path) and self.should_try_image(static_image_path, 'tvdb'): + try: + r = sickbeard.indexerApi(INDEXER_TVDB).indexer()[helpers.tryInt(tvdbid), False] + if hasattr(r, 'data') and 'poster' in r.data: + s = r.data['poster'] + except: + s = '' + if s: + helpers.download_file(s, static_image_path) + if not ek.ek(os.path.isfile, static_image_path): + self.create_dummy_image(static_image_path, 'tvdb') + + if ek.ek(os.path.isfile, static_image_path): + self.delete_all_dummy_images(static_image_path) if not ek.ek(os.path.isfile, static_image_path): self.redirect('images/trans.png') From 9d8f397d41547ee0de50941ec9763b9989b62475 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Tue, 25 Oct 2016 00:04:02 +0100 Subject: [PATCH 2/2] Pep8 and tweaks to newznab logging. --- CHANGES.md | 2 ++ sickbeard/processTV.py | 2 +- sickbeard/providers/generic.py | 2 +- sickbeard/providers/iptorrents.py | 2 +- sickbeard/providers/newznab.py | 54 +++++++++++++++---------------- sickbeard/providers/omgwtfnzbs.py | 8 ++--- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ee7400e7..234311c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -210,6 +210,8 @@ * Change make sure avi files are closed if an error occurs * Change refactor hachoir scan_index for non RIFF file processing * Change ensure sbdatetime functions return formatted string instead of tuple +* Change image cache processing for browse Trakt page +* Change logging for newznab response code 910 and add check for empty response data ### 0.11.16 (2016-10-16 17:30:00 UTC) diff --git a/sickbeard/processTV.py b/sickbeard/processTV.py index bfdb4f45..21daa31e 100644 --- a/sickbeard/processTV.py +++ b/sickbeard/processTV.py @@ -193,7 +193,7 @@ class ProcessTVShow(object): self._log_helper(u'Unable to figure out what folder to process. ' + u'If your downloader and SickGear aren\'t on the same PC then make sure ' + u'you fill out your completed TV download folder in the PP config.') - return self.result + return self.result path, dirs, files = self._get_path_dir_files(dir_name, nzb_name, pp_type) diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index 2edad3a6..fa8f72c4 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -904,7 +904,7 @@ class TorrentProvider(object, GenericProvider): self.urls[k] = v % {'home': cur_url, 'vars': getattr(self, 'url_vars', {}).get(k, '')} if last_url != cur_url or (expire and not (expire > int(time.time()))): - sickbeard.PROVIDER_HOMES[self.get_id()] = (cur_url, int(time.time()) + (15*60)) + sickbeard.PROVIDER_HOMES[self.get_id()] = (cur_url, int(time.time()) + (60*60)) sickbeard.save_config() return cur_url diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py index 2434c3fa..f1068b8d 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/iptorrents.py @@ -31,7 +31,7 @@ class IPTorrentsProvider(generic.TorrentProvider): generic.TorrentProvider.__init__(self, 'IPTorrents') self.url_home = (['https://iptorrents.%s/' % u for u in 'eu', 'com', 'me', 'ru'] + - ['http://11111.workisboring.com/']) + ['http://11111.workisboring.com/', 'https://ipt-update.com']) self.url_vars = {'login': 'getrss.php', 'search': 't?%s;q=%s;qf=ti%s%s#torrents', 'get': '%s'} self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s', diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index 137c2494..1a560e60 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -145,10 +145,10 @@ class NewznabProvider(generic.NZBProvider): if not self._last_recent_search: try: my_db = db.DBConnection('cache.db') - res = my_db.select('SELECT "datetime" FROM "lastrecentsearch" WHERE "name"=?', [self.get_id()]) + res = my_db.select('SELECT' + ' "datetime" FROM "lastrecentsearch" WHERE "name"=?', [self.get_id()]) if res: self._last_recent_search = datetime.datetime.fromtimestamp(int(res[0]['datetime'])) - except: + except (StandardError, Exception): pass return self._last_recent_search @@ -157,8 +157,8 @@ class NewznabProvider(generic.NZBProvider): try: my_db = db.DBConnection('cache.db') my_db.action('INSERT OR REPLACE INTO "lastrecentsearch" (name, datetime) VALUES (?,?)', - [self.get_id(), sbdatetime.totimestamp(value, default=0)]) - except: + [self.get_id(), sbdatetime.totimestamp(value, default=0)]) + except (StandardError, Exception): pass self._last_recent_search = value @@ -212,7 +212,7 @@ class NewznabProvider(generic.NZBProvider): for s, v in NewznabConstants.catSearchStrings.iteritems(): if None is not re.search(s, cat_name, re.IGNORECASE): cats.setdefault(v, []).append(cat_id) - except: + except (StandardError, Exception): continue elif category.get('name', '').upper() in ['XXX', 'OTHER', 'MISC']: for subcat in category.findall('subcat'): @@ -220,9 +220,9 @@ class NewznabProvider(generic.NZBProvider): if None is not re.search(r'^Anime$', subcat.attrib['name'], re.IGNORECASE): cats.setdefault(NewznabConstants.CAT_ANIME, []).append(subcat.attrib['id']) break - except: + except (StandardError, Exception): continue - except: + except (StandardError, Exception): logger.log('Error parsing result for [%s]' % self.name, logger.DEBUG) if not caps and self._caps and not all_cats and self._caps_all_cats and not cats and self._caps_cats: @@ -280,9 +280,10 @@ class NewznabProvider(generic.NZBProvider): elif '102' == code: raise AuthException('Your account isn\'t allowed to use the API on %s, contact the admin.' % self.name) elif '910' == code: - logger.log('%s %s, please check with provider.' % - (self.name, ('currently has their API disabled', description)[description not in (None, '')]), - logger.WARNING) + logger.log( + '%s %s, please check with provider.' % + (self.name, ('currently has their API disabled', description)[description not in (None, '')]), + logger.WARNING) else: logger.log('Unknown error given from %s: %s' % (self.name, data.get('description', '')), logger.ERROR) @@ -412,7 +413,7 @@ class NewznabProvider(generic.NZBProvider): try: title = item.findtext('title') url = item.findtext('link') - except Exception: + except (StandardError, Exception): pass title = title and re.sub(r'\s+', '.', '%s' % title) @@ -544,11 +545,11 @@ class NewznabProvider(generic.NZBProvider): p = parser.parse(p, fuzzy=True) try: p = p.astimezone(sb_timezone) - except: + except (StandardError, Exception): pass if isinstance(p, datetime.datetime): parsed_date = p.replace(tzinfo=None) - except: + except (StandardError, Exception): pass return parsed_date @@ -569,7 +570,7 @@ class NewznabProvider(generic.NZBProvider): base_params['apikey'] = api_key results, n_spaces = [], {} - total, cnt, search_url, exit_log = 0, len(results), '', False + total, cnt, search_url, exit_log = 0, len(results), '', True cat_sport = self.cats.get(NewznabConstants.CAT_SPORT, ['5060']) cat_anime = self.cats.get(NewznabConstants.CAT_ANIME, ['5070']) @@ -584,9 +585,9 @@ class NewznabProvider(generic.NZBProvider): cat = [] if 'Episode' == mode or 'Season' == mode: if not (any(x in params for x in [v for c, v in self.caps.iteritems() - if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON]]) or - not self.supports_tvdbid()): - logger.log('Error no id or search term available for search.') + if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON]]) + or not self.supports_tvdbid()): + logger.log('Show is missing either an id or search term for search') continue if need_anime: @@ -637,7 +638,7 @@ class NewznabProvider(generic.NZBProvider): try: parsed_xml, n_spaces = self.cache.parse_and_get_ns(data) items = parsed_xml.findall('channel/item') - except Exception: + except (StandardError, Exception): logger.log('Error trying to load %s RSS feed' % self.name, logger.ERROR) break @@ -682,7 +683,6 @@ class NewznabProvider(generic.NZBProvider): last_date = self._parse_pub_date(items[-1]) if not first_date or not last_date or not self._last_recent_search or \ last_date <= self.last_recent_search: - exit_log = True break if offset != request_params['offset']: @@ -691,9 +691,6 @@ class NewznabProvider(generic.NZBProvider): request_params['offset'] += request_params['limit'] if total <= request_params['offset']: - exit_log = True - logger.log('%s item%s found for episode matching' % (total, helpers.maybe_plural(total)), - logger.DEBUG) break # there are more items available than the amount given in one call, grab some more @@ -702,17 +699,18 @@ class NewznabProvider(generic.NZBProvider): % (items, helpers.maybe_plural(items), request_params['limit']), logger.DEBUG) batch_count = self._log_result(results, mode, cnt, search_url) + exit_log = False if 'Cache' == mode and first_date: self.last_recent_search = first_date if exit_log: - self._log_result(results, mode, cnt, search_url) - exit_log = False + self._log_search(mode, total, search_url) - if not try_all_searches and any(x in request_params for x in [v for c, v in self.caps.iteritems() - if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON, - NewznabConstants.SEARCH_TEXT]]) and len(results): + if not try_all_searches and any(x in request_params for x in [ + v for c, v in self.caps.iteritems() + if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON, + NewznabConstants.SEARCH_TEXT]]) and len(results): break return results, n_spaces @@ -814,7 +812,7 @@ class NewznabCache(tvcache.TVCache): self._checkAuth() (items, n_spaces) = self.provider.cache_data(need_anime=need_anime, need_sports=need_sports, need_sd=need_sd, need_hd=need_hd, need_uhd=need_uhd) - except Exception as e: + except (StandardError, Exception): items = None if items: diff --git a/sickbeard/providers/omgwtfnzbs.py b/sickbeard/providers/omgwtfnzbs.py index e2a96598..5b8ba6c1 100644 --- a/sickbeard/providers/omgwtfnzbs.py +++ b/sickbeard/providers/omgwtfnzbs.py @@ -36,12 +36,12 @@ class OmgwtfnzbsProvider(generic.NZBProvider): def __init__(self): generic.NZBProvider.__init__(self, 'omgwtfnzbs') - self.url = 'https://omgwtfnzbs.org/' + self.url = 'https://omgwtfnzbs.me/' - self.url_base = 'https://omgwtfnzbs.org/' - self.url_api = 'https://api.omgwtfnzbs.org/' + self.url_base = 'https://omgwtfnzbs.me/' + self.url_api = 'https://api.omgwtfnzbs.me/' self.urls = {'config_provider_home_uri': self.url_base, - 'cache': 'https://rss.omgwtfnzbs.org/rss-download.php?%s', + 'cache': 'https://rss.omgwtfnzbs.me/rss-download.php?%s', 'search': self.url_api + 'json/?%s', 'get': self.url_base + '%s', 'cache_html': self.url_base + 'browse.php?cat=tv%s',