diff --git a/CHANGES.md b/CHANGES.md index cad6a303..585bf6ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -64,6 +64,7 @@ * Change only use newznab Api key if needed * Change editshow saving empty scene exceptions * Change improve TVDB data handling +* Change increase performance by reducing TVDb API requests with a global token [develop changelog] diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index 059ec319..cd45e156 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -21,13 +21,15 @@ import requests import requests.exceptions import datetime from sickbeard.helpers import getURL +import sickbeard from lib.dateutil.parser import parse from lib.cachecontrol import CacheControl, caches from tvdb_ui import BaseUI, ConsoleUI -from tvdb_exceptions import (tvdb_error, tvdb_shownotfound, - tvdb_seasonnotfound, tvdb_episodenotfound, tvdb_attributenotfound) +from tvdb_exceptions import ( + tvdb_error, tvdb_shownotfound, tvdb_seasonnotfound, tvdb_episodenotfound, + tvdb_attributenotfound, tvdb_tokenexpired) def log(): @@ -59,6 +61,7 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): @wraps(f) def f_retry(*args, **kwargs): mtries, mdelay = tries, delay + auth_error = 0 while mtries > 1: try: return f(*args, **kwargs) @@ -69,9 +72,17 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): else: print msg time.sleep(mdelay) - mtries -= 1 - mdelay *= backoff - return f(*args, **kwargs) + if isinstance(e, tvdb_tokenexpired) and not auth_error: + auth_error += 1 + else: + mtries -= 1 + mdelay *= backoff + try: + return f(*args, **kwargs) + except tvdb_tokenexpired: + if not auth_error: + return f(*args, **kwargs) + raise tvdb_tokenexpired return f_retry # true decorator @@ -423,8 +434,6 @@ class Tvdb: else: self.config['apikey'] = '0629B785CE550C8D' # tvdb_api's API key - self.token = {'token': None, 'datetime': datetime.datetime.fromordinal(1)} - self.config['debug_enabled'] = debug # show debugging messages self.config['custom_ui'] = custom_ui @@ -501,23 +510,25 @@ class Tvdb: self.config['url_artworkPrefix'] = 'https://thetvdb.com/banners/%s' def get_new_token(self): - token = None + token = sickbeard.THETVDB_V2_API_TOKEN.get('token', None) + dt = sickbeard.THETVDB_V2_API_TOKEN.get('datetime', datetime.datetime.fromordinal(1)) url = '%s%s' % (self.config['base_url'], 'login') params = {'apikey': self.config['apikey']} resp = getURL(url.strip(), post_json=params, json=True) if resp: if 'token' in resp: token = resp['token'] + dt = datetime.datetime.now() - return {'token': token, 'datetime': datetime.datetime.now()} + return {'token': token, 'datetime': dt} def get_token(self): - if self.token.get('token') is None or datetime.datetime.now() - self.token.get( + if sickbeard.THETVDB_V2_API_TOKEN.get('token') is None or datetime.datetime.now() - sickbeard.THETVDB_V2_API_TOKEN.get( 'datetime', datetime.datetime.fromordinal(1)) > datetime.timedelta(hours=23): - self.token = self.get_new_token() - if not self.token.get('token'): + sickbeard.THETVDB_V2_API_TOKEN = self.get_new_token() + if not sickbeard.THETVDB_V2_API_TOKEN.get('token'): raise tvdb_error('Could not get Authentification Token') - return self.token.get('token') + return sickbeard.THETVDB_V2_API_TOKEN.get('token') @staticmethod def _get_temp_dir(): @@ -535,7 +546,7 @@ class Tvdb: return os.path.join(tempfile.gettempdir(), 'tvdb_api-%s' % uid) - @retry(tvdb_error) + @retry((tvdb_error, tvdb_tokenexpired)) def _load_url(self, url, params=None, language=None): log().debug('Retrieving URL %s' % url) @@ -554,7 +565,19 @@ class Tvdb: if None is not language and language in self.config['valid_languages']: session.headers.update({'Accept-Language': language}) - resp = getURL(url.strip(), params=params, session=session, json=True) + resp = None + try: + resp = getURL(url.strip(), params=params, session=session, json=True, raise_status_code=True, + raise_exceptions=True) + except requests.exceptions.HTTPError as e: + if 401 == e.response.status_code: + # token expired, get new token, raise error to retry + sickbeard.THETVDB_V2_API_TOKEN = self.get_new_token() + raise tvdb_tokenexpired + elif 404 != e.response.status_code: + raise tvdb_error + except (StandardError, Exception): + raise tvdb_error map_show = {'airstime': 'airs_time', 'airsdayofweek': 'airs_dayofweek', 'imdbid': 'imdb_id'} @@ -803,6 +826,8 @@ class Tvdb: episodes = [] while page is not None: episode_data = self._getetsrc(self.config['url_epInfo'] % (sid, page), language=language) + if [] is episode_data: + raise tvdb_error('Exception retrieving episodes for show') if isinstance(episode_data, dict) and episode_data['data'] is not None: episodes.extend(episode_data['data']) page = episode_data['links']['next'] if isinstance(episode_data, dict) \ diff --git a/lib/tvdb_api/tvdb_exceptions.py b/lib/tvdb_api/tvdb_exceptions.py index 3683ef60..e17e2e60 100644 --- a/lib/tvdb_api/tvdb_exceptions.py +++ b/lib/tvdb_api/tvdb_exceptions.py @@ -50,3 +50,8 @@ class tvdb_attributenotfound(tvdb_exception): attribute (such as a episode name) """ pass + +class tvdb_tokenexpired(tvdb_exception): + """token expired or missing thetvdb.com + """ + pass diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index fda97e5a..65c070e1 100755 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -506,6 +506,8 @@ else: TRAKT_PIN_URL = 'https://trakt.tv/pin/6314' TRAKT_BASE_URL = 'https://api.trakt.tv/' +THETVDB_V2_API_TOKEN = {'token': None, 'datetime': datetime.datetime.fromordinal(1)} + COOKIE_SECRET = base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes) CACHE_IMAGE_URL_LIST = classes.ImageUrlList() diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index df6e44fe..d998d36a 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1100,7 +1100,8 @@ def proxy_setting(proxy_setting, request_url, force=False): return (False, proxy_address)[request_url_match], True -def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False, raise_status_code=False, **kwargs): +def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False, + raise_status_code=False, raise_exceptions=False, **kwargs): """ Returns a byte-string retrieved from the url provider. """ @@ -1197,16 +1198,22 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N if 'mute_connect_err' not in mute: logger.log(u'Connection error msg:%s while loading URL%s' % ( e.message, _maybe_request_url(e)), logger.WARNING) + if raise_exceptions: + raise e return except requests.exceptions.ReadTimeout as e: if 'mute_read_timeout' not in mute: logger.log(u'Read timed out msg:%s while loading URL%s' % ( e.message, _maybe_request_url(e)), logger.WARNING) + if raise_exceptions: + raise e return except (requests.exceptions.Timeout, socket.timeout) as e: if 'mute_connect_timeout' not in mute: logger.log(u'Connection timed out msg:%s while loading URL %s' % ( e.message, _maybe_request_url(e, url)), logger.WARNING) + if raise_exceptions: + raise e return except Exception as e: if e.message: @@ -1215,6 +1222,8 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N else: logger.log(u'Unknown exception while loading URL %s\r\nDetail... %s' % (url, traceback.format_exc()), logger.WARNING) + if raise_exceptions: + raise e return if json: @@ -1223,6 +1232,8 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N return ({}, data_json)[isinstance(data_json, (dict, list))] except (TypeError, Exception) as e: logger.log(u'JSON data issue from URL %s\r\nDetail... %s' % (url, e.message), logger.WARNING) + if raise_exceptions: + raise e return None return resp.content diff --git a/sickbeard/indexers/indexer_exceptions.py b/sickbeard/indexers/indexer_exceptions.py index e0d8c3c6..173c9422 100644 --- a/sickbeard/indexers/indexer_exceptions.py +++ b/sickbeard/indexers/indexer_exceptions.py @@ -7,17 +7,22 @@ from lib.tvdb_api.tvdb_exceptions import \ tvdb_exception, tvdb_attributenotfound, tvdb_episodenotfound, tvdb_error, \ - tvdb_seasonnotfound, tvdb_shownotfound, tvdb_userabort + tvdb_seasonnotfound, tvdb_shownotfound, tvdb_userabort, tvdb_tokenexpired -indexerExcepts = ["indexer_exception", "indexer_error", "indexer_userabort", "indexer_shownotfound", - "indexer_seasonnotfound", "indexer_episodenotfound", "indexer_attributenotfound"] +indexerExcepts = [ + 'indexer_exception', 'indexer_error', 'indexer_userabort', + 'indexer_shownotfound', 'indexer_seasonnotfound', 'indexer_episodenotfound', + 'indexer_attributenotfound', 'indexer_authenticationerror'] -tvdbExcepts = ["tvdb_exception", "tvdb_error", "tvdb_userabort", "tvdb_shownotfound", - "tvdb_seasonnotfound", "tvdb_episodenotfound", "tvdb_attributenotfound"] +tvdbExcepts = [ + 'tvdb_exception', 'tvdb_error', 'tvdb_userabort', 'tvdb_shownotfound', + 'tvdb_seasonnotfound', 'tvdb_episodenotfound', 'tvdb_attributenotfound', + 'tvdb_tokenexpired'] # link API exceptions to our exception handler indexer_exception = tvdb_exception indexer_error = tvdb_error +indexer_authenticationerror = tvdb_tokenexpired indexer_userabort = tvdb_userabort indexer_attributenotfound = tvdb_attributenotfound indexer_episodenotfound = tvdb_episodenotfound