From 3bd392e67184edccd68d73b65e14670511ea7bf9 Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Thu, 10 Aug 2017 21:01:41 +0100 Subject: [PATCH] Add count of failed show updates Change add warning message to Display Show and Edit Show page if show no longer found at TV info source due to an ID change, the last successful date of show update with current ID is also displayed. Add "Shows not found previously" to Manage/Show Processes Page to highlight which shows can adjust their show IDs in order to sustain TV info updates. Add "Shows from defunct indexers" to Manage/Show Processes page to highlight which shows can be switched to a different default TV info source. Shows not found on an indexer for over 7 days will only be retried once a week. Change improve show not found detection in Tvdb_Api lib. Change add tv_shows_not_found db table. Change add cleanup of orphaned not found shows during startup. Change add UI Notification when MasterID is not changed because a show with that ID already exists in DB. --- gui/slick/interfaces/default/editShow.tmpl | 2 +- .../default/manage_showProcesses.tmpl | 62 +++++++++++++++ lib/tvdb_api/tvdb_api.py | 11 +++ sickbeard/databases/mainDB.py | 23 +++++- sickbeard/db.py | 3 +- sickbeard/tv.py | 75 +++++++++++++++++-- sickbeard/webserve.py | 54 +++++++++++-- 7 files changed, 216 insertions(+), 14 deletions(-) diff --git a/gui/slick/interfaces/default/editShow.tmpl b/gui/slick/interfaces/default/editShow.tmpl index 308b57f8..09730d05 100644 --- a/gui/slick/interfaces/default/editShow.tmpl +++ b/gui/slick/interfaces/default/editShow.tmpl @@ -12,7 +12,7 @@ #set $css = $getVar('css', 'reg') #set $has_art = $getVar('has_art', None) #set $restart = 'Restart SickGear for new features on this page' -#set $show_message = (None, $restart)[None is $has_art] +#set $show_message = ($show_message, $restart)[None is $has_art] #set global $page_body_attr = 'edit-show" class="' + $css ## #import os.path diff --git a/gui/slick/interfaces/default/manage_showProcesses.tmpl b/gui/slick/interfaces/default/manage_showProcesses.tmpl index c28b1e82..2cdbcb47 100644 --- a/gui/slick/interfaces/default/manage_showProcesses.tmpl +++ b/gui/slick/interfaces/default/manage_showProcesses.tmpl @@ -26,6 +26,48 @@ Currently running
#end if
+#if $NotFoundShows +

Shows not found previously:

+
+ + + + + + + + #set $row = 0 + #for $cur_show in $NotFoundShows: + + + + + #end for + +
Show NameLast Found
+ $cur_show['show_name'] + $cur_show['last_success']
+#end if +#if $DefunctIndexer +

Shows from defunct indexers:

+
+ + + + + + + #set $row = 0 + #for $cur_show in $DefunctIndexer: + + + + #end for + +
Show Name
+ $cur_show['show_name'] +
+#end if

Show Queue:


#if $queueLength['add'] or $queueLength['update'] or $queueLength['refresh'] or $queueLength['rename'] or $queueLength['subtitle'] @@ -38,6 +80,10 @@ Add: $len($queueLength['add']) show$sickbeard.helpers.maybe_plural($len($queu + + + + #set $row = 0 #for $cur_show in $queueLength['add']: #set $show_name = str($cur_show['name']) @@ -58,6 +104,10 @@ Update (Forced / Forced Web): $len($queueLengt + + + + #set $row = 0 #for $cur_show in $queueLength['update']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -81,6 +131,10 @@ Refresh: $len($queueLength['refresh']) show$sickbeard.helpers.maybe_plural($l + + + + #set $row = 0 #for $cur_show in $queueLength['refresh']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -105,6 +159,10 @@ Rename: $len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len + + + + #set $row = 0 #for $cur_show in $queueLength['rename']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -129,6 +187,10 @@ Rename: $len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len + + + + #set $row = 0 #for $cur_show in $queueLength['subtitle']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index d00c4b70..91b7e741 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -20,6 +20,8 @@ import logging import requests import requests.exceptions import datetime +import re + from sickbeard.helpers import getURL, tryInt import sickbeard @@ -432,6 +434,8 @@ class Tvdb: self.shows = ShowContainer() # Holds all Show classes self.corrections = {} # Holds show-name to show_id mapping + self.show_not_found = False + self.not_found = False self.config = {} @@ -575,6 +579,9 @@ class Tvdb: session.headers.update({'Accept-Language': language}) resp = None + if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url): + self.show_not_found = False + self.not_found = False try: resp = getURL(url.strip(), params=params, session=session, json=True, raise_status_code=True, raise_exceptions=True) @@ -583,6 +590,10 @@ class Tvdb: # token expired, get new token, raise error to retry sickbeard.THETVDB_V2_API_TOKEN = self.get_new_token() raise tvdb_tokenexpired + elif 404 == e.response.status_code: + if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url): + self.show_not_found = True + self.not_found = True elif 404 != e.response.status_code: raise tvdb_error except (StandardError, Exception): diff --git a/sickbeard/databases/mainDB.py b/sickbeard/databases/mainDB.py index c00b0b80..5efbf085 100644 --- a/sickbeard/databases/mainDB.py +++ b/sickbeard/databases/mainDB.py @@ -27,7 +27,7 @@ from sickbeard import encodingKludge as ek from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException MIN_DB_VERSION = 9 # oldest db version we support migrating from -MAX_DB_VERSION = 20004 +MAX_DB_VERSION = 20005 class MainSanityCheck(db.DBSanityCheck): @@ -38,6 +38,7 @@ class MainSanityCheck(db.DBSanityCheck): self.fix_orphan_episodes() self.fix_unaired_episodes() self.fix_scene_exceptions() + self.fix_orphan_not_found_show() def fix_duplicate_shows(self, column='indexer_id'): @@ -169,6 +170,13 @@ class MainSanityCheck(db.DBSanityCheck): logger.log('Fixing invalid scene exceptions') self.connection.action('UPDATE scene_exceptions SET season = -1 WHERE season = "null"') + def fix_orphan_not_found_show(self): + sql_result = self.connection.action('DELETE FROM tv_shows_not_found WHERE NOT EXISTS (SELECT NULL FROM ' + 'tv_shows WHERE tv_shows_not_found.indexer == tv_shows.indexer AND ' + 'tv_shows_not_found.indexer_id == tv_shows.indexer_id)') + if sql_result.rowcount: + logger.log('Fixed orphaned not found shows') + # ====================== # = Main DB Migrations = # ====================== @@ -1220,3 +1228,16 @@ class ChangeMapIndexer(db.SchemaUpgrade): self.setDBVersion(20004) return self.checkDBVersion() + + +# 20004 -> 20005 +class AddShowNotFoundCounter(db.SchemaUpgrade): + def execute(self): + if not self.hasTable('tv_shows_not_found'): + logger.log(u'Adding table tv_shows_not_found') + + db.backup_database('sickbeard.db', self.checkDBVersion()) + self.connection.action('CREATE TABLE tv_shows_not_found (indexer NUMERIC NOT NULL, indexer_id NUMERIC NOT NULL, fail_count NUMERIC NOT NULL DEFAULT 0, last_check NUMERIC NOT NULL, last_success NUMERIC, PRIMARY KEY (indexer_id, indexer))') + + self.setDBVersion(20005) + return self.checkDBVersion() diff --git a/sickbeard/db.py b/sickbeard/db.py index a41b09a7..1c3a9c5a 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -453,7 +453,8 @@ def MigrationCode(myDB): 20000: sickbeard.mainDB.DBIncreaseTo20001, 20001: sickbeard.mainDB.AddTvShowOverview, 20002: sickbeard.mainDB.AddTvShowTags, - 20003: sickbeard.mainDB.ChangeMapIndexer + 20003: sickbeard.mainDB.ChangeMapIndexer, + 20004: sickbeard.mainDB.AddShowNotFoundCounter # 20002: sickbeard.mainDB.AddCoolSickGearFeature3, } diff --git a/sickbeard/tv.py b/sickbeard/tv.py index 6ca9d7c7..bdea35fc 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -52,6 +52,7 @@ from sickbeard import postProcessor from sickbeard import subtitles from sickbeard import history from sickbeard import network_timezones +from sickbeard.sbdatetime import sbdatetime from sickbeard.blackandwhitelist import BlackAndWhiteList from sickbeard.indexermapper import del_mapping, save_mapping, MapStatus from sickbeard.generic_queue import QueuePriorities @@ -64,6 +65,8 @@ from common import DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVE from common import NAMING_DUPLICATE, NAMING_EXTEND, NAMING_LIMITED_EXTEND, NAMING_SEPARATED_REPEAT, \ NAMING_LIMITED_EXTEND_E_PREFIXED +concurrent_show_not_found_days = 7 +show_not_found_retry_days = 7 def dirty_setter(attr_name, types=None): def wrapper(self, val): @@ -114,6 +117,8 @@ class TVShow(object): self._overview = '' self._tag = '' self._mapped_ids = {} + self._not_found_count = -1 + self._last_found_on_indexer = -1 self.dirty = True @@ -160,6 +165,53 @@ class TVShow(object): overview = property(lambda self: self._overview, dirty_setter('_overview')) tag = property(lambda self: self._tag, dirty_setter('_tag')) + def _helper_load_failed_db(self): + if self._not_found_count == -1 or self._last_found_on_indexer == -1: + myDB = db.DBConnection() + results = myDB.select('SELECT fail_count, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', + [self.indexer, self.indexerid]) + if results: + self._not_found_count = helpers.tryInt(results[0]['fail_count']) + self._last_found_on_indexer = helpers.tryInt(results[0]['last_success']) + else: + self._not_found_count = 0 + self._last_found_on_indexer = 0 + + @property + def not_found_count(self): + self._helper_load_failed_db() + return self._not_found_count + + @not_found_count.setter + def not_found_count(self, v): + self._not_found_count = v + + @property + def last_found_on_indexer(self): + self._helper_load_failed_db() + return (self._last_found_on_indexer, self.last_update_indexer)[self._last_found_on_indexer <= 0] + + def inc_not_found_count(self): + myDB = db.DBConnection() + results = myDB.select('SELECT fail_count, last_check, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', + [self.indexer, self.indexerid]) + days = (show_not_found_retry_days - 1, 0)[self.not_found_count <= concurrent_show_not_found_days] + if not results or datetime.datetime.fromtimestamp(helpers.tryInt(results[0]['last_check'])) + datetime.timedelta(days=days, hours=18) < datetime.datetime.now(): + if self.not_found_count <= 0: + last_success = self.last_update_indexer + else: + last_success = helpers.tryInt(results[0]['last_success'], self.last_update_indexer) + self._last_found_on_indexer = last_success + self.not_found_count += 1 + myDB.upsert('tv_shows_not_found', {'fail_count': self.not_found_count, 'last_check': sbdatetime.now().totimestamp(default=0), 'last_success': last_success}, + {'indexer': self.indexer, 'indexer_id': self.indexerid}) + + def reset_not_found_count(self): + if self.not_found_count > 0: + self._not_found_count = 0 + myDB = db.DBConnection() + myDB.action('DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [self.indexer, self.indexerid]) + @property def ids(self): if not self._mapped_ids: @@ -327,6 +379,12 @@ class TVShow(object): logger.log('Status missing for showid: [%s] with status: [%s]' % (cur_indexerid, self.status), logger.DEBUG) + last_update_indexer = datetime.date.fromordinal(self.last_update_indexer) + + # if show was not found for 1 week, only retry to update once a week + if concurrent_show_not_found_days < self.not_found_count and (update_date - last_update_indexer) < datetime.timedelta(days=show_not_found_retry_days): + return False + myDB = db.DBConnection() sql_result = myDB.mass_action( [['SELECT airdate FROM [tv_episodes] WHERE showid = ? AND season > "0" ORDER BY season DESC, episode DESC LIMIT 1', [cur_indexerid]], @@ -336,8 +394,6 @@ class TVShow(object): last_airdate = datetime.date.fromordinal(sql_result[1][0]['airdate']) if sql_result and sql_result[1] else datetime.date.fromordinal(1) - last_update_indexer = datetime.date.fromordinal(self.last_update_indexer) - # if show is not 'Ended' and last episode aired less then 460 days ago or don't have an airdate for the last episode always update (status 'Continuing' or '') update_days_limit = 2013 ended_limit = datetime.timedelta(days=update_days_limit) @@ -921,8 +977,13 @@ class TVShow(object): myEp = t[self.indexerid, False] if None is myEp: - logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING) + if hasattr(t, 'show_not_found') and t.show_not_found: + self.inc_not_found_count() + logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING) + else: + logger.log('Show data [%s] not found' % self.name, logger.WARNING) return False + self.reset_not_found_count() try: self.name = myEp['seriesname'].strip() @@ -1066,7 +1127,8 @@ class TVShow(object): ["DELETE FROM scene_numbering WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]], ["DELETE FROM whitelist WHERE show_id = ?", [self.indexerid]], ["DELETE FROM blacklist WHERE show_id = ?", [self.indexerid]], - ["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]]] + ["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]], + ["DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?", [self.indexer, self.indexerid]]] myDB = db.DBConnection() myDB.mass_action(sql_l) @@ -1229,13 +1291,15 @@ class TVShow(object): [self.indexer, self.indexerid, old_indexer, old_indexerid]], ['UPDATE whitelist SET show_id = ? WHERE show_id = ?', [self.indexerid, old_indexerid]], ['UPDATE xem_refresh SET indexer = ?, indexer_id = ? WHERE indexer = ? AND indexer_id = ?', - [self.indexer, self.indexerid, old_indexer, old_indexerid]]]) + [self.indexer, self.indexerid, old_indexer, old_indexerid]], + ['DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [old_indexer, old_indexerid]]]) myFailedDB = db.DBConnection('failed.db') myFailedDB.action('UPDATE history SET showid = ? WHERE showid = ?', [self.indexerid, old_indexerid]) del_mapping(old_indexer, old_indexerid) self.ids[old_indexer]['status'] = MapStatus.NONE self.ids[self.indexer]['status'] = MapStatus.SOURCE + self.ids[self.indexer]['id'] = self.indexerid save_mapping(self) name_cache.remove_from_namecache(old_indexerid) @@ -1266,6 +1330,7 @@ class TVShow(object): sickbeard.save_config() name_cache.buildNameCache(self) + self.reset_not_found_count() # force the update try: diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index ff6e749c..00035a29 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -46,7 +46,7 @@ from sickbeard.providers import newznab, rsstorrent from sickbeard.common import Quality, Overview, statusStrings, qualityPresetStrings from sickbeard.common import SNATCHED, UNAIRED, IGNORED, ARCHIVED, WANTED, FAILED, SKIPPED, DOWNLOADED, SNATCHED_BEST, SNATCHED_PROPER from sickbeard.common import SD, HD720p, HD1080p, UHD2160p -from sickbeard.exceptions import ex +from sickbeard.exceptions import ex, MultipleShowObjectsException from sickbeard.helpers import has_image_ext, remove_article, starify from sickbeard.indexers.indexer_config import INDEXER_TVDB, INDEXER_TVRAGE, INDEXER_TRAKT from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering, get_scene_numbering_for_show, \ @@ -57,6 +57,7 @@ from sickbeard.browser import foldersAtPath from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names from sickbeard.search_backlog import FORCED_BACKLOG from sickbeard.indexermapper import MapStatus, save_mapping, map_indexers_to_show +from sickbeard.tv import show_not_found_retry_days, concurrent_show_not_found_days from tornado import gen from tornado.web import RequestHandler, StaticFileHandler, authenticated from lib import adba @@ -1318,6 +1319,11 @@ class Home(MainHandler): elif sickbeard.showQueueScheduler.action.isInSubtitleQueue(showObj): # @UndefinedVariable show_message = 'This show is queued and awaiting subtitles download.' + if showObj.not_found_count > 0: + # noinspection PyUnresolvedReferences + last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[showObj.last_found_on_indexer > 1] + show_message = 'This show was not found (last time found: %s) on the Source Indexer%s' % (last_found, ('', '
%s' % show_message)[len(show_message) > 0]) + t.force_update = 'home/updateShow?show=%d&force=1&web=1' % showObj.indexerid if not sickbeard.showQueueScheduler.action.isBeingAdded(showObj): # @UndefinedVariable if not sickbeard.showQueueScheduler.action.isBeingUpdated(showObj): # @UndefinedVariable @@ -1592,7 +1598,8 @@ class Home(MainHandler): return {'Success': 'Switched to new TV info source'} def saveMapping(self, show, **kwargs): - show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, int(show)) + show = helpers.tryInt(show) + show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, show) response = {} if not show_obj: return json.dumps(response) @@ -1632,10 +1639,22 @@ class Home(MainHandler): else: ui.notifications.message('Mappings unchanged, not saving.') - master_ids = [show] + [kwargs.get(x) for x in 'indexer', 'mindexerid', 'mindexer'] - if all([helpers.tryInt(x) > 0 for x in master_ids]): - master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted'] - response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']} + master_ids = [show] + [helpers.tryInt(kwargs.get(x)) for x in 'indexer', 'mindexerid', 'mindexer'] + if all([x > 0 for x in master_ids]) and sickbeard.indexerApi(kwargs['mindexer']).config.get('active') and \ + not sickbeard.indexerApi(kwargs['mindexer']).config.get('defunct') and \ + not sickbeard.indexerApi(kwargs['mindexer']).config.get('mapped_only') and \ + (helpers.tryInt(kwargs['mindexer']) != helpers.tryInt(kwargs['indexer']) or + helpers.tryInt(kwargs['mindexerid']) != show): + try: + new_show_obj = helpers.find_show_by_id(sickbeard.showList, {helpers.tryInt(kwargs['mindexer']): helpers.tryInt(kwargs['mindexerid'])},no_mapped_ids=False) + if not new_show_obj or (new_show_obj.indexer == show_obj.indexer and new_show_obj.indexerid == show_obj.indexerid): + master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted'] + response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']} + else: + ui.notifications.message('Master ID unchanged, because show from %s with ID: %s exists in DB.' % + (sickbeard.indexerApi(kwargs['mindexer']).name, kwargs['mindexerid'])) + except MultipleShowObjectsException: + pass response.update({ 'map': {k: {r: w for r, w in v.iteritems() if r != 'date'} for k, v in show_obj.ids.iteritems()} @@ -1758,6 +1777,17 @@ class Home(MainHandler): self.fanart_tmpl(t) t.num_ratings = len(sickbeard.FANART_RATINGS.get(str(t.show.indexerid), {})) + + show_message = '' + + if showObj.not_found_count > 0: + # noinspection PyUnresolvedReferences + last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[ + showObj.last_found_on_indexer > 1] + show_message = 'This show was not found (last time found: %s) on the Source Indexer' % last_found + + t.show_message = show_message + return t.respond() flatten_folders = config.checkbox_to_value(flatten_folders) @@ -4561,6 +4591,18 @@ class showProcesses(Manage): t.showList = sickbeard.showList t.ShowUpdateRunning = sickbeard.showQueueScheduler.action.isShowUpdateRunning() or sickbeard.showUpdateScheduler.action.amActive + myDb = db.DBConnection(row_type='dict') + sql_results = myDb.select('SELECT n.indexer, n.indexer_id, n.last_success, s.show_name FROM tv_shows_not_found as n INNER JOIN tv_shows as s ON (n.indexer == s.indexer AND n.indexer_id == s.indexer_id)') + for s in sql_results: + date = helpers.tryInt(s['last_success']) + s['last_success'] = ('never', sbdatetime.sbdatetime.fromordinal(date).sbfdate())[date > 1] + defunct_indexer = [i for i in sickbeard.indexerApi().all_indexers if sickbeard.indexerApi(i).config.get('defunct')] + sql_r = None + if defunct_indexer: + sql_r = myDb.select('SELECT indexer, indexer_id, show_name FROM tv_shows WHERE indexer IN (%s)' % ','.join(['?'] * len(defunct_indexer)), defunct_indexer) + t.DefunctIndexer = sql_r + t.NotFoundShows = sql_results + t.submenu = self.ManageMenu('Processes') return t.respond()