diff --git a/gui/slick/interfaces/default/editShow.tmpl b/gui/slick/interfaces/default/editShow.tmpl index 308b57f8..09730d05 100644 --- a/gui/slick/interfaces/default/editShow.tmpl +++ b/gui/slick/interfaces/default/editShow.tmpl @@ -12,7 +12,7 @@ #set $css = $getVar('css', 'reg') #set $has_art = $getVar('has_art', None) #set $restart = 'Restart SickGear for new features on this page' -#set $show_message = (None, $restart)[None is $has_art] +#set $show_message = ($show_message, $restart)[None is $has_art] #set global $page_body_attr = 'edit-show" class="' + $css ## #import os.path diff --git a/gui/slick/interfaces/default/manage_showProcesses.tmpl b/gui/slick/interfaces/default/manage_showProcesses.tmpl index c28b1e82..2cdbcb47 100644 --- a/gui/slick/interfaces/default/manage_showProcesses.tmpl +++ b/gui/slick/interfaces/default/manage_showProcesses.tmpl @@ -26,6 +26,48 @@ Currently running
#end if
+#if $NotFoundShows +

Shows not found previously:

+
+ + + + + + + + #set $row = 0 + #for $cur_show in $NotFoundShows: + + + + + #end for + +
Show NameLast Found
+ $cur_show['show_name'] + $cur_show['last_success']
+#end if +#if $DefunctIndexer +

Shows from defunct indexers:

+
+ + + + + + + #set $row = 0 + #for $cur_show in $DefunctIndexer: + + + + #end for + +
Show Name
+ $cur_show['show_name'] +
+#end if

Show Queue:


#if $queueLength['add'] or $queueLength['update'] or $queueLength['refresh'] or $queueLength['rename'] or $queueLength['subtitle'] @@ -38,6 +80,10 @@ Add: $len($queueLength['add']) show$sickbeard.helpers.maybe_plural($len($queu + + + + #set $row = 0 #for $cur_show in $queueLength['add']: #set $show_name = str($cur_show['name']) @@ -58,6 +104,10 @@ Update (Forced / Forced Web): $len($queueLengt + + + + #set $row = 0 #for $cur_show in $queueLength['update']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -81,6 +131,10 @@ Refresh: $len($queueLength['refresh']) show$sickbeard.helpers.maybe_plural($l + + + + #set $row = 0 #for $cur_show in $queueLength['refresh']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -105,6 +159,10 @@ Rename: $len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len + + + + #set $row = 0 #for $cur_show in $queueLength['rename']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) @@ -129,6 +187,10 @@ Rename: $len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len + + + + #set $row = 0 #for $cur_show in $queueLength['subtitle']: #set $show = $findCertainShow($showList, $cur_show['indexerid']) diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index d00c4b70..91b7e741 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -20,6 +20,8 @@ import logging import requests import requests.exceptions import datetime +import re + from sickbeard.helpers import getURL, tryInt import sickbeard @@ -432,6 +434,8 @@ class Tvdb: self.shows = ShowContainer() # Holds all Show classes self.corrections = {} # Holds show-name to show_id mapping + self.show_not_found = False + self.not_found = False self.config = {} @@ -575,6 +579,9 @@ class Tvdb: session.headers.update({'Accept-Language': language}) resp = None + if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url): + self.show_not_found = False + self.not_found = False try: resp = getURL(url.strip(), params=params, session=session, json=True, raise_status_code=True, raise_exceptions=True) @@ -583,6 +590,10 @@ class Tvdb: # token expired, get new token, raise error to retry sickbeard.THETVDB_V2_API_TOKEN = self.get_new_token() raise tvdb_tokenexpired + elif 404 == e.response.status_code: + if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url): + self.show_not_found = True + self.not_found = True elif 404 != e.response.status_code: raise tvdb_error except (StandardError, Exception): diff --git a/sickbeard/databases/mainDB.py b/sickbeard/databases/mainDB.py index c00b0b80..5efbf085 100644 --- a/sickbeard/databases/mainDB.py +++ b/sickbeard/databases/mainDB.py @@ -27,7 +27,7 @@ from sickbeard import encodingKludge as ek from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException MIN_DB_VERSION = 9 # oldest db version we support migrating from -MAX_DB_VERSION = 20004 +MAX_DB_VERSION = 20005 class MainSanityCheck(db.DBSanityCheck): @@ -38,6 +38,7 @@ class MainSanityCheck(db.DBSanityCheck): self.fix_orphan_episodes() self.fix_unaired_episodes() self.fix_scene_exceptions() + self.fix_orphan_not_found_show() def fix_duplicate_shows(self, column='indexer_id'): @@ -169,6 +170,13 @@ class MainSanityCheck(db.DBSanityCheck): logger.log('Fixing invalid scene exceptions') self.connection.action('UPDATE scene_exceptions SET season = -1 WHERE season = "null"') + def fix_orphan_not_found_show(self): + sql_result = self.connection.action('DELETE FROM tv_shows_not_found WHERE NOT EXISTS (SELECT NULL FROM ' + 'tv_shows WHERE tv_shows_not_found.indexer == tv_shows.indexer AND ' + 'tv_shows_not_found.indexer_id == tv_shows.indexer_id)') + if sql_result.rowcount: + logger.log('Fixed orphaned not found shows') + # ====================== # = Main DB Migrations = # ====================== @@ -1220,3 +1228,16 @@ class ChangeMapIndexer(db.SchemaUpgrade): self.setDBVersion(20004) return self.checkDBVersion() + + +# 20004 -> 20005 +class AddShowNotFoundCounter(db.SchemaUpgrade): + def execute(self): + if not self.hasTable('tv_shows_not_found'): + logger.log(u'Adding table tv_shows_not_found') + + db.backup_database('sickbeard.db', self.checkDBVersion()) + self.connection.action('CREATE TABLE tv_shows_not_found (indexer NUMERIC NOT NULL, indexer_id NUMERIC NOT NULL, fail_count NUMERIC NOT NULL DEFAULT 0, last_check NUMERIC NOT NULL, last_success NUMERIC, PRIMARY KEY (indexer_id, indexer))') + + self.setDBVersion(20005) + return self.checkDBVersion() diff --git a/sickbeard/db.py b/sickbeard/db.py index a41b09a7..1c3a9c5a 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -453,7 +453,8 @@ def MigrationCode(myDB): 20000: sickbeard.mainDB.DBIncreaseTo20001, 20001: sickbeard.mainDB.AddTvShowOverview, 20002: sickbeard.mainDB.AddTvShowTags, - 20003: sickbeard.mainDB.ChangeMapIndexer + 20003: sickbeard.mainDB.ChangeMapIndexer, + 20004: sickbeard.mainDB.AddShowNotFoundCounter # 20002: sickbeard.mainDB.AddCoolSickGearFeature3, } diff --git a/sickbeard/tv.py b/sickbeard/tv.py index 6ca9d7c7..bdea35fc 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -52,6 +52,7 @@ from sickbeard import postProcessor from sickbeard import subtitles from sickbeard import history from sickbeard import network_timezones +from sickbeard.sbdatetime import sbdatetime from sickbeard.blackandwhitelist import BlackAndWhiteList from sickbeard.indexermapper import del_mapping, save_mapping, MapStatus from sickbeard.generic_queue import QueuePriorities @@ -64,6 +65,8 @@ from common import DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVE from common import NAMING_DUPLICATE, NAMING_EXTEND, NAMING_LIMITED_EXTEND, NAMING_SEPARATED_REPEAT, \ NAMING_LIMITED_EXTEND_E_PREFIXED +concurrent_show_not_found_days = 7 +show_not_found_retry_days = 7 def dirty_setter(attr_name, types=None): def wrapper(self, val): @@ -114,6 +117,8 @@ class TVShow(object): self._overview = '' self._tag = '' self._mapped_ids = {} + self._not_found_count = -1 + self._last_found_on_indexer = -1 self.dirty = True @@ -160,6 +165,53 @@ class TVShow(object): overview = property(lambda self: self._overview, dirty_setter('_overview')) tag = property(lambda self: self._tag, dirty_setter('_tag')) + def _helper_load_failed_db(self): + if self._not_found_count == -1 or self._last_found_on_indexer == -1: + myDB = db.DBConnection() + results = myDB.select('SELECT fail_count, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', + [self.indexer, self.indexerid]) + if results: + self._not_found_count = helpers.tryInt(results[0]['fail_count']) + self._last_found_on_indexer = helpers.tryInt(results[0]['last_success']) + else: + self._not_found_count = 0 + self._last_found_on_indexer = 0 + + @property + def not_found_count(self): + self._helper_load_failed_db() + return self._not_found_count + + @not_found_count.setter + def not_found_count(self, v): + self._not_found_count = v + + @property + def last_found_on_indexer(self): + self._helper_load_failed_db() + return (self._last_found_on_indexer, self.last_update_indexer)[self._last_found_on_indexer <= 0] + + def inc_not_found_count(self): + myDB = db.DBConnection() + results = myDB.select('SELECT fail_count, last_check, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', + [self.indexer, self.indexerid]) + days = (show_not_found_retry_days - 1, 0)[self.not_found_count <= concurrent_show_not_found_days] + if not results or datetime.datetime.fromtimestamp(helpers.tryInt(results[0]['last_check'])) + datetime.timedelta(days=days, hours=18) < datetime.datetime.now(): + if self.not_found_count <= 0: + last_success = self.last_update_indexer + else: + last_success = helpers.tryInt(results[0]['last_success'], self.last_update_indexer) + self._last_found_on_indexer = last_success + self.not_found_count += 1 + myDB.upsert('tv_shows_not_found', {'fail_count': self.not_found_count, 'last_check': sbdatetime.now().totimestamp(default=0), 'last_success': last_success}, + {'indexer': self.indexer, 'indexer_id': self.indexerid}) + + def reset_not_found_count(self): + if self.not_found_count > 0: + self._not_found_count = 0 + myDB = db.DBConnection() + myDB.action('DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [self.indexer, self.indexerid]) + @property def ids(self): if not self._mapped_ids: @@ -327,6 +379,12 @@ class TVShow(object): logger.log('Status missing for showid: [%s] with status: [%s]' % (cur_indexerid, self.status), logger.DEBUG) + last_update_indexer = datetime.date.fromordinal(self.last_update_indexer) + + # if show was not found for 1 week, only retry to update once a week + if concurrent_show_not_found_days < self.not_found_count and (update_date - last_update_indexer) < datetime.timedelta(days=show_not_found_retry_days): + return False + myDB = db.DBConnection() sql_result = myDB.mass_action( [['SELECT airdate FROM [tv_episodes] WHERE showid = ? AND season > "0" ORDER BY season DESC, episode DESC LIMIT 1', [cur_indexerid]], @@ -336,8 +394,6 @@ class TVShow(object): last_airdate = datetime.date.fromordinal(sql_result[1][0]['airdate']) if sql_result and sql_result[1] else datetime.date.fromordinal(1) - last_update_indexer = datetime.date.fromordinal(self.last_update_indexer) - # if show is not 'Ended' and last episode aired less then 460 days ago or don't have an airdate for the last episode always update (status 'Continuing' or '') update_days_limit = 2013 ended_limit = datetime.timedelta(days=update_days_limit) @@ -921,8 +977,13 @@ class TVShow(object): myEp = t[self.indexerid, False] if None is myEp: - logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING) + if hasattr(t, 'show_not_found') and t.show_not_found: + self.inc_not_found_count() + logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING) + else: + logger.log('Show data [%s] not found' % self.name, logger.WARNING) return False + self.reset_not_found_count() try: self.name = myEp['seriesname'].strip() @@ -1066,7 +1127,8 @@ class TVShow(object): ["DELETE FROM scene_numbering WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]], ["DELETE FROM whitelist WHERE show_id = ?", [self.indexerid]], ["DELETE FROM blacklist WHERE show_id = ?", [self.indexerid]], - ["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]]] + ["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]], + ["DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?", [self.indexer, self.indexerid]]] myDB = db.DBConnection() myDB.mass_action(sql_l) @@ -1229,13 +1291,15 @@ class TVShow(object): [self.indexer, self.indexerid, old_indexer, old_indexerid]], ['UPDATE whitelist SET show_id = ? WHERE show_id = ?', [self.indexerid, old_indexerid]], ['UPDATE xem_refresh SET indexer = ?, indexer_id = ? WHERE indexer = ? AND indexer_id = ?', - [self.indexer, self.indexerid, old_indexer, old_indexerid]]]) + [self.indexer, self.indexerid, old_indexer, old_indexerid]], + ['DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [old_indexer, old_indexerid]]]) myFailedDB = db.DBConnection('failed.db') myFailedDB.action('UPDATE history SET showid = ? WHERE showid = ?', [self.indexerid, old_indexerid]) del_mapping(old_indexer, old_indexerid) self.ids[old_indexer]['status'] = MapStatus.NONE self.ids[self.indexer]['status'] = MapStatus.SOURCE + self.ids[self.indexer]['id'] = self.indexerid save_mapping(self) name_cache.remove_from_namecache(old_indexerid) @@ -1266,6 +1330,7 @@ class TVShow(object): sickbeard.save_config() name_cache.buildNameCache(self) + self.reset_not_found_count() # force the update try: diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index ff6e749c..00035a29 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -46,7 +46,7 @@ from sickbeard.providers import newznab, rsstorrent from sickbeard.common import Quality, Overview, statusStrings, qualityPresetStrings from sickbeard.common import SNATCHED, UNAIRED, IGNORED, ARCHIVED, WANTED, FAILED, SKIPPED, DOWNLOADED, SNATCHED_BEST, SNATCHED_PROPER from sickbeard.common import SD, HD720p, HD1080p, UHD2160p -from sickbeard.exceptions import ex +from sickbeard.exceptions import ex, MultipleShowObjectsException from sickbeard.helpers import has_image_ext, remove_article, starify from sickbeard.indexers.indexer_config import INDEXER_TVDB, INDEXER_TVRAGE, INDEXER_TRAKT from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering, get_scene_numbering_for_show, \ @@ -57,6 +57,7 @@ from sickbeard.browser import foldersAtPath from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names from sickbeard.search_backlog import FORCED_BACKLOG from sickbeard.indexermapper import MapStatus, save_mapping, map_indexers_to_show +from sickbeard.tv import show_not_found_retry_days, concurrent_show_not_found_days from tornado import gen from tornado.web import RequestHandler, StaticFileHandler, authenticated from lib import adba @@ -1318,6 +1319,11 @@ class Home(MainHandler): elif sickbeard.showQueueScheduler.action.isInSubtitleQueue(showObj): # @UndefinedVariable show_message = 'This show is queued and awaiting subtitles download.' + if showObj.not_found_count > 0: + # noinspection PyUnresolvedReferences + last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[showObj.last_found_on_indexer > 1] + show_message = 'This show was not found (last time found: %s) on the Source Indexer%s' % (last_found, ('', '
%s' % show_message)[len(show_message) > 0]) + t.force_update = 'home/updateShow?show=%d&force=1&web=1' % showObj.indexerid if not sickbeard.showQueueScheduler.action.isBeingAdded(showObj): # @UndefinedVariable if not sickbeard.showQueueScheduler.action.isBeingUpdated(showObj): # @UndefinedVariable @@ -1592,7 +1598,8 @@ class Home(MainHandler): return {'Success': 'Switched to new TV info source'} def saveMapping(self, show, **kwargs): - show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, int(show)) + show = helpers.tryInt(show) + show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, show) response = {} if not show_obj: return json.dumps(response) @@ -1632,10 +1639,22 @@ class Home(MainHandler): else: ui.notifications.message('Mappings unchanged, not saving.') - master_ids = [show] + [kwargs.get(x) for x in 'indexer', 'mindexerid', 'mindexer'] - if all([helpers.tryInt(x) > 0 for x in master_ids]): - master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted'] - response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']} + master_ids = [show] + [helpers.tryInt(kwargs.get(x)) for x in 'indexer', 'mindexerid', 'mindexer'] + if all([x > 0 for x in master_ids]) and sickbeard.indexerApi(kwargs['mindexer']).config.get('active') and \ + not sickbeard.indexerApi(kwargs['mindexer']).config.get('defunct') and \ + not sickbeard.indexerApi(kwargs['mindexer']).config.get('mapped_only') and \ + (helpers.tryInt(kwargs['mindexer']) != helpers.tryInt(kwargs['indexer']) or + helpers.tryInt(kwargs['mindexerid']) != show): + try: + new_show_obj = helpers.find_show_by_id(sickbeard.showList, {helpers.tryInt(kwargs['mindexer']): helpers.tryInt(kwargs['mindexerid'])},no_mapped_ids=False) + if not new_show_obj or (new_show_obj.indexer == show_obj.indexer and new_show_obj.indexerid == show_obj.indexerid): + master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted'] + response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']} + else: + ui.notifications.message('Master ID unchanged, because show from %s with ID: %s exists in DB.' % + (sickbeard.indexerApi(kwargs['mindexer']).name, kwargs['mindexerid'])) + except MultipleShowObjectsException: + pass response.update({ 'map': {k: {r: w for r, w in v.iteritems() if r != 'date'} for k, v in show_obj.ids.iteritems()} @@ -1758,6 +1777,17 @@ class Home(MainHandler): self.fanart_tmpl(t) t.num_ratings = len(sickbeard.FANART_RATINGS.get(str(t.show.indexerid), {})) + + show_message = '' + + if showObj.not_found_count > 0: + # noinspection PyUnresolvedReferences + last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[ + showObj.last_found_on_indexer > 1] + show_message = 'This show was not found (last time found: %s) on the Source Indexer' % last_found + + t.show_message = show_message + return t.respond() flatten_folders = config.checkbox_to_value(flatten_folders) @@ -4561,6 +4591,18 @@ class showProcesses(Manage): t.showList = sickbeard.showList t.ShowUpdateRunning = sickbeard.showQueueScheduler.action.isShowUpdateRunning() or sickbeard.showUpdateScheduler.action.amActive + myDb = db.DBConnection(row_type='dict') + sql_results = myDb.select('SELECT n.indexer, n.indexer_id, n.last_success, s.show_name FROM tv_shows_not_found as n INNER JOIN tv_shows as s ON (n.indexer == s.indexer AND n.indexer_id == s.indexer_id)') + for s in sql_results: + date = helpers.tryInt(s['last_success']) + s['last_success'] = ('never', sbdatetime.sbdatetime.fromordinal(date).sbfdate())[date > 1] + defunct_indexer = [i for i in sickbeard.indexerApi().all_indexers if sickbeard.indexerApi(i).config.get('defunct')] + sql_r = None + if defunct_indexer: + sql_r = myDb.select('SELECT indexer, indexer_id, show_name FROM tv_shows WHERE indexer IN (%s)' % ','.join(['?'] * len(defunct_indexer)), defunct_indexer) + t.DefunctIndexer = sql_r + t.NotFoundShows = sql_results + t.submenu = self.ManageMenu('Processes') return t.respond()