Add count of failed show updates

Change add warning message to Display Show and Edit Show page if show no longer found at TV info source due to an ID change, the last successful date of show update with current ID is also displayed.
Add "Shows not found previously" to Manage/Show Processes Page to highlight which shows can adjust their show IDs in order to sustain TV info updates.
Add "Shows from defunct indexers" to Manage/Show Processes page to highlight which shows can be switched to a different default TV info source.
Shows not found on an indexer for over 7 days will only be retried once a week.
Change improve show not found detection in Tvdb_Api lib.
Change add tv_shows_not_found db table.
Change add cleanup of orphaned not found shows during startup.
Change add UI Notification when MasterID is not changed because a show with that ID already exists in DB.
This commit is contained in:
Prinz23 2017-08-10 21:01:41 +01:00 committed by JackDandy
parent c150c83802
commit 3bd392e671
7 changed files with 216 additions and 14 deletions

View file

@ -12,7 +12,7 @@
#set $css = $getVar('css', 'reg')
#set $has_art = $getVar('has_art', None)
#set $restart = 'Restart SickGear for new features on this page'
#set $show_message = (None, $restart)[None is $has_art]
#set $show_message = ($show_message, $restart)[None is $has_art]
#set global $page_body_attr = 'edit-show" class="' + $css
##
#import os.path

View file

@ -26,6 +26,48 @@
Currently running<br />
#end if
</br>
#if $NotFoundShows
<h3>Shows not found previously:</h3>
<input type="button" class="shows-more btn" id="notfound-btn-more" value="Expand" style="display:none"><input type="button" class="shows-less btn" id="notfound-btn-less" value="Collapse"></br>
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th>Last Found</th>
</tr>
#set $row = 0
#for $cur_show in $NotFoundShows:
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
<td style="width:80%;text-align:left">
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_show['indexer_id']">$cur_show['show_name']</a>
</td>
<td style="width:20%;text-align:center;color:white">$cur_show['last_success']</td>
</tr>
#end for
</tbody>
</table>
#end if
#if $DefunctIndexer
<h3>Shows from defunct indexers:</h3>
<input type="button" class="shows-more btn" id="defunct-btn-more" value="Expand" style="display:none"><input type="button" class="shows-less btn" id="defunct-btn-less" value="Collapse"></br>
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
</tr>
#set $row = 0
#for $cur_show in $DefunctIndexer:
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
<td style="width:80%;text-align:left">
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_show['indexer_id']">$cur_show['show_name']</a>
</td>
</tr>
#end for
</tbody>
</table>
#end if
<h3>Show Queue:</h3>
</br>
#if $queueLength['add'] or $queueLength['update'] or $queueLength['refresh'] or $queueLength['rename'] or $queueLength['subtitle']
@ -38,6 +80,10 @@ Add: <i>$len($queueLength['add']) show$sickbeard.helpers.maybe_plural($len($queu
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th></th>
</tr>
#set $row = 0
#for $cur_show in $queueLength['add']:
#set $show_name = str($cur_show['name'])
@ -58,6 +104,10 @@ Update <span class="grey-text">(Forced / Forced Web)</span>: <i>$len($queueLengt
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th>Schedule Type</th>
</tr>
#set $row = 0
#for $cur_show in $queueLength['update']:
#set $show = $findCertainShow($showList, $cur_show['indexerid'])
@ -81,6 +131,10 @@ Refresh: <i>$len($queueLength['refresh']) show$sickbeard.helpers.maybe_plural($l
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th>Schedule Type</th>
</tr>
#set $row = 0
#for $cur_show in $queueLength['refresh']:
#set $show = $findCertainShow($showList, $cur_show['indexerid'])
@ -105,6 +159,10 @@ Rename: <i>$len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th>Schedule Type</th>
</tr>
#set $row = 0
#for $cur_show in $queueLength['rename']:
#set $show = $findCertainShow($showList, $cur_show['indexerid'])
@ -129,6 +187,10 @@ Rename: <i>$len($queueLength['rename']) show$sickbeard.helpers.maybe_plural($len
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
<thead></thead>
<tbody>
<tr>
<th>Show Name</th>
<th>Schedule Type</th>
</tr>
#set $row = 0
#for $cur_show in $queueLength['subtitle']:
#set $show = $findCertainShow($showList, $cur_show['indexerid'])

View file

@ -20,6 +20,8 @@ import logging
import requests
import requests.exceptions
import datetime
import re
from sickbeard.helpers import getURL, tryInt
import sickbeard
@ -432,6 +434,8 @@ class Tvdb:
self.shows = ShowContainer() # Holds all Show classes
self.corrections = {} # Holds show-name to show_id mapping
self.show_not_found = False
self.not_found = False
self.config = {}
@ -575,6 +579,9 @@ class Tvdb:
session.headers.update({'Accept-Language': language})
resp = None
if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url):
self.show_not_found = False
self.not_found = False
try:
resp = getURL(url.strip(), params=params, session=session, json=True, raise_status_code=True,
raise_exceptions=True)
@ -583,6 +590,10 @@ class Tvdb:
# token expired, get new token, raise error to retry
sickbeard.THETVDB_V2_API_TOKEN = self.get_new_token()
raise tvdb_tokenexpired
elif 404 == e.response.status_code:
if re.search(re.escape(self.config['url_seriesInfo']).replace('%s', '.*'), url):
self.show_not_found = True
self.not_found = True
elif 404 != e.response.status_code:
raise tvdb_error
except (StandardError, Exception):

View file

@ -27,7 +27,7 @@ from sickbeard import encodingKludge as ek
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
MIN_DB_VERSION = 9 # oldest db version we support migrating from
MAX_DB_VERSION = 20004
MAX_DB_VERSION = 20005
class MainSanityCheck(db.DBSanityCheck):
@ -38,6 +38,7 @@ class MainSanityCheck(db.DBSanityCheck):
self.fix_orphan_episodes()
self.fix_unaired_episodes()
self.fix_scene_exceptions()
self.fix_orphan_not_found_show()
def fix_duplicate_shows(self, column='indexer_id'):
@ -169,6 +170,13 @@ class MainSanityCheck(db.DBSanityCheck):
logger.log('Fixing invalid scene exceptions')
self.connection.action('UPDATE scene_exceptions SET season = -1 WHERE season = "null"')
def fix_orphan_not_found_show(self):
sql_result = self.connection.action('DELETE FROM tv_shows_not_found WHERE NOT EXISTS (SELECT NULL FROM '
'tv_shows WHERE tv_shows_not_found.indexer == tv_shows.indexer AND '
'tv_shows_not_found.indexer_id == tv_shows.indexer_id)')
if sql_result.rowcount:
logger.log('Fixed orphaned not found shows')
# ======================
# = Main DB Migrations =
# ======================
@ -1220,3 +1228,16 @@ class ChangeMapIndexer(db.SchemaUpgrade):
self.setDBVersion(20004)
return self.checkDBVersion()
# 20004 -> 20005
class AddShowNotFoundCounter(db.SchemaUpgrade):
def execute(self):
if not self.hasTable('tv_shows_not_found'):
logger.log(u'Adding table tv_shows_not_found')
db.backup_database('sickbeard.db', self.checkDBVersion())
self.connection.action('CREATE TABLE tv_shows_not_found (indexer NUMERIC NOT NULL, indexer_id NUMERIC NOT NULL, fail_count NUMERIC NOT NULL DEFAULT 0, last_check NUMERIC NOT NULL, last_success NUMERIC, PRIMARY KEY (indexer_id, indexer))')
self.setDBVersion(20005)
return self.checkDBVersion()

View file

@ -453,7 +453,8 @@ def MigrationCode(myDB):
20000: sickbeard.mainDB.DBIncreaseTo20001,
20001: sickbeard.mainDB.AddTvShowOverview,
20002: sickbeard.mainDB.AddTvShowTags,
20003: sickbeard.mainDB.ChangeMapIndexer
20003: sickbeard.mainDB.ChangeMapIndexer,
20004: sickbeard.mainDB.AddShowNotFoundCounter
# 20002: sickbeard.mainDB.AddCoolSickGearFeature3,
}

View file

@ -52,6 +52,7 @@ from sickbeard import postProcessor
from sickbeard import subtitles
from sickbeard import history
from sickbeard import network_timezones
from sickbeard.sbdatetime import sbdatetime
from sickbeard.blackandwhitelist import BlackAndWhiteList
from sickbeard.indexermapper import del_mapping, save_mapping, MapStatus
from sickbeard.generic_queue import QueuePriorities
@ -64,6 +65,8 @@ from common import DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVE
from common import NAMING_DUPLICATE, NAMING_EXTEND, NAMING_LIMITED_EXTEND, NAMING_SEPARATED_REPEAT, \
NAMING_LIMITED_EXTEND_E_PREFIXED
concurrent_show_not_found_days = 7
show_not_found_retry_days = 7
def dirty_setter(attr_name, types=None):
def wrapper(self, val):
@ -114,6 +117,8 @@ class TVShow(object):
self._overview = ''
self._tag = ''
self._mapped_ids = {}
self._not_found_count = -1
self._last_found_on_indexer = -1
self.dirty = True
@ -160,6 +165,53 @@ class TVShow(object):
overview = property(lambda self: self._overview, dirty_setter('_overview'))
tag = property(lambda self: self._tag, dirty_setter('_tag'))
def _helper_load_failed_db(self):
if self._not_found_count == -1 or self._last_found_on_indexer == -1:
myDB = db.DBConnection()
results = myDB.select('SELECT fail_count, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?',
[self.indexer, self.indexerid])
if results:
self._not_found_count = helpers.tryInt(results[0]['fail_count'])
self._last_found_on_indexer = helpers.tryInt(results[0]['last_success'])
else:
self._not_found_count = 0
self._last_found_on_indexer = 0
@property
def not_found_count(self):
self._helper_load_failed_db()
return self._not_found_count
@not_found_count.setter
def not_found_count(self, v):
self._not_found_count = v
@property
def last_found_on_indexer(self):
self._helper_load_failed_db()
return (self._last_found_on_indexer, self.last_update_indexer)[self._last_found_on_indexer <= 0]
def inc_not_found_count(self):
myDB = db.DBConnection()
results = myDB.select('SELECT fail_count, last_check, last_success FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?',
[self.indexer, self.indexerid])
days = (show_not_found_retry_days - 1, 0)[self.not_found_count <= concurrent_show_not_found_days]
if not results or datetime.datetime.fromtimestamp(helpers.tryInt(results[0]['last_check'])) + datetime.timedelta(days=days, hours=18) < datetime.datetime.now():
if self.not_found_count <= 0:
last_success = self.last_update_indexer
else:
last_success = helpers.tryInt(results[0]['last_success'], self.last_update_indexer)
self._last_found_on_indexer = last_success
self.not_found_count += 1
myDB.upsert('tv_shows_not_found', {'fail_count': self.not_found_count, 'last_check': sbdatetime.now().totimestamp(default=0), 'last_success': last_success},
{'indexer': self.indexer, 'indexer_id': self.indexerid})
def reset_not_found_count(self):
if self.not_found_count > 0:
self._not_found_count = 0
myDB = db.DBConnection()
myDB.action('DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [self.indexer, self.indexerid])
@property
def ids(self):
if not self._mapped_ids:
@ -327,6 +379,12 @@ class TVShow(object):
logger.log('Status missing for showid: [%s] with status: [%s]' %
(cur_indexerid, self.status), logger.DEBUG)
last_update_indexer = datetime.date.fromordinal(self.last_update_indexer)
# if show was not found for 1 week, only retry to update once a week
if concurrent_show_not_found_days < self.not_found_count and (update_date - last_update_indexer) < datetime.timedelta(days=show_not_found_retry_days):
return False
myDB = db.DBConnection()
sql_result = myDB.mass_action(
[['SELECT airdate FROM [tv_episodes] WHERE showid = ? AND season > "0" ORDER BY season DESC, episode DESC LIMIT 1', [cur_indexerid]],
@ -336,8 +394,6 @@ class TVShow(object):
last_airdate = datetime.date.fromordinal(sql_result[1][0]['airdate']) if sql_result and sql_result[1] else datetime.date.fromordinal(1)
last_update_indexer = datetime.date.fromordinal(self.last_update_indexer)
# if show is not 'Ended' and last episode aired less then 460 days ago or don't have an airdate for the last episode always update (status 'Continuing' or '')
update_days_limit = 2013
ended_limit = datetime.timedelta(days=update_days_limit)
@ -921,8 +977,13 @@ class TVShow(object):
myEp = t[self.indexerid, False]
if None is myEp:
logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING)
if hasattr(t, 'show_not_found') and t.show_not_found:
self.inc_not_found_count()
logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING)
else:
logger.log('Show data [%s] not found' % self.name, logger.WARNING)
return False
self.reset_not_found_count()
try:
self.name = myEp['seriesname'].strip()
@ -1066,7 +1127,8 @@ class TVShow(object):
["DELETE FROM scene_numbering WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]],
["DELETE FROM whitelist WHERE show_id = ?", [self.indexerid]],
["DELETE FROM blacklist WHERE show_id = ?", [self.indexerid]],
["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]]]
["DELETE FROM indexer_mapping WHERE indexer_id = ? AND indexer = ?", [self.indexerid, self.indexer]],
["DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?", [self.indexer, self.indexerid]]]
myDB = db.DBConnection()
myDB.mass_action(sql_l)
@ -1229,13 +1291,15 @@ class TVShow(object):
[self.indexer, self.indexerid, old_indexer, old_indexerid]],
['UPDATE whitelist SET show_id = ? WHERE show_id = ?', [self.indexerid, old_indexerid]],
['UPDATE xem_refresh SET indexer = ?, indexer_id = ? WHERE indexer = ? AND indexer_id = ?',
[self.indexer, self.indexerid, old_indexer, old_indexerid]]])
[self.indexer, self.indexerid, old_indexer, old_indexerid]],
['DELETE FROM tv_shows_not_found WHERE indexer = ? AND indexer_id = ?', [old_indexer, old_indexerid]]])
myFailedDB = db.DBConnection('failed.db')
myFailedDB.action('UPDATE history SET showid = ? WHERE showid = ?', [self.indexerid, old_indexerid])
del_mapping(old_indexer, old_indexerid)
self.ids[old_indexer]['status'] = MapStatus.NONE
self.ids[self.indexer]['status'] = MapStatus.SOURCE
self.ids[self.indexer]['id'] = self.indexerid
save_mapping(self)
name_cache.remove_from_namecache(old_indexerid)
@ -1266,6 +1330,7 @@ class TVShow(object):
sickbeard.save_config()
name_cache.buildNameCache(self)
self.reset_not_found_count()
# force the update
try:

View file

@ -46,7 +46,7 @@ from sickbeard.providers import newznab, rsstorrent
from sickbeard.common import Quality, Overview, statusStrings, qualityPresetStrings
from sickbeard.common import SNATCHED, UNAIRED, IGNORED, ARCHIVED, WANTED, FAILED, SKIPPED, DOWNLOADED, SNATCHED_BEST, SNATCHED_PROPER
from sickbeard.common import SD, HD720p, HD1080p, UHD2160p
from sickbeard.exceptions import ex
from sickbeard.exceptions import ex, MultipleShowObjectsException
from sickbeard.helpers import has_image_ext, remove_article, starify
from sickbeard.indexers.indexer_config import INDEXER_TVDB, INDEXER_TVRAGE, INDEXER_TRAKT
from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering, get_scene_numbering_for_show, \
@ -57,6 +57,7 @@ from sickbeard.browser import foldersAtPath
from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names
from sickbeard.search_backlog import FORCED_BACKLOG
from sickbeard.indexermapper import MapStatus, save_mapping, map_indexers_to_show
from sickbeard.tv import show_not_found_retry_days, concurrent_show_not_found_days
from tornado import gen
from tornado.web import RequestHandler, StaticFileHandler, authenticated
from lib import adba
@ -1318,6 +1319,11 @@ class Home(MainHandler):
elif sickbeard.showQueueScheduler.action.isInSubtitleQueue(showObj): # @UndefinedVariable
show_message = 'This show is queued and awaiting subtitles download.'
if showObj.not_found_count > 0:
# noinspection PyUnresolvedReferences
last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[showObj.last_found_on_indexer > 1]
show_message = 'This show was not found (last time found: %s) on the Source Indexer%s' % (last_found, ('', '<br>%s' % show_message)[len(show_message) > 0])
t.force_update = 'home/updateShow?show=%d&amp;force=1&amp;web=1' % showObj.indexerid
if not sickbeard.showQueueScheduler.action.isBeingAdded(showObj): # @UndefinedVariable
if not sickbeard.showQueueScheduler.action.isBeingUpdated(showObj): # @UndefinedVariable
@ -1592,7 +1598,8 @@ class Home(MainHandler):
return {'Success': 'Switched to new TV info source'}
def saveMapping(self, show, **kwargs):
show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, int(show))
show = helpers.tryInt(show)
show_obj = sickbeard.helpers.findCertainShow(sickbeard.showList, show)
response = {}
if not show_obj:
return json.dumps(response)
@ -1632,10 +1639,22 @@ class Home(MainHandler):
else:
ui.notifications.message('Mappings unchanged, not saving.')
master_ids = [show] + [kwargs.get(x) for x in 'indexer', 'mindexerid', 'mindexer']
if all([helpers.tryInt(x) > 0 for x in master_ids]):
master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted']
response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']}
master_ids = [show] + [helpers.tryInt(kwargs.get(x)) for x in 'indexer', 'mindexerid', 'mindexer']
if all([x > 0 for x in master_ids]) and sickbeard.indexerApi(kwargs['mindexer']).config.get('active') and \
not sickbeard.indexerApi(kwargs['mindexer']).config.get('defunct') and \
not sickbeard.indexerApi(kwargs['mindexer']).config.get('mapped_only') and \
(helpers.tryInt(kwargs['mindexer']) != helpers.tryInt(kwargs['indexer']) or
helpers.tryInt(kwargs['mindexerid']) != show):
try:
new_show_obj = helpers.find_show_by_id(sickbeard.showList, {helpers.tryInt(kwargs['mindexer']): helpers.tryInt(kwargs['mindexerid'])},no_mapped_ids=False)
if not new_show_obj or (new_show_obj.indexer == show_obj.indexer and new_show_obj.indexerid == show_obj.indexerid):
master_ids += [bool(helpers.tryInt(kwargs.get(x))) for x in 'paused', 'markwanted']
response = {'switch': self.switchIndexer(*master_ids), 'mid': kwargs['mindexerid']}
else:
ui.notifications.message('Master ID unchanged, because show from %s with ID: %s exists in DB.' %
(sickbeard.indexerApi(kwargs['mindexer']).name, kwargs['mindexerid']))
except MultipleShowObjectsException:
pass
response.update({
'map': {k: {r: w for r, w in v.iteritems() if r != 'date'} for k, v in show_obj.ids.iteritems()}
@ -1758,6 +1777,17 @@ class Home(MainHandler):
self.fanart_tmpl(t)
t.num_ratings = len(sickbeard.FANART_RATINGS.get(str(t.show.indexerid), {}))
show_message = ''
if showObj.not_found_count > 0:
# noinspection PyUnresolvedReferences
last_found = ('never', sbdatetime.sbdatetime.fromordinal(showObj.last_found_on_indexer).sbfdate())[
showObj.last_found_on_indexer > 1]
show_message = 'This show was not found (last time found: %s) on the Source Indexer' % last_found
t.show_message = show_message
return t.respond()
flatten_folders = config.checkbox_to_value(flatten_folders)
@ -4561,6 +4591,18 @@ class showProcesses(Manage):
t.showList = sickbeard.showList
t.ShowUpdateRunning = sickbeard.showQueueScheduler.action.isShowUpdateRunning() or sickbeard.showUpdateScheduler.action.amActive
myDb = db.DBConnection(row_type='dict')
sql_results = myDb.select('SELECT n.indexer, n.indexer_id, n.last_success, s.show_name FROM tv_shows_not_found as n INNER JOIN tv_shows as s ON (n.indexer == s.indexer AND n.indexer_id == s.indexer_id)')
for s in sql_results:
date = helpers.tryInt(s['last_success'])
s['last_success'] = ('never', sbdatetime.sbdatetime.fromordinal(date).sbfdate())[date > 1]
defunct_indexer = [i for i in sickbeard.indexerApi().all_indexers if sickbeard.indexerApi(i).config.get('defunct')]
sql_r = None
if defunct_indexer:
sql_r = myDb.select('SELECT indexer, indexer_id, show_name FROM tv_shows WHERE indexer IN (%s)' % ','.join(['?'] * len(defunct_indexer)), defunct_indexer)
t.DefunctIndexer = sql_r
t.NotFoundShows = sql_results
t.submenu = self.ManageMenu('Processes')
return t.respond()