Change showupdate add integrity safeguards and improve image processing. Add an episode view filter, improve WEB PROPER selection.

Change show update, don't delete any ep in DB if eps are not returned from indexer.
Change prevent unneeded error message during show update.

Change improve performance, don't fetch episode list when retrieving a show image.
Change don't remove episodes from DB with status: SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED.
Change add additional episode removal protections for TVDb_api v2.

Change filter SKIPPED items from episode view.
Change improve clarity of various error message by including relevant show name.
Change extend WEB PROPER release group check to ignore SD releases.

CHANGES.md

@@ -65,6 +65,14 @@
 * Change editshow saving empty scene exceptions
 * Change improve TVDB data handling
 * Change improve post processing by using more snatch history data
+* Change show update, don't delete any ep in DB if eps are not returned from indexer
+* Change prevent unneeded error message during show update
+* Change improve performance, don't fetch episode list when retrieving a show image
+* Change don't remove episodes from DB with status: SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED
+* Change add additional episode removal protections for TVDb_api v2
+* Change filter SKIPPED items from episode view
+* Change improve clarity of various error message by including relevant show name
+* Change extend WEB PROPER release group check to ignore SD releases
 
 
 [develop changelog]

sickbeard/helpers.py

@@ -54,7 +54,7 @@ except ImportError:
 
 from sickbeard.exceptions import MultipleShowObjectsException, ex
 from sickbeard import logger, db, notifiers, clients
-from sickbeard.common import USER_AGENT, mediaExtensions, subtitleExtensions, cpu_presets
+from sickbeard.common import USER_AGENT, mediaExtensions, subtitleExtensions, cpu_presets, statusStrings, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED, Quality
 from sickbeard import encodingKludge as ek
 
 from lib.cachecontrol import CacheControl, caches
@@ -1540,3 +1540,11 @@ def set_file_timestamp(filename, min_age=3, new_time=None):
             ek.ek(os.utime, filename, new_time)
     except (StandardError, Exception):
         pass
+
+
+def should_delete_episode(status):
+    s = Quality.splitCompositeStatus(status)
+    if s not in (SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED):
+        return True
+    logger.log('not safe to delete episode from db because of status: %s' % statusStrings[s], logger.DEBUG)
+    return False

sickbeard/metadata/generic.py

@@ -517,7 +517,7 @@ class GenericMetadata():
             logger.log(u"No thumb is available for this episode, not creating a thumb", logger.DEBUG)
             return False
 
-        thumb_data = metadata_helpers.getShowImage(thumb_url)
+        thumb_data = metadata_helpers.getShowImage(thumb_url, showName=ep_obj.show.name)
 
         result = self._write_image(thumb_data, file_path)
 
@@ -620,7 +620,7 @@ class GenericMetadata():
                            logger.DEBUG)
                 continue
 
-            seasonData = metadata_helpers.getShowImage(season_url)
+            seasonData = metadata_helpers.getShowImage(season_url, showName=show_obj.name)
 
             if not seasonData:
                 logger.log(u"No season poster data available, skipping this season", logger.DEBUG)
@@ -668,7 +668,7 @@ class GenericMetadata():
                            logger.DEBUG)
                 continue
 
-            seasonData = metadata_helpers.getShowImage(season_url)
+            seasonData = metadata_helpers.getShowImage(season_url, showName=show_obj.name)
 
             if not seasonData:
                 logger.log(u"No season banner data available, skipping this season", logger.DEBUG)
@@ -771,7 +771,7 @@ class GenericMetadata():
                 lINDEXER_API_PARMS['language'] = indexer_lang
 
             t = sickbeard.indexerApi(show_obj.indexer).indexer(**lINDEXER_API_PARMS)
-            indexer_show_obj = t[show_obj.indexerid]
+            indexer_show_obj = t[show_obj.indexerid, False]
         except (sickbeard.indexer_error, IOError) as e:
             logger.log(u"Unable to look up show on " + sickbeard.indexerApi(
                 show_obj.indexer).name + ", not downloading images: " + ex(e), logger.ERROR)
@@ -827,7 +827,7 @@ class GenericMetadata():
             if return_links:
                 return image_urls
             else:
-                image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which)
+                image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which, show_obj.name)
 
         if None is not image_data:
             return image_data

sickbeard/metadata/helpers.py

@@ -20,7 +20,7 @@ from sickbeard import helpers
 from sickbeard import logger
 
 
-def getShowImage(url, imgNum=None):
+def getShowImage(url, imgNum=None, showName=None):
 
     if None is url:
         return None
@@ -32,7 +32,7 @@ def getShowImage(url, imgNum=None):
 
     image_data = helpers.getURL(temp_url)
     if None is image_data:
-        logger.log(u'There was an error trying to retrieve the image, aborting', logger.ERROR)
+        logger.log('There was an error trying to retrieve the image%s, aborting' % ('', ' for show: %s' % showName)[None is not showName], logger.ERROR)
         return
 
     return image_data

sickbeard/properFinder.py

@@ -21,6 +21,7 @@ import operator
 import os
 import threading
 import traceback
+import re
 
 import sickbeard
 
@@ -165,7 +166,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime):
         # check if we actually want this proper (if it's the right quality)
         my_db = db.DBConnection()
         sql_results = my_db.select(
-            'SELECT release_group, status, version FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
+            'SELECT release_group, status, version, release_name FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
             [cur_proper.indexerid, cur_proper.season, cur_proper.episode])
         if not sql_results:
             continue
@@ -182,7 +183,8 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime):
 
         # for webldls, prevent propers from different groups
         if sickbeard.PROPERS_WEBDL_ONEGRP and \
-                old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) and \
+                (old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) or
+                     (old_quality == Quality.SDTV and re.search(r'\Wweb.?(dl|rip|.[hx]26[45])\W', str(sql_results[0]['release_name']), re.I))) and \
                 cur_proper.release_group != old_release_group:
             logger.log(log_same_grp, logger.DEBUG)
             continue

sickbeard/show_queue.py

@@ -22,12 +22,13 @@ import traceback
 
 import sickbeard
 
-from sickbeard.common import SKIPPED, WANTED, UNAIRED
+from sickbeard.common import SKIPPED, WANTED, UNAIRED, statusStrings
 from sickbeard.tv import TVShow
 from sickbeard import exceptions, logger, ui, db
 from sickbeard import generic_queue
 from sickbeard import name_cache
 from sickbeard.exceptions import ex
+from sickbeard.helpers import should_delete_episode
 from sickbeard.blackandwhitelist import BlackAndWhiteList
 
 
@@ -615,7 +616,7 @@ class QueueItemUpdate(ShowQueueItem):
         try:
             self.show.saveToDB()
         except Exception as e:
-            logger.log('Error saving the episode to the database: %s' % ex(e), logger.ERROR)
+            logger.log('Error saving the show to the database: %s' % ex(e), logger.ERROR)
             logger.log(traceback.format_exc(), logger.ERROR)
 
         # get episode list from DB
@@ -631,9 +632,12 @@ class QueueItemUpdate(ShowQueueItem):
                        (sickbeard.indexerApi(self.show.indexer).name, ex(e)), logger.ERROR)
             IndexerEpList = None
 
-        if IndexerEpList == None:
+        if None is IndexerEpList:
-            logger.log('No data returned from %s, unable to update this show' %
+            logger.log('No data returned from %s, unable to update episodes for show: %s' %
-                       sickbeard.indexerApi(self.show.indexer).name, logger.ERROR)
+                       (sickbeard.indexerApi(self.show.indexer).name, self.show.name), logger.ERROR)
+        elif not IndexerEpList or 0 == len(IndexerEpList):
+            logger.log('No episodes returned from %s for show: %s' %
+                       (sickbeard.indexerApi(self.show.indexer).name, self.show.name), logger.WARNING)
         else:
             # for each ep we found on TVDB delete it from the DB list
             for curSeason in IndexerEpList:
@@ -645,13 +649,18 @@ class QueueItemUpdate(ShowQueueItem):
             # for the remaining episodes in the DB list just delete them from the DB
             for curSeason in DBEpList:
                 for curEpisode in DBEpList[curSeason]:
-                    logger.log('Permanently deleting episode %sx%s from the database' %
-                               (curSeason, curEpisode), logger.MESSAGE)
                     curEp = self.show.getEpisode(curSeason, curEpisode)
-                    try:
+                    status = sickbeard.common.Quality.splitCompositeStatus(curEp.status)[0]
-                        curEp.deleteEpisode()
+                    if should_delete_episode(status):
-                    except exceptions.EpisodeDeletedException:
+                        logger.log('Permanently deleting episode %sx%s from the database' %
-                        pass
+                                   (curSeason, curEpisode), logger.MESSAGE)
+                        try:
+                            curEp.deleteEpisode()
+                        except exceptions.EpisodeDeletedException:
+                            pass
+                    else:
+                        logger.log('Not deleting episode %sx%s from the database because status is: %s' %
+                                   (curSeason, curEpisode, statusStrings[status]), logger.MESSAGE)
 
         if self.priority != generic_queue.QueuePriorities.NORMAL:
             self.kwargs['priority'] = self.priority

sickbeard/tv.py

@@ -484,7 +484,7 @@ class TVShow(object):
 
     def loadEpisodesFromDB(self, update=False):
 
-        logger.log('Loading all episodes from the DB')
+        logger.log('Loading all episodes for [%s] from the DB' % self.name)
 
         myDB = db.DBConnection()
         sql = 'SELECT * FROM tv_episodes WHERE showid = ? AND indexer = ?'
@@ -518,27 +518,27 @@ class TVShow(object):
                 try:
                     cachedSeasons[curSeason] = cachedShow[curSeason]
                 except sickbeard.indexer_seasonnotfound as e:
-                    logger.log('Error when trying to load the episode from %s: %s' %
+                    logger.log('Error when trying to load the episode for [%s] from %s: %s' %
-                               (sickbeard.indexerApi(self.indexer).name, e.message), logger.WARNING)
+                               (self.name, sickbeard.indexerApi(self.indexer).name, e.message), logger.WARNING)
                     deleteEp = True
 
             if not curSeason in scannedEps:
                 scannedEps[curSeason] = {}
 
-            logger.log('Loading episode %sx%s from the DB' % (curSeason, curEpisode), logger.DEBUG)
+            logger.log('Loading episode %sx%s for [%s] from the DB' % (curSeason, curEpisode, self.name), logger.DEBUG)
 
             try:
                 curEp = self.getEpisode(curSeason, curEpisode)
 
                 # if we found out that the ep is no longer on TVDB then delete it from our database too
-                if deleteEp:
+                if deleteEp and helpers.should_delete_episode(curEp.status):
                     curEp.deleteEpisode()
 
                 curEp.loadFromDB(curSeason, curEpisode)
                 curEp.loadFromIndexer(tvapi=t, cachedSeason=cachedSeasons[curSeason], update=update)
                 scannedEps[curSeason][curEpisode] = True
             except exceptions.EpisodeDeletedException:
-                logger.log('Tried loading an episode from the DB that should have been deleted, skipping it',
+                logger.log('Tried loading an episode from [%s] from the DB that should have been deleted, skipping it' % self.name,
                            logger.DEBUG)
                 continue
 
@@ -557,14 +557,14 @@ class TVShow(object):
         if self.dvdorder != 0:
             lINDEXER_API_PARMS['dvdorder'] = True
 
-        logger.log('%s: Loading all episodes from %s..' % (self.indexerid, sickbeard.indexerApi(self.indexer).name))
+        logger.log('%s: Loading all episodes for [%s] from %s..' % (self.indexerid, self.name, sickbeard.indexerApi(self.indexer).name))
 
         try:
             t = sickbeard.indexerApi(self.indexer).indexer(**lINDEXER_API_PARMS)
             showObj = t[self.indexerid]
         except sickbeard.indexer_error:
-            logger.log('%s timed out, unable to update episodes from %s' %
+            logger.log('%s timed out, unable to update episodes for [%s] from %s' %
-                       (sickbeard.indexerApi(self.indexer).name, sickbeard.indexerApi(self.indexer).name), logger.ERROR)
+                       (sickbeard.indexerApi(self.indexer).name, self.name, sickbeard.indexerApi(self.indexer).name), logger.ERROR)
             return None
 
         scannedEps = {}
@@ -579,19 +579,19 @@ class TVShow(object):
                 try:
                     ep = self.getEpisode(season, episode)
                 except exceptions.EpisodeNotFoundException:
-                    logger.log('%s: %s object for %sx%s is incomplete, skipping this episode' %
+                    logger.log('%s: %s object for %sx%s from [%s] is incomplete, skipping this episode' %
-                               (self.indexerid, sickbeard.indexerApi(self.indexer).name, season, episode))
+                               (self.indexerid, sickbeard.indexerApi(self.indexer).name, season, episode, self.name))
                     continue
                 else:
                     try:
                         ep.loadFromIndexer(tvapi=t, update=update)
                     except exceptions.EpisodeDeletedException:
-                        logger.log('The episode was deleted, skipping the rest of the load')
+                        logger.log('The episode from [%s] was deleted, skipping the rest of the load' % self.name)
                         continue
 
                 with ep.lock:
-                    logger.log('%s: Loading info from %s for episode %sx%s' %
+                    logger.log('%s: Loading info from %s for episode %sx%s from [%s]' %
-                               (self.indexerid, sickbeard.indexerApi(self.indexer).name, season, episode), logger.DEBUG)
+                               (self.indexerid, sickbeard.indexerApi(self.indexer).name, season, episode, self.name), logger.DEBUG)
                     ep.loadFromIndexer(season, episode, tvapi=t, update=update)
 
                     result = ep.get_sql()
@@ -779,10 +779,10 @@ class TVShow(object):
         sqlResults = myDB.select('SELECT * FROM tv_shows WHERE indexer_id = ?', [self.indexerid])
 
         if len(sqlResults) > 1:
-            logger.log('%s: Loading show info from database' % self.indexerid)
+            logger.log('%s: Loading show info [%s] from database' % (self.indexerid, self.name))
             raise exceptions.MultipleDBShowsException()
         elif len(sqlResults) == 0:
-            logger.log('%s: Unable to find the show in the database' % self.indexerid)
+            logger.log('%s: Unable to find the show [%s] in the database' % (self.indexerid, self.name))
             return
         else:
             if not self.indexer:
@@ -889,7 +889,7 @@ class TVShow(object):
 
     def loadFromIndexer(self, cache=True, tvapi=None, cachedSeason=None):
 
-        logger.log('%s: Loading show info from %s' % (self.indexerid, sickbeard.indexerApi(self.indexer).name))
+        logger.log('%s: Loading show info [%s] from %s' % (self.indexerid, self.name, sickbeard.indexerApi(self.indexer).name))
 
         # There's gotta be a better way of doing this but we don't wanna
         # change the cache value elsewhere
@@ -912,7 +912,7 @@ class TVShow(object):
 
         myEp = t[self.indexerid, False]
         if None is myEp:
-            logger.log('Show not found (maybe even removed?)', logger.WARNING)
+            logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING)
             return False
 
         try:
@@ -944,7 +944,7 @@ class TVShow(object):
 
         from lib.imdb import _exceptions as imdb_exceptions
 
-        logger.log('Retrieving show info from IMDb', logger.DEBUG)
+        logger.log('Retrieving show info [%s] from IMDb' % self.name, logger.DEBUG)
         try:
             self._get_imdb_info()
         except imdb_exceptions.IMDbDataAccessError as e:
@@ -1029,7 +1029,7 @@ class TVShow(object):
         logger.log('%s: Parsed latest IMDb show info for [%s]' % (self.indexerid, self.name))
 
     def nextEpisode(self):
-        logger.log('%s: Finding the episode which airs next' % self.indexerid, logger.DEBUG)
+        logger.log('%s: Finding the episode which airs next for: %s' % (self.indexerid, self.name), logger.DEBUG)
 
         curDate = datetime.date.today().toordinal()
         if not self.nextaired or self.nextaired and curDate > self.nextaired:
@@ -1135,7 +1135,7 @@ class TVShow(object):
         self.loadEpisodesFromDir()
 
         # run through all locations from DB, check that they exist
-        logger.log('%s: Loading all episodes with a location from the database' % self.indexerid)
+        logger.log('%s: Loading all episodes for [%s] with a location from the database' % (self.indexerid, self.name))
 
         myDB = db.DBConnection()
         sqlResults = myDB.select("SELECT * FROM tv_episodes WHERE showid = ? AND location != ''", [self.indexerid])
@@ -1149,7 +1149,7 @@ class TVShow(object):
             try:
                 curEp = self.getEpisode(season, episode)
             except exceptions.EpisodeDeletedException:
-                logger.log('The episode was deleted while we were refreshing it, moving on to the next one',
+                logger.log('The episode from [%s] was deleted while we were refreshing it, moving on to the next one' % self.name,
                            logger.DEBUG)
                 continue
 
@@ -1783,7 +1783,7 @@ class TVEpisode(object):
             logger.log('Unable to find the episode on %s... has it been removed? Should I delete from db?' %
                        sickbeard.indexerApi(self.indexer).name, logger.DEBUG)
             # if I'm no longer on the Indexers but I once was then delete myself from the DB
-            if -1 != self.indexerid:
+            if -1 != self.indexerid and helpers.should_delete_episode(self.status):
                 self.deleteEpisode()
             return
 
@@ -1827,7 +1827,7 @@ class TVEpisode(object):
             logger.log('Malformed air date retrieved from %s (%s - %sx%s)' %
                        (sickbeard.indexerApi(self.indexer).name, self.show.name, season, episode), logger.ERROR)
             # if I'm incomplete on TVDB but I once was complete then just delete myself from the DB for now
-            if -1 != self.indexerid:
+            if -1 != self.indexerid and helpers.should_delete_episode(self.status):
                 self.deleteEpisode()
             return False
 
@@ -1835,7 +1835,8 @@ class TVEpisode(object):
         self.indexerid = getattr(myEp, 'id', None)
         if None is self.indexerid:
             logger.log('Failed to retrieve ID from %s' % sickbeard.indexerApi(self.indexer).name, logger.ERROR)
-            self.deleteEpisode()
+            if helpers.should_delete_episode(self.status):
+                self.deleteEpisode()
             return False
 
         # don't update show status if show dir is missing, unless it's missing on purpose

sickbeard/webserve.py

@@ -447,7 +447,7 @@ class MainHandler(WebHandler):
         recently = (yesterday_dt - datetime.timedelta(days=sickbeard.EPISODE_VIEW_MISSED_RANGE)).toordinal()
 
         done_show_list = []
-        qualities = Quality.DOWNLOADED + Quality.SNATCHED + [ARCHIVED, IGNORED]
+        qualities = Quality.DOWNLOADED + Quality.SNATCHED + [ARCHIVED, IGNORED, SKIPPED]
 
         myDB = db.DBConnection()
         sql_results = myDB.select(
@@ -475,7 +475,7 @@ class MainHandler(WebHandler):
         # make a dict out of the sql results
         sql_results = [dict(row) for row in sql_results
                        if Quality.splitCompositeStatus(helpers.tryInt(row['status']))[0] not in
-                       [DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVED, IGNORED]]
+                       [DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVED, IGNORED, SKIPPED]]
 
         # multi dimension sort
         sorts = {