Change showupdate add integrity safeguards and improve image processing. Add an episode view filter, improve WEB PROPER selection.

Change show update, don't delete any ep in DB if eps are not returned from indexer.
Change prevent unneeded error message during show update.

Change improve performance, don't fetch episode list when retrieving a show image.
Change don't remove episodes from DB with status: SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED.
Change add additional episode removal protections for TVDb_api v2.

Change filter SKIPPED items from episode view.
Change improve clarity of various error message by including relevant show name.
Change extend WEB PROPER release group check to ignore SD releases.

CHANGES.md

@@ -64,6 +64,14 @@
 * Change only use newznab Api key if needed
 * Change editshow saving empty scene exceptions
 * Change improve TVDB data handling
+* Change show update, don't delete any ep in DB if eps are not returned from indexer
+* Change prevent unneeded error message during show update
+* Change improve performance, don't fetch episode list when retrieving a show image
+* Change don't remove episodes from DB with status: SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED
+* Change add additional episode removal protections for TVDb_api v2
+* Change filter SKIPPED items from episode view
+* Change improve clarity of various error message by including relevant show name
+* Change extend WEB PROPER release group check to ignore SD releases
 
 
 [develop changelog]

sickbeard/helpers.py

@@ -54,7 +54,7 @@ except ImportError:
 
 from sickbeard.exceptions import MultipleShowObjectsException, ex
 from sickbeard import logger, db, notifiers, clients
-from sickbeard.common import USER_AGENT, mediaExtensions, subtitleExtensions, cpu_presets
+from sickbeard.common import USER_AGENT, mediaExtensions, subtitleExtensions, cpu_presets, statusStrings, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED, Quality
 from sickbeard import encodingKludge as ek
 
 from lib.cachecontrol import CacheControl, caches
@@ -1540,3 +1540,11 @@ def set_file_timestamp(filename, min_age=3, new_time=None):
             ek.ek(os.utime, filename, new_time)
     except (StandardError, Exception):
         pass
+
+
+def should_delete_episode(status):
+    s = Quality.splitCompositeStatus(status)
+    if s not in (SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED, ARCHIVED, IGNORED):
+        return True
+    logger.log('not safe to delete episode from db because of status: %s' % statusStrings[s], logger.DEBUG)
+    return False

sickbeard/metadata/generic.py

@@ -517,7 +517,7 @@ class GenericMetadata():
             logger.log(u"No thumb is available for this episode, not creating a thumb", logger.DEBUG)
             return False
 
-        thumb_data = metadata_helpers.getShowImage(thumb_url)
+        thumb_data = metadata_helpers.getShowImage(thumb_url, showName=ep_obj.show.name)
 
         result = self._write_image(thumb_data, file_path)
 
@@ -620,7 +620,7 @@ class GenericMetadata():
                            logger.DEBUG)
                 continue
 
-            seasonData = metadata_helpers.getShowImage(season_url)
+            seasonData = metadata_helpers.getShowImage(season_url, showName=show_obj.name)
 
             if not seasonData:
                 logger.log(u"No season poster data available, skipping this season", logger.DEBUG)
@@ -668,7 +668,7 @@ class GenericMetadata():
                            logger.DEBUG)
                 continue
 
-            seasonData = metadata_helpers.getShowImage(season_url)
+            seasonData = metadata_helpers.getShowImage(season_url, showName=show_obj.name)
 
             if not seasonData:
                 logger.log(u"No season banner data available, skipping this season", logger.DEBUG)
@@ -771,7 +771,7 @@ class GenericMetadata():
                 lINDEXER_API_PARMS['language'] = indexer_lang
 
             t = sickbeard.indexerApi(show_obj.indexer).indexer(**lINDEXER_API_PARMS)
-            indexer_show_obj = t[show_obj.indexerid]
+            indexer_show_obj = t[show_obj.indexerid, False]
         except (sickbeard.indexer_error, IOError) as e:
             logger.log(u"Unable to look up show on " + sickbeard.indexerApi(
                 show_obj.indexer).name + ", not downloading images: " + ex(e), logger.ERROR)
@@ -827,7 +827,7 @@ class GenericMetadata():
             if return_links:
                 return image_urls
             else:
-                image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which)
+                image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which, show_obj.name)
 
         if None is not image_data:
             return image_data

sickbeard/metadata/helpers.py

@@ -20,7 +20,7 @@ from sickbeard import helpers
 from sickbeard import logger
 
 
-def getShowImage(url, imgNum=None):
+def getShowImage(url, imgNum=None, showName=None):
 
     if None is url:
         return None
@@ -32,7 +32,7 @@ def getShowImage(url, imgNum=None):
 
     image_data = helpers.getURL(temp_url)
     if None is image_data:
-        logger.log(u'There was an error trying to retrieve the image, aborting', logger.ERROR)
+        logger.log('There was an error trying to retrieve the image%s, aborting' % ('', ' for show: %s' % showName)[None is not showName], logger.ERROR)
         return
 
     return image_data

sickbeard/properFinder.py

@@ -21,6 +21,7 @@ import operator
 import os
 import threading
 import traceback
+import re
 
 import sickbeard
 
@@ -165,7 +166,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime):
         # check if we actually want this proper (if it's the right quality)
         my_db = db.DBConnection()
         sql_results = my_db.select(
-            'SELECT release_group, status, version FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
+            'SELECT release_group, status, version, release_name FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
             [cur_proper.indexerid, cur_proper.season, cur_proper.episode])
         if not sql_results:
             continue
@@ -182,7 +183,8 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime):
 
         # for webldls, prevent propers from different groups
         if sickbeard.PROPERS_WEBDL_ONEGRP and \
-                old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) and \
+                (old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) or
+                     (old_quality == Quality.SDTV and re.search(r'\Wweb.?(dl|rip|.[hx]26[45])\W', str(sql_results[0]['release_name']), re.I))) and \
                 cur_proper.release_group != old_release_group:
             logger.log(log_same_grp, logger.DEBUG)
             continue

sickbeard/show_queue.py

@@ -22,12 +22,13 @@ import traceback
 
 import sickbeard
 
-from sickbeard.common import SKIPPED, WANTED, UNAIRED
+from sickbeard.common import SKIPPED, WANTED, UNAIRED, statusStrings
 from sickbeard.tv import TVShow
 from sickbeard import exceptions, logger, ui, db
 from sickbeard import generic_queue
 from sickbeard import name_cache
 from sickbeard.exceptions import ex
+from sickbeard.helpers import should_delete_episode
 from sickbeard.blackandwhitelist import BlackAndWhiteList
 
 
@@ -615,7 +616,7 @@ class QueueItemUpdate(ShowQueueItem):
         try:
             self.show.saveToDB()
         except Exception as e:
-            logger.log('Error saving the episode to the database: %s' % ex(e), logger.ERROR)
+            logger.log('Error saving the show to the database: %s' % ex(e), logger.ERROR)
             logger.log(traceback.format_exc(), logger.ERROR)
 
         # get episode list from DB
@@ -631,9 +632,12 @@ class QueueItemUpdate(ShowQueueItem):
                        (sickbeard.indexerApi(self.show.indexer).name, ex(e)), logger.ERROR)
             IndexerEpList = None
 
-        if IndexerEpList == None:
+        if None is IndexerEpList:
-            logger.log('No data returned from %s, unable to update this show' %
+            logger.log('No data returned from %s, unable to update episodes for show: %s' %
-                       sickbeard.indexerApi(self.show.indexer).name, logger.ERROR)
+                       (sickbeard.indexerApi(self.show.indexer).name, self.show.name), logger.ERROR)
+        elif not IndexerEpList or 0 == len(IndexerEpList):
+            logger.log('No episodes returned from %s for show: %s' %
+                       (sickbeard.indexerApi(self.show.indexer).name, self.show.name), logger.WARNING)
         else:
             # for each ep we found on TVDB delete it from the DB list
             for curSeason in IndexerEpList:
@@ -645,13 +649,18 @@ class QueueItemUpdate(ShowQueueItem):
             # for the remaining episodes in the DB list just delete them from the DB
             for curSeason in DBEpList:
                 for curEpisode in DBEpList[curSeason]:
-                    logger.log('Permanently deleting episode %sx%s from the database' %
-                               (curSeason, curEpisode), logger.MESSAGE)
                     curEp = self.show.getEpisode(curSeason, curEpisode)
-                    try:
+                    status = sickbeard.common.Quality.splitCompositeStatus(curEp.status)[0]
-                        curEp.deleteEpisode()
+                    if should_delete_episode(status):
-                    except exceptions.EpisodeDeletedException:
+                        logger.log('Permanently deleting episode %sx%s from the database' %
-                        pass
+                                   (curSeason, curEpisode), logger.MESSAGE)
+                        try:
+                            curEp.deleteEpisode()
+                        except exceptions.EpisodeDeletedException:
+                            pass
+                    else:
+                        logger.log('Not deleting episode %sx%s from the database because status is: %s' %
+                                   (curSeason, curEpisode, statusStrings[status]), logger.MESSAGE)
 
         if self.priority != generic_queue.QueuePriorities.NORMAL:
             self.kwargs['priority'] = self.priority

sickbeard/tv.py

@@ -531,7 +531,7 @@ class TVShow(object):
                 curEp = self.getEpisode(curSeason, curEpisode)
 
                 # if we found out that the ep is no longer on TVDB then delete it from our database too
-                if deleteEp:
+                if deleteEp and helpers.should_delete_episode(curEp.status):
                     curEp.deleteEpisode()
 
                 curEp.loadFromDB(curSeason, curEpisode)
@@ -912,7 +912,7 @@ class TVShow(object):
 
         myEp = t[self.indexerid, False]
         if None is myEp:
-            logger.log('Show not found (maybe even removed?)', logger.WARNING)
+            logger.log('Show [%s] not found (maybe even removed?)' % self.name, logger.WARNING)
             return False
 
         try:
@@ -1783,7 +1783,7 @@ class TVEpisode(object):
             logger.log('Unable to find the episode on %s... has it been removed? Should I delete from db?' %
                        sickbeard.indexerApi(self.indexer).name, logger.DEBUG)
             # if I'm no longer on the Indexers but I once was then delete myself from the DB
-            if -1 != self.indexerid:
+            if -1 != self.indexerid and helpers.should_delete_episode(self.status):
                 self.deleteEpisode()
             return
 
@@ -1827,7 +1827,7 @@ class TVEpisode(object):
             logger.log('Malformed air date retrieved from %s (%s - %sx%s)' %
                        (sickbeard.indexerApi(self.indexer).name, self.show.name, season, episode), logger.ERROR)
             # if I'm incomplete on TVDB but I once was complete then just delete myself from the DB for now
-            if -1 != self.indexerid:
+            if -1 != self.indexerid and helpers.should_delete_episode(self.status):
                 self.deleteEpisode()
             return False
 
@@ -1835,7 +1835,8 @@ class TVEpisode(object):
         self.indexerid = getattr(myEp, 'id', None)
         if None is self.indexerid:
             logger.log('Failed to retrieve ID from %s' % sickbeard.indexerApi(self.indexer).name, logger.ERROR)
-            self.deleteEpisode()
+            if helpers.should_delete_episode(self.status):
+                self.deleteEpisode()
             return False
 
         # don't update show status if show dir is missing, unless it's missing on purpose

sickbeard/webserve.py

@@ -447,7 +447,7 @@ class MainHandler(WebHandler):
         recently = (yesterday_dt - datetime.timedelta(days=sickbeard.EPISODE_VIEW_MISSED_RANGE)).toordinal()
 
         done_show_list = []
-        qualities = Quality.DOWNLOADED + Quality.SNATCHED + [ARCHIVED, IGNORED]
+        qualities = Quality.DOWNLOADED + Quality.SNATCHED + [ARCHIVED, IGNORED, SKIPPED]
 
         myDB = db.DBConnection()
         sql_results = myDB.select(
@@ -475,7 +475,7 @@ class MainHandler(WebHandler):
         # make a dict out of the sql results
         sql_results = [dict(row) for row in sql_results
                        if Quality.splitCompositeStatus(helpers.tryInt(row['status']))[0] not in
-                       [DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVED, IGNORED]]
+                       [DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, ARCHIVED, IGNORED, SKIPPED]]
 
         # multi dimension sort
         sorts = {