Change optimise TheTVDB processes, 40% to 66% saved adding new and existing shows, 40% to 50% saved per show update.

Change improve shows with more episodes gain largest reductions in time spent processing.
Change when using "Add new show" reduce search time outs.
Change always allow incomplete show data.
Remove redundant config/general/"Allow incomplete show data".

CHANGES.md

@@ -159,6 +159,11 @@
 * Change show loaded log message at start up and include info source
 * Change if episode has no airdate then set status to unaired (was skipped)
 * Fix only replace initial quality releases from the upgrade to list
+* Change optimise TheTVDB processes, 40% to 66% saved adding new and existing shows, 40% to 50% saved per show update
+* Change improve shows with more episodes gain largest reductions in time spent processing
+* Change when using "Add new show" reduce search time outs
+* Change always allow incomplete show data
+* Remove redundant config/general/"Allow incomplete show data"
 
 [develop changelog]
 * Change send nzb data to NZBGet for Anizb instead of url

gui/slick/interfaces/default/config_general.tmpl

@@ -88,18 +88,6 @@
 							</label>
 						</div>
 
-#if hasattr($sickbeard, 'ALLOW_INCOMPLETE_SHOWDATA')
-						<div class="field-pair">
-							<label for="allow_incomplete_showdata">
-								<span class="component-title">Allow incomplete show data</span>
-								<span class="component-desc">
-									<input type="checkbox" name="allow_incomplete_showdata" id="allow_incomplete_showdata"#echo ('', $checked)[$sickbeard.ALLOW_INCOMPLETE_SHOWDATA]#>
-									<p>add partial show data for future updates to complete</p>
-								</span>
-							</label>
-						</div>
-#end if
-
 						<div class="field-pair">
 							<span class="component-title">Send to trash for actions</span>
 							<span class="component-desc">

lib/etreetodict.py

@@ -0,0 +1,141 @@
+try:
+    from lxml import ElementTree
+except ImportError:
+    try:
+        import xml.etree.cElementTree as ElementTree
+    except ImportError:
+        import xml.etree.ElementTree as ElementTree
+
+
+class XmlDictObject(dict):
+    """
+    Adds object like functionality to the standard dictionary.
+    """
+
+    def __init__(self, initdict=None):
+        if initdict is None:
+            initdict = {}
+        dict.__init__(self, initdict)
+
+    def __getattr__(self, item):
+        return self.__getitem__(item)
+
+    def __setattr__(self, item, value):
+        self.__setitem__(item, value)
+
+    def __str__(self):
+        if self.has_key('_text'):
+            return self.__getitem__('_text')
+        else:
+            return ''
+
+    @staticmethod
+    def Wrap(x):
+        """
+        Static method to wrap a dictionary recursively as an XmlDictObject
+        """
+
+        if isinstance(x, dict):
+            return XmlDictObject((k, XmlDictObject.Wrap(v)) for (k, v) in x.iteritems())
+        elif isinstance(x, list):
+            return [XmlDictObject.Wrap(v) for v in x]
+        else:
+            return x
+
+    @staticmethod
+    def _UnWrap(x):
+        if isinstance(x, dict):
+            return dict((k, XmlDictObject._UnWrap(v)) for (k, v) in x.iteritems())
+        elif isinstance(x, list):
+            return [XmlDictObject._UnWrap(v) for v in x]
+        else:
+            return x
+
+    def UnWrap(self):
+        """
+        Recursively converts an XmlDictObject to a standard dictionary and returns the result.
+        """
+
+        return XmlDictObject._UnWrap(self)
+
+
+def _ConvertDictToXmlRecurse(parent, dictitem):
+    assert type(dictitem) is not type([])
+
+    if isinstance(dictitem, dict):
+        for (tag, child) in dictitem.iteritems():
+            if str(tag) == '_text':
+                parent.text = str(child)
+            elif type(child) is type([]):
+                # iterate through the array and convert
+                for listchild in child:
+                    elem = ElementTree.Element(tag)
+                    parent.append(elem)
+                    _ConvertDictToXmlRecurse(elem, listchild)
+            else:
+                elem = ElementTree.Element(tag)
+                parent.append(elem)
+                _ConvertDictToXmlRecurse(elem, child)
+    else:
+        parent.text = str(dictitem)
+
+
+def ConvertDictToXml(xmldict):
+    """
+    Converts a dictionary to an XML ElementTree Element
+    """
+
+    roottag = xmldict.keys()[0]
+    root = ElementTree.Element(roottag)
+    _ConvertDictToXmlRecurse(root, xmldict[roottag])
+    return root
+
+
+def _ConvertXmlToDictRecurse(node, dictclass):
+    nodedict = dictclass()
+
+    if len(node.items()) > 0:
+        # if we have attributes, set them
+        nodedict.update(dict(node.items()))
+
+    for child in node:
+        # recursively add the element's children
+        newitem = _ConvertXmlToDictRecurse(child, dictclass)
+        if nodedict.has_key(child.tag):
+            # found duplicate tag, force a list
+            if type(nodedict[child.tag]) is type([]):
+                # append to existing list
+                nodedict[child.tag].append(newitem)
+            else:
+                # convert to list
+                nodedict[child.tag] = [nodedict[child.tag], newitem]
+        else:
+            # only one, directly set the dictionary
+            nodedict[child.tag] = newitem
+
+    if node.text is None:
+        text = ''
+    else:
+        text = node.text.strip()
+
+    if len(nodedict) > 0:
+        # if we have a dictionary add the text as a dictionary value (if there is any)
+        if len(text) > 0:
+            nodedict['_text'] = text
+    else:
+        # if we don't have child nodes or attributes, just set the text
+        nodedict = text
+
+    return nodedict
+
+def ConvertXmlToDict(root, dictclass=XmlDictObject):
+    """
+    Converts an XML String to a dictionary
+    """
+
+    if isinstance(root, basestring):
+        root = ElementTree.fromstring(root)
+    elif not isinstance(root, ElementTree.Element):
+        raise TypeError, 'Expected string'
+
+    return dictclass({root.tag: _ConvertXmlToDictRecurse(root, dictclass)})

lib/tvdb_api/tvdb_api.py

@@ -5,8 +5,8 @@
 # repository:http://github.com/dbr/tvdb_api
 # license:unlicense (http://unlicense.org/)
 
-from functools import wraps
 import traceback
+from functools import wraps
 
 __author__ = 'dbr/Ben'
 __version__ = '1.9'
@@ -21,12 +21,6 @@ import logging
 import zipfile
 import requests
 import requests.exceptions
-import xmltodict
-
-try:
-    import xml.etree.cElementTree as ElementTree
-except ImportError:
-    import xml.etree.ElementTree as ElementTree
 
 try:
     import gzip
@@ -36,6 +30,7 @@ except ImportError:
 from lib.dateutil.parser import parse
 from lib.cachecontrol import CacheControl, caches
 
+from lib.etreetodict import ConvertXmlToDict
 from tvdb_ui import BaseUI, ConsoleUI
 from tvdb_exceptions import (tvdb_error, tvdb_shownotfound,
                              tvdb_seasonnotfound, tvdb_episodenotfound, tvdb_attributenotfound)
@@ -565,18 +560,17 @@ class Tvdb:
         except Exception:
             raise tvdb_error('Unknown exception while loading URL %s: %s' % (url, traceback.format_exc()))
 
-        def process(path, key, value):
-            key = key.lower()
-
-            # clean up value and do type changes
-            if value:
-                if 'firstaired' == key:
-                    try:
-                        value = parse(value, fuzzy=True).strftime('%Y-%m-%d')
-                    except:
-                        value = None
-
-            return key, value
+        def process_data(data):
+            te = ConvertXmlToDict(data)
+            if isinstance(te, dict) and 'Data' in te and isinstance(te['Data'], dict) and \
+                            'Series' in te['Data'] and isinstance(te['Data']['Series'], dict) and \
+                            'FirstAired' in te['Data']['Series']:
+                try:
+                    value = parse(te['Data']['Series']['FirstAired'], fuzzy=True).strftime('%Y-%m-%d')
+                except:
+                    value = None
+                te['Data']['Series']['firstaired'] = value
+            return te
 
         if resp.ok:
             if 'application/zip' in resp.headers.get('Content-Type', ''):
@@ -586,12 +580,12 @@ class Tvdb:
                     zipdata = StringIO.StringIO()
                     zipdata.write(resp.content)
                     myzipfile = zipfile.ZipFile(zipdata)
-                    return xmltodict.parse(myzipfile.read('%s.xml' % language), postprocessor=process)
+                    return process_data(myzipfile.read('%s.xml' % language))
                 except zipfile.BadZipfile:
                     raise tvdb_error('Bad zip file received from thetvdb.com, could not read it')
             else:
                 try:
-                    return xmltodict.parse(resp.content.strip(), postprocessor=process)
+                    return process_data(resp.content.strip())
                 except:
                     return dict([(u'data', None)])
 
@@ -641,7 +635,7 @@ class Tvdb:
         - Replaces & with &
         - Trailing whitespace
         """
-        return data if data is None else data.strip().replace(u'&', u'&')
+        return data if not isinstance(data, basestring) else data.strip().replace(u'&', u'&')
 
     def search(self, series):
         """This searches TheTVDB.com for the series name
@@ -654,6 +648,7 @@ class Tvdb:
         try:
             series_found = self._getetsrc(self.config['url_get_series'], self.config['params_get_series'])
             if series_found:
+                series_found['Series'] = [{k.lower(): v for k, v in s.iteritems()} for s in series_found['Series']]
                 return series_found.values()[0]
         except:
             pass
@@ -804,21 +799,21 @@ class Tvdb:
 
         # Parse show information
         log().debug('Getting all series data for %s' % sid)
-        url = self.config['url_epInfo%s' % ('', '_zip')[self.config['useZip']]] % (sid, language)
+        url = (self.config['url_seriesInfo'] % (sid, language), self.config['url_epInfo%s' % ('', '_zip')[self.config['useZip']]] % (sid, language))[get_ep_info]
         show_data = self._getetsrc(url, language=get_show_in_language)
 
         # check and make sure we have data to process and that it contains a series name
-        if not len(show_data) or (isinstance(show_data, dict) and 'seriesname' not in show_data['series']):
+        if not len(show_data) or (isinstance(show_data, dict) and 'SeriesName' not in show_data['Series']):
             return False
 
-        for k, v in show_data['series'].items():
+        for k, v in show_data['Series'].iteritems():
             if None is not v:
                 if k in ['banner', 'fanart', 'poster']:
                     v = self.config['url_artworkPrefix'] % v
                 else:
                     v = self._clean_data(v)
 
-            self._set_show_data(sid, k, v)
+            self._set_show_data(sid, k.lower(), v)
 
         if get_ep_info:
             # Parse banners
@@ -832,24 +827,24 @@ class Tvdb:
             # Parse episode data
             log().debug('Getting all episodes of %s' % sid)
 
-            if 'episode' not in show_data:
+            if 'Episode' not in show_data:
                 return False
 
-            episodes = show_data['episode']
+            episodes = show_data['Episode']
             if not isinstance(episodes, list):
                 episodes = [episodes]
 
             for cur_ep in episodes:
                 if self.config['dvdorder']:
                     log().debug('Using DVD ordering.')
-                    use_dvd = None is not cur_ep['dvd_season'] and None is not cur_ep['dvd_episodenumber']
+                    use_dvd = None is not cur_ep['DVD_season'] and None is not cur_ep['DVD_episodenumber']
                 else:
                     use_dvd = False
 
                 if use_dvd:
-                    elem_seasnum, elem_epno = cur_ep['dvd_season'], cur_ep['dvd_episodenumber']
+                    elem_seasnum, elem_epno = cur_ep['DVD_season'], cur_ep['DVD_episodenumber']
                 else:
-                    elem_seasnum, elem_epno = cur_ep['seasonnumber'], cur_ep['episodenumber']
+                    elem_seasnum, elem_epno = cur_ep['SeasonNumber'], cur_ep['EpisodeNumber']
 
                 if None is elem_seasnum or None is elem_epno:
                     log().warning('An episode has incomplete season/episode number (season: %r, episode: %r)' % (
@@ -895,10 +890,16 @@ class Tvdb:
         """Handles tvdb_instance['seriesname'] calls.
         The dict index should be the show id
         """
+        arg = None
+        if isinstance(key, tuple) and 2 == len(key):
+            key, arg = key
+            if not isinstance(arg, bool):
+                arg = None
+
         if isinstance(key, (int, long)):
             # Item is integer, treat as show id
             if key not in self.shows:
-                self._get_show_data(key, self.config['language'], True)
+                self._get_show_data(key, self.config['language'], (True, arg)[arg is not None])
             return None if key not in self.shows else self.shows[key]
 
         key = str(key).lower()

lib/tvrage_api/tvrage_api.py

@@ -657,10 +657,16 @@ class TVRage:
         """Handles tvrage_instance['seriesname'] calls.
         The dict index should be the show id
         """
+        arg = None
+        if isinstance(key, tuple) and 2 == len(key):
+            key, arg = key
+            if not isinstance(arg, bool):
+                arg = None
+
         if isinstance(key, (int, long)):
             # Item is integer, treat as show id
             if key not in self.shows:
-                self._getShowData(key, True)
+                self._getShowData(key, (True, arg)[arg is not None])
             return None if key not in self.shows else self.shows[key]
 
         key = key.lower()

sickbeard/__init__.py

@@ -89,7 +89,6 @@ background_mapping_task = None
 showList = None
 UPDATE_SHOWS_ON_START = False
 SHOW_UPDATE_HOUR = 3
-ALLOW_INCOMPLETE_SHOWDATA = False
 
 providerList = []
 newznabProviderList = []
@@ -514,7 +513,7 @@ def initialize(consoleLogging=True):
             PLEX_UPDATE_LIBRARY, PLEX_SERVER_HOST, PLEX_HOST, PLEX_USERNAME, PLEX_PASSWORD, \
             USE_TRAKT, TRAKT_CONNECTED_ACCOUNT, TRAKT_ACCOUNTS, TRAKT_MRU, TRAKT_VERIFY, TRAKT_REMOVE_WATCHLIST, TRAKT_TIMEOUT, TRAKT_USE_WATCHLIST, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, TRAKT_SYNC, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_UPDATE_COLLECTION, \
             BACKLOG_FREQUENCY, DEFAULT_BACKLOG_FREQUENCY, MIN_BACKLOG_FREQUENCY, MAX_BACKLOG_FREQUENCY, BACKLOG_STARTUP, SKIP_REMOVED_FILES, \
-            showUpdateScheduler, __INITIALIZED__, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, HOME_SEARCH_FOCUS, SORT_ARTICLE, showList, loadingShowList, UPDATE_SHOWS_ON_START, SHOW_UPDATE_HOUR, ALLOW_INCOMPLETE_SHOWDATA, \
+            showUpdateScheduler, __INITIALIZED__, LAUNCH_BROWSER, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, HOME_SEARCH_FOCUS, SORT_ARTICLE, showList, loadingShowList, UPDATE_SHOWS_ON_START, SHOW_UPDATE_HOUR, \
             NEWZNAB_DATA, INDEXER_DEFAULT, INDEXER_TIMEOUT, USENET_RETENTION, TORRENT_DIR, \
             QUALITY_DEFAULT, FLATTEN_FOLDERS_DEFAULT, SUBTITLES_DEFAULT, STATUS_DEFAULT, WANTED_BEGIN_DEFAULT, WANTED_LATEST_DEFAULT, RECENTSEARCH_STARTUP, \
             GROWL_NOTIFY_ONSNATCH, GROWL_NOTIFY_ONDOWNLOAD, GROWL_NOTIFY_ONSUBTITLEDOWNLOAD, TWITTER_NOTIFY_ONSNATCH, TWITTER_NOTIFY_ONDOWNLOAD, TWITTER_NOTIFY_ONSUBTITLEDOWNLOAD, \
@@ -666,7 +665,6 @@ def initialize(consoleLogging=True):
         UPDATE_SHOWS_ON_START = bool(check_setting_int(CFG, 'General', 'update_shows_on_start', 0))
         SHOW_UPDATE_HOUR = check_setting_int(CFG, 'General', 'show_update_hour', 3)
         SHOW_UPDATE_HOUR = minimax(SHOW_UPDATE_HOUR, 3, 0, 23)
-        ALLOW_INCOMPLETE_SHOWDATA = bool(check_setting_int(CFG, 'General', 'allow_incomplete_showdata', 0))
 
         TRASH_REMOVE_SHOW = bool(check_setting_int(CFG, 'General', 'trash_remove_show', 0))
         TRASH_ROTATE_LOGS = bool(check_setting_int(CFG, 'General', 'trash_rotate_logs', 0))
@@ -1524,7 +1522,6 @@ def save_config():
     new_config['General']['launch_browser'] = int(LAUNCH_BROWSER)
     new_config['General']['update_shows_on_start'] = int(UPDATE_SHOWS_ON_START)
     new_config['General']['show_update_hour'] = int(SHOW_UPDATE_HOUR)
-    new_config['General']['allow_incomplete_showdata'] = int(ALLOW_INCOMPLETE_SHOWDATA)
     new_config['General']['trash_remove_show'] = int(TRASH_REMOVE_SHOW)
     new_config['General']['trash_rotate_logs'] = int(TRASH_ROTATE_LOGS)
     new_config['General']['home_search_focus'] = int(HOME_SEARCH_FOCUS)

sickbeard/show_queue.py

@@ -296,7 +296,7 @@ class QueueItemAdd(ShowQueueItem):
             logger.log(u'' + str(sickbeard.indexerApi(self.indexer).name) + ': ' + repr(lINDEXER_API_PARMS))
 
             t = sickbeard.indexerApi(self.indexer).indexer(**lINDEXER_API_PARMS)
-            s = t[self.indexer_id]
+            s = t[self.indexer_id, False]
 
             # this usually only happens if they have an NFO in their show dir which gave us a Indexer ID that has no proper english version of the show
             if getattr(s, 'seriesname', None) is None:
@@ -307,14 +307,6 @@ class QueueItemAdd(ShowQueueItem):
                                        (self.showDir, sickbeard.indexerApi(self.indexer).name))
                 self._finishEarly()
                 return
-            # if the show has no episodes/seasons
-            if not sickbeard.ALLOW_INCOMPLETE_SHOWDATA and not s:
-                msg = 'Show %s is on %s but contains no season/episode data. Only the show folder was created.'\
-                      % (s['seriesname'], sickbeard.indexerApi(self.indexer).name)
-                logger.log(msg, logger.ERROR)
-                ui.notifications.error('Unable to add show', msg)
-                self._finishEarly()
-                return
         except Exception as e:
             logger.log('Unable to find show ID:%s on Indexer: %s' % (self.indexer_id, sickbeard.indexerApi(self.indexer).name),
                        logger.ERROR)

sickbeard/tv.py

@@ -559,6 +559,8 @@ class TVShow(object):
         if self.dvdorder != 0:
             lINDEXER_API_PARMS['dvdorder'] = True
 
+        logger.log('%s: Loading all episodes from %s..' % (self.indexerid, sickbeard.indexerApi(self.indexer).name))
+
         try:
             t = sickbeard.indexerApi(self.indexer).indexer(**lINDEXER_API_PARMS)
             showObj = t[self.indexerid]
@@ -567,8 +569,6 @@ class TVShow(object):
                        (sickbeard.indexerApi(self.indexer).name, sickbeard.indexerApi(self.indexer).name), logger.ERROR)
             return None
 
-        logger.log('%s: Loading all episodes from %s..' % (self.indexerid, sickbeard.indexerApi(self.indexer).name))
-
         scannedEps = {}
 
         sql_l = []
@@ -912,7 +912,7 @@ class TVShow(object):
         else:
             t = tvapi
 
-        myEp = t[self.indexerid]
+        myEp = t[self.indexerid, False]
         if None is myEp:
             logger.log('Show not found (maybe even removed?)', logger.WARNING)
             return False
@@ -931,7 +931,7 @@ class TVShow(object):
         self.imdbid = getattr(myEp, 'imdb_id', '')
 
         if getattr(myEp, 'airs_dayofweek', None) is not None and getattr(myEp, 'airs_time', None) is not None:
-            self.airs = myEp["airs_dayofweek"] + " " + myEp["airs_time"]
+            self.airs = ('%s %s' % (myEp['airs_dayofweek'], myEp['airs_time'])).strip()
 
         if getattr(myEp, 'firstaired', None) is not None:
             self.startyear = int(str(myEp["firstaired"]).split('-')[0])
@@ -1076,15 +1076,15 @@ class TVShow(object):
         for path, dirs, files in ek.ek(os.walk, image_cache_dir):
             for filename in ek.ek(fnmatch.filter, files, '%s.*' % self.indexerid):
                 cache_file = ek.ek(os.path.join, path, filename)
-                logger.log('Attempt to %s cache file %s' % (action, cache_file))
-                try:
-                    if sickbeard.TRASH_REMOVE_SHOW:
-                        send2trash(cache_file)
-                    else:
-                        os.remove(cache_file)
+            logger.log('Attempt to %s cache file %s' % (action, cache_file))
+            try:
+                if sickbeard.TRASH_REMOVE_SHOW:
+                    send2trash(cache_file)
+                else:
+                    os.remove(cache_file)
 
-                except OSError as e:
-                    logger.log('Unable to %s %s: %s / %s' % (action, cache_file, repr(e), str(e)), logger.WARNING)
+            except OSError as e:
+                logger.log('Unable to %s %s: %s / %s' % (action, cache_file, repr(e), str(e)), logger.WARNING)
 
         # remove entire show folder
         if full:
@@ -1771,15 +1771,7 @@ class TVEpisode(object):
                 self.deleteEpisode()
             return
 
-        if not sickbeard.ALLOW_INCOMPLETE_SHOWDATA and None is getattr(myEp, 'episodename', None):
-            logger.log('This episode (%s - %sx%s) has no name on %s' %
-                       (self.show.name, season, episode, sickbeard.indexerApi(self.indexer).name))
-            # if I'm incomplete on TVDB but I once was complete then just delete myself from the DB for now
-            if -1 != self.indexerid:
-                self.deleteEpisode()
-            return False
-
-        if None is getattr(myEp, 'absolute_number', None):
+        if getattr(myEp, 'absolute_number', None) in (None, ''):
             logger.log('This episode (%s - %sx%s) has no absolute number on %s' %
                        (self.show.name, season, episode, sickbeard.indexerApi(self.indexer).name), logger.DEBUG)
         else:
@@ -1827,8 +1819,7 @@ class TVEpisode(object):
         self.indexerid = getattr(myEp, 'id', None)
         if None is self.indexerid:
             logger.log('Failed to retrieve ID from %s' % sickbeard.indexerApi(self.indexer).name, logger.ERROR)
-            if -1 != self.indexerid:
-                self.deleteEpisode()
+            self.deleteEpisode()
             return False
 
         # don't update show status if show dir is missing, unless it's missing on purpose
@@ -2060,42 +2051,26 @@ class TVEpisode(object):
             logger.log('%s: Not creating SQL queue - record is not dirty' % self.show.indexerid, logger.DEBUG)
             return
 
-        myDB = db.DBConnection()
-        rows = myDB.select(
-            'SELECT episode_id FROM tv_episodes WHERE showid = ? AND indexer=? AND season = ? AND episode = ?',
-            [self.show.indexerid, self.show.indexer, self.season, self.episode])
-
-        epID = None
-        if rows:
-            epID = int(rows[0]['episode_id'])
-
         self.dirty = False
-        if epID:
-            # use a custom update method to get the data into the DB for existing records.
-            return [
-                'UPDATE tv_episodes SET indexerid = ?, indexer = ?, name = ?, description = ?, subtitles = ?, '
-                'subtitles_searchcount = ?, subtitles_lastsearch = ?, airdate = ?, hasnfo = ?, hastbn = ?, status = ?, '
-                'location = ?, file_size = ?, release_name = ?, is_proper = ?, showid = ?, season = ?, episode = ?, '
-                'absolute_number = ?, version = ?, release_group = ? WHERE episode_id = ?',
-                [self.indexerid, self.indexer, self.name, self.description, ",".join([sub for sub in self.subtitles]),
-                 self.subtitles_searchcount, self.subtitles_lastsearch, self.airdate.toordinal(), self.hasnfo,
-                 self.hastbn,
-                 self.status, self.location, self.file_size, self.release_name, self.is_proper, self.show.indexerid,
-                 self.season, self.episode, self.absolute_number, self.version, self.release_group, epID]]
-        else:
-            # use a custom insert method to get the data into the DB.
-            return [
-                'INSERT OR IGNORE INTO tv_episodes (episode_id, indexerid, indexer, name, description, subtitles, '
-                'subtitles_searchcount, subtitles_lastsearch, airdate, hasnfo, hastbn, status, location, file_size, '
-                'release_name, is_proper, showid, season, episode, absolute_number, version, release_group) VALUES '
-                '((SELECT episode_id FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?)'
-                ',?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);',
-                [self.show.indexerid, self.season, self.episode, self.indexerid, self.indexer, self.name,
-                 self.description,
-                 ",".join([sub for sub in self.subtitles]), self.subtitles_searchcount, self.subtitles_lastsearch,
-                 self.airdate.toordinal(), self.hasnfo, self.hastbn, self.status, self.location, self.file_size,
-                 self.release_name, self.is_proper, self.show.indexerid, self.season, self.episode,
-                 self.absolute_number, self.version, self.release_group]]
+        return [
+            'INSERT OR REPLACE INTO tv_episodes (episode_id, indexerid, indexer, name, description, subtitles, '
+            'subtitles_searchcount, subtitles_lastsearch, airdate, hasnfo, hastbn, status, location, file_size, '
+            'release_name, is_proper, showid, season, episode, absolute_number, version, release_group, '
+            'scene_absolute_number, scene_season, scene_episode) VALUES '
+            '((SELECT episode_id FROM tv_episodes WHERE indexer = ? AND showid = ? AND season = ? AND episode = ?)'
+            ',?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,'
+            '(SELECT scene_absolute_number FROM tv_episodes WHERE indexer = ? AND showid = ? AND season = ? AND episode = ?),'
+            '(SELECT scene_season FROM tv_episodes WHERE indexer = ? AND showid = ? AND season = ? AND episode = ?),'
+            '(SELECT scene_episode FROM tv_episodes WHERE indexer = ? AND showid = ? AND season = ? AND episode = ?));',
+            [self.show.indexer, self.show.indexerid, self.season, self.episode, self.indexerid, self.indexer, self.name,
+             self.description,
+             ",".join([sub for sub in self.subtitles]), self.subtitles_searchcount, self.subtitles_lastsearch,
+             self.airdate.toordinal(), self.hasnfo, self.hastbn, self.status, self.location, self.file_size,
+             self.release_name, self.is_proper, self.show.indexerid, self.season, self.episode,
+             self.absolute_number, self.version, self.release_group,
+             self.show.indexer, self.show.indexerid, self.season, self.episode,
+             self.show.indexer, self.show.indexerid, self.season, self.episode,
+             self.show.indexer, self.show.indexerid, self.season, self.episode]]
 
     def saveToDB(self, forceSave=False):
         """

sickbeard/webserve.py

@@ -2368,11 +2368,11 @@ class NewHomeAddShows(Home):
                     logger.log('Fetching show using id: %s (%s) from tv datasource %s' % (
                         search_id, search_term, sickbeard.indexerApi(indexer).name), logger.DEBUG)
                     results.setdefault('tt' in search_id and 3 or indexer, []).extend(
-                        [{'id': indexer_id, 'seriesname': t[indexer_id]['seriesname'],
-                          'firstaired': t[indexer_id]['firstaired'], 'network': t[indexer_id]['network'],
-                          'overview': t[indexer_id]['overview'],
-                          'genres': '' if not t[indexer_id]['genre'] else
-                            t[indexer_id]['genre'].lower().strip('|').replace('|', ', '),
+                        [{'id': indexer_id, 'seriesname': t[indexer_id, False]['seriesname'],
+                          'firstaired': t[indexer_id, False]['firstaired'], 'network': t[indexer_id, False]['network'],
+                          'overview': t[indexer_id, False]['overview'],
+                          'genres': '' if not t[indexer_id, False]['genre'] else
+                            t[indexer_id, False]['genre'].lower().strip('|').replace('|', ', '),
                           }])
                     break
                 else:
@@ -4305,7 +4305,7 @@ class ConfigGeneral(Config):
         return m.hexdigest()
 
     def saveGeneral(self, log_dir=None, web_port=None, web_log=None, encryption_version=None, web_ipv6=None,
-                    update_shows_on_start=None, show_update_hour=None, allow_incomplete_showdata=None,
+                    update_shows_on_start=None, show_update_hour=None,
                     trash_remove_show=None, trash_rotate_logs=None, update_frequency=None, launch_browser=None, web_username=None,
                     use_api=None, api_key=None, indexer_default=None, timezone_display=None, cpu_preset=None, file_logging_preset=None,
                     web_password=None, version_notify=None, enable_https=None, https_cert=None, https_key=None,
@@ -4327,7 +4327,6 @@ class ConfigGeneral(Config):
 
         sickbeard.UPDATE_SHOWS_ON_START = config.checkbox_to_value(update_shows_on_start)
         sickbeard.SHOW_UPDATE_HOUR = config.minimax(show_update_hour, 3, 0, 23)
-        sickbeard.ALLOW_INCOMPLETE_SHOWDATA = config.checkbox_to_value(allow_incomplete_showdata)
         sickbeard.TRASH_REMOVE_SHOW = config.checkbox_to_value(trash_remove_show)
         sickbeard.TRASH_ROTATE_LOGS = config.checkbox_to_value(trash_rotate_logs)
         config.change_UPDATE_FREQUENCY(update_frequency)