From d47ad5f332d3e70188d95277e5339cc3e8e9c3cb Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 19 May 2015 22:38:33 +0800 Subject: [PATCH] Change to only rebuild namecache on show update instead of on every search Add removal of old entries from namecache on show deletion --- CHANGES.md | 2 + sickbeard/name_cache.py | 125 ++++++++++++++------------------------- sickbeard/search.py | 3 - sickbeard/showUpdater.py | 3 + sickbeard/show_queue.py | 6 +- sickbeard/tv.py | 4 +- 6 files changed, 56 insertions(+), 87 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2d0a8a52..f7094a67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,8 @@ * Update Requests library 2.6.2 to 2.7.0 (ab1f493) * Update change to suppress HTTPS verification InsecureRequestWarning to updated package (ref:hacks.txt) * Change to consolidate cache database migration code +* Change to only rebuild namecache on show update instead of on every search +* Add removal of old entries from namecache on show deletion ### 0.9.1 (2015-05-25 03:03:00 UTC) diff --git a/sickbeard/name_cache.py b/sickbeard/name_cache.py index 7c80a222..845505b8 100644 --- a/sickbeard/name_cache.py +++ b/sickbeard/name_cache.py @@ -18,36 +18,30 @@ import threading import sickbeard from sickbeard import db -from sickbeard import logger nameCache = {} nameCacheLock = threading.Lock() + def addNameToCache(name, indexer_id=0): - """ - Adds the show & tvdb id to the scene_names table in cache.db. - - name: The show name to cache - indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown) + """Adds the show & tvdb id to the namecache + + :param name: the show name to cache + :param indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown) """ global nameCache - cacheDB = db.DBConnection('cache.db') - # standardize the name we're using to account for small differences in providers name = sickbeard.helpers.full_sanitizeSceneName(name) if name not in nameCache: nameCache[name] = int(indexer_id) - cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name]) def retrieveNameFromCache(name): - """ - Looks up the given name in the scene_names table in cache.db. - - name: The show name to look up. - - Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache + """Looks up the given name in the name cache + + :param name: The show name to look up. + :return: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache """ global nameCache @@ -55,78 +49,49 @@ def retrieveNameFromCache(name): if name in nameCache: return int(nameCache[name]) -def clearCache(): - """ - Deletes all "unknown" entries from the cache (names with indexer_id of 0). - """ - global nameCache - - # init name cache - if not nameCache: - nameCache = {} - - cacheDB = db.DBConnection('cache.db') - cacheDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [0]) - - toRemove = [key for key, value in nameCache.iteritems() if value == 0] - for key in toRemove: - del nameCache[key] - - -def saveNameCacheToDb(): - cacheDB = db.DBConnection('cache.db') - - for name, indexer_id in nameCache.items(): - cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name]) - def buildNameCache(show=None): - global nameCache + """Adds all new name exceptions to the namecache memory and flushes any removed name exceptions + :param show (optional): Only update namecache for this show object + """ + global nameCache with nameCacheLock: - # update scene exception names - sickbeard.scene_exceptions.retrieve_exceptions() - - if not show: - # clear internal name cache - clearCache() - - logger.log(u"Building internal name cache for all shows", logger.MESSAGE) - - cacheDB = db.DBConnection('cache.db') - cache_results = cacheDB.select("SELECT * FROM scene_names") - for cache_result in cache_results: - name = sickbeard.helpers.full_sanitizeSceneName(cache_result["name"]) - if name in nameCache: - continue - - indexer_id = int(cache_result["indexer_id"]) - nameCache[name] = indexer_id - - for show in sickbeard.showList: - for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid): - for name in list(set( - sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason) + [ - show.name])): - name = sickbeard.helpers.full_sanitizeSceneName(name) - if name in nameCache: - continue - - nameCache[name] = int(show.indexerid) - else: - # remove old show cache entries + if show: + # search for only the requested show id and flush old show entries from namecache + indexer_ids = [show.indexerid] nameCache = dict((k, v) for k, v in nameCache.items() if v != show.indexerid) - logger.log(u"Building internal name cache for " + show.name, logger.MESSAGE) + # add standard indexer name to namecache + nameCache[show.name] = show.indexerid + else: + # generate list of indexer ids to look up in cache.db + indexer_ids = [x.indexerid for x in sickbeard.showList if x] - for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid): - for name in list(set(sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason) + [ - show.name])): - name = sickbeard.helpers.full_sanitizeSceneName(name) - if name in nameCache: - continue + # add all standard show indexer names to namecache + nameCache = dict( + (sickbeard.helpers.full_sanitizeSceneName(x.name), x.indexerid) for x in sickbeard.showList if x) - nameCache[name] = int(show.indexerid) + cacheDB = db.DBConnection('cache.db') - logger.log(u"Internal name cache set to: " + str(nameCache), logger.DEBUG) \ No newline at end of file + cache_results = cacheDB.select( + 'SELECT show_name, indexer_id FROM scene_exceptions WHERE indexer_id IN (%s)' % ','.join( + ['?'] * len(indexer_ids)), indexer_ids) + + if cache_results: + for cache_result in cache_results: + indexer_id = int(cache_result['indexer_id']) + name = sickbeard.helpers.full_sanitizeSceneName(cache_result['show_name']) + nameCache[name] = indexer_id + + +def remove_from_namecache(indexer_id): + """Deletes all entries from the namecache for a particular show + + :param indexer_id: tvdbid or rageid to be removed from the namecache + """ + global nameCache + + if nameCache: + nameCache = dict((k, v) for k, v in nameCache.items() if v != indexer_id) \ No newline at end of file diff --git a/sickbeard/search.py b/sickbeard/search.py index 401242d6..a3ce2299 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -457,9 +457,6 @@ def searchProviders(show, episodes, manualSearch=False): didSearch = False - # build name cache for show - sickbeard.name_cache.buildNameCache(show) - origThreadName = threading.currentThread().name providers = [x for x in sickbeard.providers.sortedProviderList() if x.isActive() and x.enable_backlog] diff --git a/sickbeard/showUpdater.py b/sickbeard/showUpdater.py index 886c73b4..88867524 100644 --- a/sickbeard/showUpdater.py +++ b/sickbeard/showUpdater.py @@ -45,6 +45,9 @@ class ShowUpdater(): # refresh network timezones network_timezones.update_network_dict() + # update scene exceptions + sickbeard.scene_exceptions.retrieve_exceptions() + # sure, why not? if sickbeard.USE_FAILED_DOWNLOADS: failed_history.trimHistory() diff --git a/sickbeard/show_queue.py b/sickbeard/show_queue.py index eec79724..c1226b64 100644 --- a/sickbeard/show_queue.py +++ b/sickbeard/show_queue.py @@ -396,9 +396,6 @@ class QueueItemAdd(ShowQueueItem): logger.ERROR) logger.log(traceback.format_exc(), logger.DEBUG) - # update internal name cache - name_cache.buildNameCache() - try: self.show.loadEpisodesFromDir() except Exception, e: @@ -474,6 +471,9 @@ class QueueItemAdd(ShowQueueItem): # Load XEM data to DB for show sickbeard.scene_numbering.xem_refresh(self.show.indexerid, self.show.indexer, force=True) + # update internal name cache + name_cache.buildNameCache(self.show) + # check if show has XEM mapping so we can determine if searches should go by scene numbering or indexer numbering. if not self.scene and sickbeard.scene_numbering.get_xem_numbering_for_show(self.show.indexerid, self.show.indexer): diff --git a/sickbeard/tv.py b/sickbeard/tv.py index ec540f2c..1e8628f6 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -43,7 +43,7 @@ except ImportError: from lib.imdb import imdb from sickbeard import db -from sickbeard import helpers, exceptions, logger +from sickbeard import helpers, exceptions, logger, name_cache from sickbeard.exceptions import ex from sickbeard import image_cache from sickbeard import notifiers @@ -1034,6 +1034,8 @@ class TVShow(object): myDB = db.DBConnection() myDB.mass_action(sql_l) + name_cache.remove_from_namecache(self.indexerid) + action = ('delete', 'trash')[sickbeard.TRASH_REMOVE_SHOW] # remove self from show list