mirror of
https://github.com/SickGear/SickGear.git
synced 2025-01-20 16:43:43 +00:00
SickRage now uses a internal name cache comprised of (showlist, scene exceptions, scene names) used to search and create show objects from which is needed for parsing search results.
Scene exceptions now uses a internal cache for scene exceptions and scene season exceptions, helps reduce overhead to DB and performs faster lookups when making scene exception requests.
This commit is contained in:
echel0n
parent
f0146f728e
commit
56e2c28bad
9 changed files with 118 additions and 75 deletions
|
|
@ -107,9 +107,6 @@ class DailySearcher():
|
|||
for show in todaysEps:
|
||||
segment = todaysEps[show]
|
||||
|
||||
# remove show from name cache if marked invalid
|
||||
sickbeard.name_cache.clearCache(show)
|
||||
|
||||
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment)
|
||||
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -164,6 +164,8 @@ class DBConnection(object):
|
|||
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
||||
raise
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
return sqlResult
|
||||
|
||||
def action(self, query, args=None, fetchall=False, fetchone=False):
|
||||
|
|
@ -199,6 +201,8 @@ class DBConnection(object):
|
|||
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
||||
raise
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
return sqlResult
|
||||
|
||||
def select(self, query, args=None):
|
||||
|
|
|
|||
|
|
@ -730,7 +730,6 @@ def sanitizeSceneName(name, ezrss=False):
|
|||
"""
|
||||
|
||||
if name:
|
||||
|
||||
if not ezrss:
|
||||
bad_chars = u",:()'!?\u2019"
|
||||
# ezrss leaves : and ! in their show names as far as I can tell
|
||||
|
|
@ -1066,37 +1065,24 @@ def _check_against_names(nameInQuestion, show, season=-1):
|
|||
|
||||
|
||||
def get_show_by_name(name, useIndexer=False):
|
||||
name = full_sanitizeSceneName(name)
|
||||
|
||||
try:
|
||||
# check cache for show
|
||||
showObj = sickbeard.name_cache.retrieveShowFromCache(name)
|
||||
if showObj:
|
||||
return showObj
|
||||
|
||||
if not showObj and sickbeard.showList:
|
||||
db_indexerid = searchDBForShow(name)
|
||||
if db_indexerid:
|
||||
showObj = findCertainShow(sickbeard.showList, db_indexerid)
|
||||
|
||||
if not showObj:
|
||||
scene_indexerid, scene_season = sickbeard.scene_exceptions.get_scene_exception_by_name(name)
|
||||
if scene_indexerid:
|
||||
showObj = findCertainShow(sickbeard.showList, scene_indexerid)
|
||||
|
||||
if useIndexer and not showObj:
|
||||
(sn, idx, id) = searchIndexerForShowID(name, ui=classes.ShowListUI)
|
||||
if id:
|
||||
showObj = findCertainShow(sickbeard.showList, int(id))
|
||||
if useIndexer and sickbeard.showList and not showObj:
|
||||
(sn, idx, id) = searchIndexerForShowID(full_sanitizeSceneName(name), ui=classes.ShowListUI)
|
||||
if id:
|
||||
showObj = findCertainShow(sickbeard.showList, int(id))
|
||||
|
||||
# add show to cache
|
||||
if showObj:
|
||||
sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
|
||||
|
||||
return showObj
|
||||
except:
|
||||
showObj = None
|
||||
|
||||
return showObj
|
||||
|
||||
pass
|
||||
|
||||
def is_hidden_folder(folder):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import sickbeard
|
|||
from sickbeard import scene_exceptions
|
||||
from sickbeard import failed_history
|
||||
from sickbeard import network_timezones
|
||||
|
||||
from sickbeard import name_cache
|
||||
|
||||
class Maintenance():
|
||||
def __init__(self):
|
||||
|
|
@ -31,15 +31,18 @@ class Maintenance():
|
|||
|
||||
self.amActive = False
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
def run(self, force=False):
|
||||
self.amActive = True
|
||||
|
||||
# clear internal name cache
|
||||
name_cache.clearCache()
|
||||
|
||||
# get and update scene exceptions lists
|
||||
scene_exceptions.retrieve_exceptions()
|
||||
|
||||
# build internal name cache for searches and parsing
|
||||
name_cache.buildNameCache()
|
||||
|
||||
# refresh network timezones
|
||||
network_timezones.update_network_dict()
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ from sickbeard import db
|
|||
from sickbeard.helpers import sanitizeSceneName
|
||||
from sickbeard import logger
|
||||
|
||||
nameCache = None
|
||||
|
||||
def addNameToCache(name, indexer_id=0):
|
||||
"""
|
||||
Adds the show & tvdb id to the scene_names table in cache.db.
|
||||
|
|
@ -29,11 +31,15 @@ def addNameToCache(name, indexer_id=0):
|
|||
name: The show name to cache
|
||||
indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown)
|
||||
"""
|
||||
global nameCache
|
||||
|
||||
cacheDB = db.DBConnection('cache.db')
|
||||
|
||||
# standardize the name we're using to account for small differences in providers
|
||||
name = sanitizeSceneName(name)
|
||||
myDB = db.DBConnection('cache.db')
|
||||
myDB.action("INSERT INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
||||
name = sickbeard.helpers.full_sanitizeSceneName(name)
|
||||
if name not in nameCache:
|
||||
nameCache[name] = int(indexer_id)
|
||||
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
||||
|
||||
|
||||
def retrieveNameFromCache(name):
|
||||
|
|
@ -44,33 +50,66 @@ def retrieveNameFromCache(name):
|
|||
|
||||
Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache
|
||||
"""
|
||||
global nameCache
|
||||
|
||||
cache_results = None
|
||||
|
||||
# standardize the name we're using to account for small differences in providers
|
||||
name = sanitizeSceneName(name)
|
||||
|
||||
myDB = db.DBConnection('cache.db')
|
||||
if myDB.hasTable('scene_names'):
|
||||
cache_results = myDB.select("SELECT * FROM scene_names WHERE name = ?", [name])
|
||||
|
||||
if cache_results:
|
||||
return int(cache_results[0]["indexer_id"])
|
||||
name = sickbeard.helpers.full_sanitizeSceneName(name)
|
||||
if name in nameCache:
|
||||
return int(nameCache[name])
|
||||
|
||||
def retrieveShowFromCache(name):
|
||||
indexerid = retrieveNameFromCache(name)
|
||||
if indexerid:
|
||||
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexerid))
|
||||
global nameCache
|
||||
|
||||
def clearCache(show=None, season=-1, indexer_id=0):
|
||||
indexer_id = retrieveNameFromCache(name)
|
||||
if indexer_id:
|
||||
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexer_id))
|
||||
|
||||
def clearCache():
|
||||
"""
|
||||
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
|
||||
"""
|
||||
global nameCache
|
||||
|
||||
myDB = db.DBConnection('cache.db')
|
||||
if show:
|
||||
showNames = sickbeard.show_name_helpers.allPossibleShowNames(show, season=season)
|
||||
for showName in showNames:
|
||||
myDB.action("DELETE FROM scene_names WHERE name = ? and indexer_id = ?", [showName, indexer_id])
|
||||
else:
|
||||
myDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [indexer_id])
|
||||
# init name cache
|
||||
if not nameCache:
|
||||
nameCache = {}
|
||||
|
||||
cacheDB = db.DBConnection('cache.db')
|
||||
cacheDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [0])
|
||||
|
||||
toRemove = [key for key, value in nameCache.iteritems() if value == 0]
|
||||
for key in toRemove:
|
||||
del nameCache[key]
|
||||
|
||||
def saveNameCacheToDb():
|
||||
cacheDB = db.DBConnection('cache.db')
|
||||
|
||||
for name, indexer_id in nameCache.items():
|
||||
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
||||
|
||||
def buildNameCache():
|
||||
global nameCache
|
||||
|
||||
# init name cache
|
||||
if not nameCache:
|
||||
nameCache = {}
|
||||
|
||||
# clear internal name cache
|
||||
clearCache()
|
||||
|
||||
logger.log(u"Updating internal name cache", logger.MESSAGE)
|
||||
|
||||
cacheDB = db.DBConnection('cache.db')
|
||||
cache_results = cacheDB.select("SELECT * FROM scene_names")
|
||||
for cache_result in cache_results:
|
||||
name = sickbeard.helpers.full_sanitizeSceneName(cache_result["name"])
|
||||
indexer_id = int(cache_result["indexer_id"])
|
||||
nameCache[name] = indexer_id
|
||||
|
||||
for show in sickbeard.showList:
|
||||
for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid):
|
||||
nameCache[sickbeard.helpers.full_sanitizeSceneName(show.name)] = show.indexerid
|
||||
for name in sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason):
|
||||
nameCache[sickbeard.helpers.full_sanitizeSceneName(name)] = show.indexerid
|
||||
|
||||
logger.log(u"Updated internal name cache", logger.MESSAGE)
|
||||
logger.log(u"Internal name cache set to: " + str(nameCache), logger.DEBUG)
|
||||
|
|
@ -343,8 +343,6 @@ class BTNCache(tvcache.TVCache):
|
|||
if ci is not None:
|
||||
cl.append(ci)
|
||||
|
||||
|
||||
|
||||
if cl:
|
||||
myDB = self._getDB()
|
||||
myDB.mass_action(cl)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ from sickbeard import name_cache
|
|||
from sickbeard import logger
|
||||
from sickbeard import db
|
||||
|
||||
scene_lock = threading.Lock()
|
||||
exceptionsCache = None
|
||||
exceptionsSeasonCache = None
|
||||
|
||||
def shouldRefresh(list):
|
||||
MAX_REFRESH_AGE_SECS = 86400 # 1 day
|
||||
|
|
@ -51,14 +52,21 @@ def get_scene_exceptions(indexer_id, season=-1):
|
|||
"""
|
||||
Given a indexer_id, return a list of all the scene exceptions.
|
||||
"""
|
||||
|
||||
global exceptionsCache
|
||||
exceptionsList = []
|
||||
|
||||
myDB = db.DBConnection('cache.db')
|
||||
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
|
||||
[indexer_id, season])
|
||||
if exceptions:
|
||||
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
|
||||
if indexer_id not in exceptionsCache or season not in exceptionsCache[indexer_id]:
|
||||
myDB = db.DBConnection('cache.db')
|
||||
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
|
||||
[indexer_id, season])
|
||||
if exceptions:
|
||||
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
|
||||
|
||||
if not indexer_id in exceptionsCache:
|
||||
exceptionsCache[indexer_id] = {}
|
||||
exceptionsCache[indexer_id][season] = exceptionsList
|
||||
else:
|
||||
exceptionsList = exceptionsCache[indexer_id][season]
|
||||
|
||||
if season == 1: # if we where looking for season 1 we can add generic names
|
||||
exceptionsList += get_scene_exceptions(indexer_id, season=-1)
|
||||
|
|
@ -84,14 +92,24 @@ def get_scene_seasons(indexer_id):
|
|||
"""
|
||||
return a list of season numbers that have scene exceptions
|
||||
"""
|
||||
global exceptionsSeasonCache
|
||||
exceptionsSeasonList = []
|
||||
|
||||
myDB = db.DBConnection('cache.db')
|
||||
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
|
||||
[indexer_id])
|
||||
if indexer_id not in exceptionsSeasonCache:
|
||||
myDB = db.DBConnection('cache.db')
|
||||
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
|
||||
[indexer_id])
|
||||
if sqlResults:
|
||||
exceptionsSeasonList = list(set([int(x["season"]) for x in sqlResults]))
|
||||
|
||||
if sqlResults:
|
||||
return [int(x["season"]) for x in sqlResults]
|
||||
if not indexer_id in exceptionsSeasonCache:
|
||||
exceptionsSeasonCache[indexer_id] = {}
|
||||
|
||||
exceptionsSeasonCache[indexer_id] = exceptionsSeasonList
|
||||
else:
|
||||
exceptionsSeasonList = exceptionsSeasonCache[indexer_id]
|
||||
|
||||
return exceptionsSeasonList
|
||||
|
||||
def get_scene_exception_by_name(show_name):
|
||||
return get_scene_exception_by_name_multiple(show_name)[0]
|
||||
|
|
@ -136,7 +154,10 @@ def retrieve_exceptions():
|
|||
Looks up the exceptions on github, parses them into a dict, and inserts them into the
|
||||
scene_exceptions table in cache.db. Also clears the scene name cache.
|
||||
"""
|
||||
global exceptionsCache, exceptionsSeasonCache
|
||||
|
||||
exceptionsCache = {}
|
||||
exceptionsSeasonCache = {}
|
||||
exception_dict = {}
|
||||
|
||||
# exceptions are stored on github pages
|
||||
|
|
@ -209,7 +230,6 @@ def retrieve_exceptions():
|
|||
# since this could invalidate the results of the cache we clear it out after updating
|
||||
if changed_exceptions:
|
||||
logger.log(u"Updated scene exceptions")
|
||||
name_cache.clearCache()
|
||||
else:
|
||||
logger.log(u"No scene exceptions update needed")
|
||||
|
||||
|
|
@ -230,8 +250,6 @@ def update_scene_exceptions(indexer_id, scene_exceptions):
|
|||
myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season, custom) VALUES (?,?,?,?)",
|
||||
[indexer_id, cur_exception, cur_season, 1])
|
||||
|
||||
name_cache.clearCache()
|
||||
|
||||
def _retrieve_anidb_mainnames():
|
||||
|
||||
anidb_mainNames = {}
|
||||
|
|
|
|||
|
|
@ -279,9 +279,6 @@ class QueueItemAdd(ShowQueueItem):
|
|||
return
|
||||
|
||||
try:
|
||||
# clear the name cache
|
||||
name_cache.clearCache()
|
||||
|
||||
newShow = TVShow(self.indexer, self.indexer_id, self.lang)
|
||||
newShow.loadFromIndexer()
|
||||
|
||||
|
|
@ -361,6 +358,9 @@ class QueueItemAdd(ShowQueueItem):
|
|||
# before we parse local files lets update exceptions
|
||||
sickbeard.scene_exceptions.retrieve_exceptions()
|
||||
|
||||
# update internal name cache
|
||||
name_cache.buildNameCache()
|
||||
|
||||
try:
|
||||
self.show.loadEpisodesFromDir()
|
||||
except Exception, e:
|
||||
|
|
|
|||
|
|
@ -128,8 +128,6 @@ class TVCache():
|
|||
if ci is not None:
|
||||
cl.append(ci)
|
||||
|
||||
time.sleep(.2)
|
||||
|
||||
if cl:
|
||||
myDB = self._getDB()
|
||||
myDB.mass_action(cl)
|
||||
|
|
|
|||
Loading…
Reference in a new issue