mirror of
https://github.com/SickGear/SickGear.git
synced 2025-01-20 16:43:43 +00:00
Change anime release groups to in memory storage for lowered latency
This commit is contained in:
adam
parent
ce93c4216a
commit
ea99736939
7 changed files with 105 additions and 254 deletions
|
|
@ -8,6 +8,7 @@
|
|||
* Change backlog search to occur 10 minutes after start up
|
||||
* Change UI footer to display time left until a backlog search
|
||||
* Remove obsolete tvtorrents search provider
|
||||
* Change anime release groups to in memory storage for lowered latency
|
||||
|
||||
[develop changelog]
|
||||
|
||||
|
|
|
|||
|
|
@ -229,17 +229,19 @@
|
|||
#if $show.rls_ignore_words:
|
||||
<tr><td class="showLegend">Ignored Words: </td><td>#echo $show.rls_ignore_words#</td></tr>
|
||||
#end if
|
||||
#if $bwl and $bwl.get_white_keywords_for("release_group"):
|
||||
<tr><td class="showLegend">Wanted Group#if len($bwl.get_white_keywords_for("release_group"))>1 then "s" else ""#:</td>
|
||||
<td>#echo ', '.join($bwl.get_white_keywords_for("release_group"))#</td>
|
||||
|
||||
#if $bwl and $bwl.whitelist:
|
||||
<tr><td class="showLegend">Wanted Group#if len($bwl.whitelist)>1 then "s" else ""#:</td>
|
||||
<td>#echo ', '.join($bwl.whitelist)#</td>
|
||||
</tr>
|
||||
#end if
|
||||
#if $bwl and $bwl.get_black_keywords_for("release_group"):
|
||||
<tr><td class="showLegend">Unwanted Group#if len($bwl.get_black_keywords_for("release_group"))>1 then "s" else ""#:</td>
|
||||
<td>#echo ', '.join($bwl.get_black_keywords_for("release_group"))#</td>
|
||||
#if $bwl and $bwl.blacklist:
|
||||
<tr><td class="showLegend">Unwanted Group#if len($bwl.blacklist)>1 then "s" else ""#:</td>
|
||||
<td>#echo ', '.join($bwl.blacklist)#</td>
|
||||
</tr>
|
||||
#end if
|
||||
|
||||
|
||||
<tr><td class="showLegend">Size:</td><td>$sickbeard.helpers.human(sickbeard.helpers.get_size($showLoc[0]))</td></tr>
|
||||
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -18,195 +18,75 @@
|
|||
|
||||
from sickbeard import db, logger
|
||||
|
||||
class BlackAndWhiteList(object):
|
||||
_tableBlack = "blacklist"
|
||||
_tableWhite = "whitelist"
|
||||
blackList = []
|
||||
whiteList = []
|
||||
blackDict = {}
|
||||
whiteDict = {}
|
||||
|
||||
last_black_valid_result = None
|
||||
last_white_valid_result = None
|
||||
class BlackAndWhiteList(object):
|
||||
blacklist = []
|
||||
whitelist = []
|
||||
|
||||
def __init__(self, show_id):
|
||||
if not show_id:
|
||||
raise BlackWhitelistNoShowIDException()
|
||||
self.show_id = show_id
|
||||
self.refresh()
|
||||
self.load()
|
||||
|
||||
def refresh(self):
|
||||
logger.log(u"Building black and white list for " + str(self.show_id), logger.DEBUG)
|
||||
def load(self):
|
||||
logger.log(u'Building black and white list for ' + str(self.show_id), logger.DEBUG)
|
||||
self.blacklist = self._load_list('blacklist')
|
||||
self.whitelist = self._load_list('whitelist')
|
||||
|
||||
(self.blackList, self.blackDict) = self.load_blacklist()
|
||||
(self.whiteList, self.whiteDict) = self.load_whitelist()
|
||||
|
||||
def load_blacklist(self):
|
||||
return self._load_list(self._tableBlack)
|
||||
|
||||
def load_whitelist(self):
|
||||
return self._load_list(self._tableWhite)
|
||||
|
||||
def get_black_keywords_for(self, range):
|
||||
if range in self.blackDict:
|
||||
return self.blackDict[range]
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_white_keywords_for(self, range):
|
||||
if range in self.whiteDict:
|
||||
return self.whiteDict[range]
|
||||
else:
|
||||
return []
|
||||
|
||||
def set_black_keywords(self, range, values):
|
||||
self._del_all_black_keywords()
|
||||
self._add_keywords(self._tableBlack, range, values)
|
||||
|
||||
def set_white_keywords(self, range, values):
|
||||
self._del_all_white_keywords()
|
||||
self._add_keywords(self._tableWhite, range, values)
|
||||
|
||||
def set_black_keywords_for(self, range, values):
|
||||
self._del_all_black_keywords_for(range)
|
||||
self._add_keywords(self._tableBlack, range, values)
|
||||
|
||||
def set_white_keywords_for(self, range, values):
|
||||
self._del_all_white_keywords_for(range)
|
||||
self._add_keywords(self._tableWhite, range, values)
|
||||
|
||||
def add_black_keyword(self, range, value):
|
||||
self._add_keywords(self._tableBlack, range, [value])
|
||||
|
||||
def add_white_keyword(self, range, value):
|
||||
self._add_keywords(self._tableWhite, range, [value])
|
||||
|
||||
def get_last_result_msg(self):
|
||||
blackResult = whiteResult = "Untested"
|
||||
if self.last_black_valid_result == True:
|
||||
blackResult = "Valid"
|
||||
elif self.last_black_valid_result == False:
|
||||
blackResult = "Invalid"
|
||||
|
||||
if self.last_white_valid_result == True:
|
||||
whiteResult = "Valid"
|
||||
elif self.last_white_valid_result == False:
|
||||
whiteResult = "Invalid"
|
||||
|
||||
return "Blacklist: " + blackResult + ", Whitelist: " + whiteResult
|
||||
|
||||
def _add_keywords(self, table, range, values):
|
||||
def _add_keywords(self, table, values):
|
||||
myDB = db.DBConnection()
|
||||
for value in values:
|
||||
myDB.action("INSERT INTO [" + table + "] (show_id, range , keyword) VALUES (?,?,?)", [self.show_id, range, value])
|
||||
myDB.action('INSERT INTO [' + table + '] (show_id, keyword) VALUES (?,?)', [self.show_id, value])
|
||||
|
||||
self.refresh()
|
||||
def set_black_keywords(self, values):
|
||||
self._del_all_keywords('blacklist')
|
||||
self._add_keywords('blacklist', values)
|
||||
self.blacklist = values
|
||||
logger.log('Blacklist set to: %s' % self.blacklist, logger.DEBUG)
|
||||
|
||||
def _del_all_black_keywords(self):
|
||||
self._del_all_keywords(self._tableBlack)
|
||||
|
||||
def _del_all_white_keywords(self):
|
||||
self._del_all_keywords(self._tableWhite)
|
||||
|
||||
def _del_all_black_keywords_for(self, range):
|
||||
self._del_all_keywords_for(self._tableBlack, range)
|
||||
|
||||
def _del_all_white_keywords_for(self, range):
|
||||
self._del_all_keywords_for(self._tableWhite, range)
|
||||
def set_white_keywords(self, values):
|
||||
self._del_all_keywords('whitelist')
|
||||
self._add_keywords('whitelist', values)
|
||||
self.whitelist = values
|
||||
logger.log('Whitelist set to: %s' % self.whitelist, logger.DEBUG)
|
||||
|
||||
def _del_all_keywords(self, table):
|
||||
logger.log(u"Deleting all " + table + " keywords for " + str(self.show_id), logger.DEBUG)
|
||||
myDB = db.DBConnection()
|
||||
myDB.action("DELETE FROM [" + table + "] WHERE show_id = ?", [self.show_id])
|
||||
self.refresh()
|
||||
|
||||
def _del_all_keywords_for(self, table, range):
|
||||
logger.log(u"Deleting all " + range + " " + table + " keywords for " + str(self.show_id), logger.DEBUG)
|
||||
myDB = db.DBConnection()
|
||||
myDB.action("DELETE FROM [" + table + "] WHERE show_id = ? and range = ?", [self.show_id, range])
|
||||
self.refresh()
|
||||
myDB.action('DELETE FROM [' + table + '] WHERE show_id = ?', [self.show_id])
|
||||
|
||||
def _load_list(self, table):
|
||||
myDB = db.DBConnection()
|
||||
sqlResults = myDB.select("SELECT range,keyword FROM [" + table + "] WHERE show_id = ? ", [self.show_id])
|
||||
sqlResults = myDB.select('SELECT keyword FROM [' + table + '] WHERE show_id = ?', [self.show_id])
|
||||
if not sqlResults or not len(sqlResults):
|
||||
return ([], {})
|
||||
return []
|
||||
|
||||
list, dict = self._build_keyword_dict(sqlResults)
|
||||
logger.log("BWL: " + str(self.show_id) + " loaded keywords from " + table + ": " + str(dict), logger.DEBUG)
|
||||
return list, dict
|
||||
groups = []
|
||||
for result in sqlResults:
|
||||
groups.append(result["keyword"])
|
||||
|
||||
def _build_keyword_dict(self, sql_result):
|
||||
list = []
|
||||
dict = {}
|
||||
for row in sql_result:
|
||||
list.append(row["keyword"])
|
||||
if row["range"] in dict:
|
||||
dict[row["range"]].append(row["keyword"])
|
||||
else:
|
||||
dict[row["range"]] = [row["keyword"]]
|
||||
logger.log('BWL: ' + str(self.show_id) + ' loaded keywords from ' + table + ': ' + str(groups), logger.DEBUG)
|
||||
|
||||
return (list, dict)
|
||||
return groups
|
||||
|
||||
def is_valid_for_black(self, haystack):
|
||||
logger.log(u"BWL: " + str(self.show_id) + " is valid black", logger.DEBUG)
|
||||
result = self._is_valid_for(self.blackDict, False, haystack)
|
||||
self.last_black_valid_result = result
|
||||
return result
|
||||
def is_valid(self, result):
|
||||
|
||||
def is_valid_for_white(self, haystack):
|
||||
logger.log(u"BWL: " + str(self.show_id) + " is valid white", logger.DEBUG)
|
||||
result = self._is_valid_for(self.whiteDict, True, haystack)
|
||||
self.last_white_valid_result = result
|
||||
return result
|
||||
|
||||
def is_valid(self, haystack):
|
||||
return self.is_valid_for_black(haystack) and self.is_valid_for_white(haystack)
|
||||
|
||||
def _is_valid_for(self, list, mood, haystack):
|
||||
if not len(list):
|
||||
return True
|
||||
|
||||
results = []
|
||||
for range in list:
|
||||
for keyword in list[range]:
|
||||
string = None
|
||||
if range == "global":
|
||||
string = haystack.name
|
||||
elif range in haystack.__dict__:
|
||||
string = haystack.__dict__[range]
|
||||
elif not range in haystack.__dict__:
|
||||
results.append((not mood))
|
||||
else:
|
||||
results.append(False)
|
||||
|
||||
if string:
|
||||
results.append(self._is_keyword_in_string(string, keyword) == mood)
|
||||
|
||||
# black: mood = False
|
||||
# white: mood = True
|
||||
if mood in results:
|
||||
return mood
|
||||
if result.release_group.lower() in [x.lower() for x in self.whitelist]:
|
||||
white_result = True
|
||||
else:
|
||||
return (not mood)
|
||||
white_result = False
|
||||
|
||||
def _is_keyword_in_string(self, fromPost, fromBWList):
|
||||
"""
|
||||
will return true if fromBWList is found in fromPost
|
||||
for now a basic find is used
|
||||
"""
|
||||
fromPost = fromPost.lower()
|
||||
fromBWList = fromBWList.lower()
|
||||
logger.log(u"BWL: " + str(self.show_id) + " comparing fromPost: " + fromPost + " vs fromBWlist: " + fromBWList, logger.DEBUG)
|
||||
return (fromPost.find(fromBWList) >= 0)
|
||||
if result.release_group.lower() in [x.lower() for x in self.blacklist]:
|
||||
black_result = False
|
||||
else:
|
||||
black_result = True
|
||||
|
||||
class BlackWhiteKeyword(object):
|
||||
range = ""
|
||||
value = []
|
||||
logger.log('Whitelist check passed: %s. Blacklist check passed: %s' % (white_result, black_result), logger.DEBUG)
|
||||
|
||||
def __init__(self, range, values):
|
||||
self.range = range # "global" or a parser group
|
||||
self.value = values # a list of values may contain only one item (still a list)
|
||||
if white_result and black_result:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
class BlackWhitelistNoShowIDException(Exception):
|
||||
"No show_id was given"
|
||||
'No show_id was given'
|
||||
|
|
@ -198,23 +198,14 @@ def filter_release_name(name, filter_words):
|
|||
def pickBestResult(results, show, quality_list=None):
|
||||
logger.log(u"Picking the best result out of " + str([x.name for x in results]), logger.DEBUG)
|
||||
|
||||
# build the black And white list
|
||||
bwl = None
|
||||
if show:
|
||||
if show.is_anime:
|
||||
bwl = BlackAndWhiteList(show.indexerid)
|
||||
else:
|
||||
logger.log("Could not create black and white list no show was given", logger.DEBUG)
|
||||
|
||||
# find the best result for the current episode
|
||||
bestResult = None
|
||||
for cur_result in results:
|
||||
|
||||
logger.log("Quality of " + cur_result.name + " is " + Quality.qualityStrings[cur_result.quality])
|
||||
|
||||
if bwl:
|
||||
if not bwl.is_valid(cur_result):
|
||||
logger.log(cur_result.name+" does not match the blacklist or the whitelist, rejecting it. Result: " + bwl.get_last_result_msg(), logger.MESSAGE)
|
||||
if show.is_anime:
|
||||
if not show.release_groups.is_valid(cur_result):
|
||||
continue
|
||||
|
||||
if quality_list and cur_result.quality not in quality_list:
|
||||
|
|
@ -269,9 +260,7 @@ def isFinalResult(result):
|
|||
|
||||
show_obj = result.episodes[0].show
|
||||
|
||||
bwl = None
|
||||
if show_obj.is_anime:
|
||||
bwl = BlackAndWhiteList(show_obj.indexerid)
|
||||
|
||||
|
||||
any_qualities, best_qualities = Quality.splitQuality(show_obj.quality)
|
||||
|
||||
|
|
@ -280,7 +269,7 @@ def isFinalResult(result):
|
|||
return False
|
||||
|
||||
# if it does not match the shows black and white list its no good
|
||||
elif bwl and not bwl.is_valid(result):
|
||||
elif show_obj.is_anime and show_obj.release_groups.is_valid(result):
|
||||
return False
|
||||
|
||||
# if there's no redownload that's higher (above) and this is the highest initial download then we're good
|
||||
|
|
|
|||
|
|
@ -164,7 +164,6 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
|
|||
numseasons = int(numseasonsSQlResult[0][0])
|
||||
seasonStrings = ["S%02d" % int(ep_obj.scene_season)]
|
||||
|
||||
bwl = BlackAndWhiteList(show.indexerid)
|
||||
showNames = set(makeSceneShowSearchStrings(show, ep_obj.scene_season))
|
||||
|
||||
toReturn = []
|
||||
|
|
@ -179,8 +178,8 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
|
|||
# for providers that don't allow multiple searches in one request we only search for Sxx style stuff
|
||||
else:
|
||||
for cur_season in seasonStrings:
|
||||
if len(bwl.whiteList) > 0:
|
||||
for keyword in bwl.whiteList:
|
||||
if len(show.release_groups.whitelist) > 0:
|
||||
for keyword in show.release_groups.whitelist:
|
||||
toReturn.append(keyword + '.' + curShow+ "." + cur_season)
|
||||
else:
|
||||
toReturn.append(curShow + "." + cur_season)
|
||||
|
|
@ -210,15 +209,14 @@ def makeSceneSearchString(show, ep_obj):
|
|||
if numseasons == 1 and not ep_obj.show.is_anime:
|
||||
epStrings = ['']
|
||||
|
||||
bwl = BlackAndWhiteList(ep_obj.show.indexerid)
|
||||
showNames = set(makeSceneShowSearchStrings(show, ep_obj.scene_season))
|
||||
|
||||
toReturn = []
|
||||
|
||||
for curShow in showNames:
|
||||
for curEpString in epStrings:
|
||||
if len(bwl.whiteList) > 0:
|
||||
for keyword in bwl.whiteList:
|
||||
if len(ep_obj.show.release_groups.whitelist) > 0:
|
||||
for keyword in ep_obj.show.release_groups.whitelist:
|
||||
toReturn.append(keyword + '.' + curShow + '.' + curEpString)
|
||||
else:
|
||||
toReturn.append(curShow + '.' + curEpString)
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from sickbeard import notifiers
|
|||
from sickbeard import postProcessor
|
||||
from sickbeard import subtitles
|
||||
from sickbeard import history
|
||||
from sickbeard.blackandwhitelist import BlackAndWhiteList
|
||||
|
||||
from sickbeard import encodingKludge as ek
|
||||
|
||||
|
|
@ -105,6 +106,7 @@ class TVShow(object):
|
|||
self.isDirGood = False
|
||||
self.episodes = {}
|
||||
self.nextaired = ""
|
||||
self.release_groups = None
|
||||
|
||||
otherShow = helpers.findCertainShow(sickbeard.showList, self.indexerid)
|
||||
if otherShow != None:
|
||||
|
|
@ -828,6 +830,9 @@ class TVShow(object):
|
|||
if not self.imdbid:
|
||||
self.imdbid = sqlResults[0]["imdb_id"]
|
||||
|
||||
if self.is_anime:
|
||||
self.release_groups = BlackAndWhiteList(self.indexerid)
|
||||
|
||||
# Get IMDb_info from database
|
||||
myDB = db.DBConnection()
|
||||
sqlResults = myDB.select("SELECT * FROM imdb_info WHERE indexer_id = ?", [self.indexerid])
|
||||
|
|
|
|||
|
|
@ -3748,7 +3748,7 @@ class Home(MainHandler):
|
|||
|
||||
t.bwl = None
|
||||
if showObj.is_anime:
|
||||
t.bwl = BlackAndWhiteList(showObj.indexerid)
|
||||
t.bwl = showObj.release_groups
|
||||
|
||||
t.epCounts = epCounts
|
||||
t.epCats = epCats
|
||||
|
|
@ -3790,7 +3790,7 @@ class Home(MainHandler):
|
|||
def editShow(self, show=None, location=None, anyQualities=[], bestQualities=[], exceptions_list=[],
|
||||
flatten_folders=None, paused=None, directCall=False, air_by_date=None, sports=None, dvdorder=None,
|
||||
indexerLang=None, subtitles=None, archive_firstmatch=None, rls_ignore_words=None,
|
||||
rls_require_words=None, anime=None, blackWords=None, whiteWords=None, blacklist=None, whitelist=None,
|
||||
rls_require_words=None, anime=None, blacklist=None, whitelist=None,
|
||||
scene=None):
|
||||
|
||||
if show is None:
|
||||
|
|
@ -3816,23 +3816,8 @@ class Home(MainHandler):
|
|||
t.submenu = HomeMenu()
|
||||
|
||||
if showObj.is_anime:
|
||||
bwl = BlackAndWhiteList(showObj.indexerid)
|
||||
|
||||
t.whiteWords = ""
|
||||
if "global" in bwl.whiteDict:
|
||||
t.whiteWords = ", ".join(bwl.whiteDict["global"])
|
||||
|
||||
t.blackWords = ""
|
||||
if "global" in bwl.blackDict:
|
||||
t.blackWords = ", ".join(bwl.blackDict["global"])
|
||||
|
||||
t.whitelist = []
|
||||
if bwl.whiteDict.has_key("release_group"):
|
||||
t.whitelist = bwl.whiteDict["release_group"]
|
||||
|
||||
t.blacklist = []
|
||||
if bwl.blackDict.has_key("release_group"):
|
||||
t.blacklist = bwl.blackDict["release_group"]
|
||||
t.whitelist = showObj.release_groups.whitelist
|
||||
t.blacklist = showObj.release_groups.blacklist
|
||||
|
||||
t.groups = []
|
||||
if helpers.set_up_anidb_connection():
|
||||
|
|
@ -3889,55 +3874,46 @@ class Home(MainHandler):
|
|||
else:
|
||||
do_update_exceptions = True
|
||||
|
||||
if showObj.is_anime:
|
||||
bwl = BlackAndWhiteList(showObj.indexerid)
|
||||
if whitelist:
|
||||
whitelist = whitelist.split(",")
|
||||
shortWhiteList = []
|
||||
if helpers.set_up_anidb_connection():
|
||||
for groupName in whitelist:
|
||||
group = sickbeard.ADBA_CONNECTION.group(gname=groupName)
|
||||
for line in group.datalines:
|
||||
if line["shortname"]:
|
||||
shortWhiteList.append(line["shortname"])
|
||||
else:
|
||||
if not groupName in shortWhiteList:
|
||||
shortWhiteList.append(groupName)
|
||||
with showObj.lock:
|
||||
if anime:
|
||||
if not showObj.release_groups:
|
||||
showObj.release_groups = BlackAndWhiteList(showObj.indexerid)
|
||||
|
||||
if whitelist:
|
||||
whitelist = whitelist.split(",")
|
||||
shortWhiteList = []
|
||||
if helpers.set_up_anidb_connection():
|
||||
for groupName in whitelist:
|
||||
group = sickbeard.ADBA_CONNECTION.group(gname=groupName)
|
||||
for line in group.datalines:
|
||||
if line["shortname"]:
|
||||
shortWhiteList.append(line["shortname"])
|
||||
else:
|
||||
if not groupName in shortWhiteList:
|
||||
shortWhiteList.append(groupName)
|
||||
else:
|
||||
shortWhiteList = whitelist
|
||||
showObj.release_groups.set_white_keywords(shortWhiteList)
|
||||
else:
|
||||
shortWhiteList = whitelist
|
||||
bwl.set_white_keywords_for("release_group", shortWhiteList)
|
||||
else:
|
||||
bwl.set_white_keywords_for("release_group", [])
|
||||
showObj.release_groups.set_white_keywords([])
|
||||
|
||||
if blacklist:
|
||||
blacklist = blacklist.split(",")
|
||||
shortBlacklist = []
|
||||
if helpers.set_up_anidb_connection():
|
||||
for groupName in blacklist:
|
||||
group = sickbeard.ADBA_CONNECTION.group(gname=groupName)
|
||||
for line in group.datalines:
|
||||
if line["shortname"]:
|
||||
shortBlacklist.append(line["shortname"])
|
||||
else:
|
||||
if not groupName in shortBlacklist:
|
||||
shortBlacklist.append(groupName)
|
||||
if blacklist:
|
||||
blacklist = blacklist.split(",")
|
||||
shortBlacklist = []
|
||||
if helpers.set_up_anidb_connection():
|
||||
for groupName in blacklist:
|
||||
group = sickbeard.ADBA_CONNECTION.group(gname=groupName)
|
||||
for line in group.datalines:
|
||||
if line["shortname"]:
|
||||
shortBlacklist.append(line["shortname"])
|
||||
else:
|
||||
if not groupName in shortBlacklist:
|
||||
shortBlacklist.append(groupName)
|
||||
else:
|
||||
shortBlacklist = blacklist
|
||||
showObj.release_groups.set_black_keywords(shortBlacklist)
|
||||
else:
|
||||
shortBlacklist = blacklist
|
||||
bwl.set_black_keywords_for("release_group", shortBlacklist)
|
||||
else:
|
||||
bwl.set_black_keywords_for("release_group", [])
|
||||
|
||||
if whiteWords:
|
||||
whiteWords = [x.strip() for x in whiteWords.split(",")]
|
||||
bwl.set_white_keywords_for("global", whiteWords)
|
||||
else:
|
||||
bwl.set_white_keywords_for("global", [])
|
||||
|
||||
if blackWords:
|
||||
blackWords = [x.strip() for x in blackWords.split(",")]
|
||||
bwl.set_black_keywords_for("global", blackWords)
|
||||
else:
|
||||
bwl.set_black_keywords_for("global", [])
|
||||
showObj.release_groups.set_black_keywords([])
|
||||
|
||||
errors = []
|
||||
with showObj.lock:
|
||||
|
|
|
|||
Loading…
Reference in a new issue