monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2025-01-22 01:23:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #104 from adam111316/feature/FixCacheUrlIndexes.

Browse source
This commit is contained in:
JackDandy 2014-12-14 12:29:37 +00:00
commit 26bc577092
4 changed files with 396 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES.md
View file

@ -33,6 +33,7 @@
* Add validation to Go button to ensure at least one item is checked on Episode Overview page * Add validation to Go button to ensure at least one item is checked on Episode Overview page
* Add highlight to current status text in header on Episode Overview page * Add highlight to current status text in header on Episode Overview page
* Fix table alignment on homepage * Fix table alignment on homepage
* Fix duplicate entries in cache database
[develop changelog] [develop changelog]
* Add TVRage network name standardization * Add TVRage network name standardization

668
sickbeard/databases/mainDB.py
View file

File diff suppressed because it is too large Load diff

8
sickbeard/db.py
View file

@ -261,6 +261,14 @@ class DBConnection(object):
def hasColumn(self, tableName, column): def hasColumn(self, tableName, column):
return column in self.tableInfo(tableName) return column in self.tableInfo(tableName)
def hasIndex(self, tableName, index):
sqlResults = self.select('PRAGMA index_list([%s])' % tableName)
for result in sqlResults:
if result['name'] == index:
return True
return False
def addColumn(self, table, column, type="NUMERIC", default=0): def addColumn(self, table, column, type="NUMERIC", default=0):
self.action("ALTER TABLE [%s] ADD %s %s" % (table, column, type)) self.action("ALTER TABLE [%s] ADD %s %s" % (table, column, type))
self.action("UPDATE [%s] SET %s = ?" % (table, column), (default,)) self.action("UPDATE [%s] SET %s = ?" % (table, column), (default,))

95
sickbeard/tvcache.py
View file

@ -36,41 +36,44 @@ import itertools
class CacheDBConnection(db.DBConnection): class CacheDBConnection(db.DBConnection):
def __init__(self, providerName): def __init__(self, providerName):
db.DBConnection.__init__(self, "cache.db") db.DBConnection.__init__(self, 'cache.db')
# Create the table if it's not already there # Create the table if it's not already there
try: try:
if not self.hasTable(providerName): if not self.hasTable(providerName):
self.action( self.action(
"CREATE TABLE [" + providerName + "] (name TEXT, season NUMERIC, episodes TEXT, indexerid NUMERIC, url TEXT, time NUMERIC, quality TEXT, release_group TEXT)") 'CREATE TABLE [' + providerName + '] (name TEXT, season NUMERIC, episodes TEXT, indexerid NUMERIC, url TEXT, time NUMERIC, quality TEXT, release_group TEXT)')
else: self.action(
'CREATE UNIQUE INDEX IF NOT EXISTS [idx_' + providerName + '_url] ON [' + providerName + '] (url)')
elif not self.hasIndex(providerName, 'idx_%s_url' % providerName):
sqlResults = self.select( sqlResults = self.select(
"SELECT url, COUNT(url) as count FROM [" + providerName + "] GROUP BY url HAVING count > 1") 'SELECT url, COUNT(url) as count FROM [' + providerName + '] GROUP BY url HAVING count > 1')
for cur_dupe in sqlResults: for cur_dupe in sqlResults:
self.action("DELETE FROM [" + providerName + "] WHERE url = ?", [cur_dupe["url"]]) self.action('DELETE FROM [' + providerName + '] WHERE url = ?', [cur_dupe['url']])
self.action(
'CREATE UNIQUE INDEX IF NOT EXISTS [idx_' + providerName + '_url] ON [' + providerName + '] (url)')
# add unique index to prevent further dupes from happening if one does not exist
self.action("CREATE UNIQUE INDEX IF NOT EXISTS idx_url ON [" + providerName + "] (url)")
# add release_group column to table if missing # add release_group column to table if missing
if not self.hasColumn(providerName, 'release_group'): if not self.hasColumn(providerName, 'release_group'):
self.addColumn(providerName, 'release_group', "TEXT", "") self.addColumn(providerName, 'release_group', 'TEXT', '')
# add version column to table if missing # add version column to table if missing
if not self.hasColumn(providerName, 'version'): if not self.hasColumn(providerName, 'version'):
self.addColumn(providerName, 'version', "NUMERIC", "-1") self.addColumn(providerName, 'version', 'NUMERIC', '-1')
except Exception, e: except Exception, e:
if str(e) != "table [" + providerName + "] already exists": if str(e) != 'table [' + providerName + '] already exists':
raise raise
# Create the table if it's not already there # Create the table if it's not already there
try: try:
if not self.hasTable('lastUpdate'): if not self.hasTable('lastUpdate'):
self.action("CREATE TABLE lastUpdate (provider TEXT, time NUMERIC)") self.action('CREATE TABLE lastUpdate (provider TEXT, time NUMERIC)')
except Exception, e: except Exception, e:
if str(e) != "table lastUpdate already exists": if str(e) != 'table lastUpdate already exists':
raise raise
class TVCache(): class TVCache():
@ -91,7 +94,7 @@ class TVCache():
def _clearCache(self): def _clearCache(self):
if self.shouldClearCache(): if self.shouldClearCache():
myDB = self._getDB() myDB = self._getDB()
myDB.action("DELETE FROM [" + self.providerID + "] WHERE 1") myDB.action('DELETE FROM [' + self.providerID + '] WHERE 1')
def _get_title_and_url(self, item): def _get_title_and_url(self, item):
# override this in the provider if daily search has a different data layout to backlog searches # override this in the provider if daily search has a different data layout to backlog searches
@ -151,22 +154,22 @@ class TVCache():
title = self._translateTitle(title) title = self._translateTitle(title)
url = self._translateLinkURL(url) url = self._translateLinkURL(url)
logger.log(u"Attempting to add item to cache: " + title, logger.DEBUG) logger.log(u'Attempting to add item to cache: ' + title, logger.DEBUG)
return self._addCacheEntry(title, url) return self._addCacheEntry(title, url)
else: else:
logger.log( logger.log(
u"The data returned from the " + self.provider.name + " feed is incomplete, this result is unusable", u'The data returned from the ' + self.provider.name + ' feed is incomplete, this result is unusable',
logger.DEBUG) logger.DEBUG)
return None return None
def _getLastUpdate(self): def _getLastUpdate(self):
myDB = self._getDB() myDB = self._getDB()
sqlResults = myDB.select("SELECT time FROM lastUpdate WHERE provider = ?", [self.providerID]) sqlResults = myDB.select('SELECT time FROM lastUpdate WHERE provider = ?', [self.providerID])
if sqlResults: if sqlResults:
lastTime = int(sqlResults[0]["time"]) lastTime = int(sqlResults[0]['time'])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
lastTime = 0 lastTime = 0
else: else:
@ -176,10 +179,10 @@ class TVCache():
def _getLastSearch(self): def _getLastSearch(self):
myDB = self._getDB() myDB = self._getDB()
sqlResults = myDB.select("SELECT time FROM lastSearch WHERE provider = ?", [self.providerID]) sqlResults = myDB.select('SELECT time FROM lastSearch WHERE provider = ?', [self.providerID])
if sqlResults: if sqlResults:
lastTime = int(sqlResults[0]["time"]) lastTime = int(sqlResults[0]['time'])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
lastTime = 0 lastTime = 0
else: else:
@ -193,7 +196,7 @@ class TVCache():
toDate = datetime.datetime.today() toDate = datetime.datetime.today()
myDB = self._getDB() myDB = self._getDB()
myDB.upsert("lastUpdate", myDB.upsert('lastUpdate',
{'time': int(time.mktime(toDate.timetuple()))}, {'time': int(time.mktime(toDate.timetuple()))},
{'provider': self.providerID}) {'provider': self.providerID})
@ -202,7 +205,7 @@ class TVCache():
toDate = datetime.datetime.today() toDate = datetime.datetime.today()
myDB = self._getDB() myDB = self._getDB()
myDB.upsert("lastSearch", myDB.upsert('lastSearch',
{'time': int(time.mktime(toDate.timetuple()))}, {'time': int(time.mktime(toDate.timetuple()))},
{'provider': self.providerID}) {'provider': self.providerID})
@ -212,7 +215,7 @@ class TVCache():
def shouldUpdate(self): def shouldUpdate(self):
# if we've updated recently then skip the update # if we've updated recently then skip the update
if datetime.datetime.today() - self.lastUpdate < datetime.timedelta(minutes=self.minTime): if datetime.datetime.today() - self.lastUpdate < datetime.timedelta(minutes=self.minTime):
logger.log(u"Last update was too soon, using old cache: today()-" + str(self.lastUpdate) + "<" + str( logger.log(u'Last update was too soon, using old cache: today()-' + str(self.lastUpdate) + '<' + str(
datetime.timedelta(minutes=self.minTime)), logger.DEBUG) datetime.timedelta(minutes=self.minTime)), logger.DEBUG)
return False return False
@ -239,10 +242,10 @@ class TVCache():
myParser = NameParser(showObj=showObj, convert=True) myParser = NameParser(showObj=showObj, convert=True)
parse_result = myParser.parse(name) parse_result = myParser.parse(name)
except InvalidNameException: except InvalidNameException:
logger.log(u"Unable to parse the filename " + name + " into a valid episode", logger.DEBUG) logger.log(u'Unable to parse the filename ' + name + ' into a valid episode', logger.DEBUG)
return None return None
except InvalidShowException: except InvalidShowException:
logger.log(u"Unable to parse the filename " + name + " into a valid show", logger.DEBUG) logger.log(u'Unable to parse the filename ' + name + ' into a valid show', logger.DEBUG)
return None return None
if not parse_result or not parse_result.series_name: if not parse_result or not parse_result.series_name:
@ -254,7 +257,7 @@ class TVCache():
if season and episodes: if season and episodes:
# store episodes as a seperated string # store episodes as a seperated string
episodeText = "|" + "|".join(map(str, episodes)) + "|" episodeText = '|' + '|'.join(map(str, episodes)) + '|'
# get the current timestamp # get the current timestamp
curTimestamp = int(time.mktime(datetime.datetime.today().timetuple())) curTimestamp = int(time.mktime(datetime.datetime.today().timetuple()))
@ -271,10 +274,10 @@ class TVCache():
# get version # get version
version = parse_result.version version = parse_result.version
logger.log(u"Added RSS item: [" + name + "] to cache: [" + self.providerID + "]", logger.DEBUG) logger.log(u'Added RSS item: [' + name + '] to cache: [' + self.providerID + ']', logger.DEBUG)
return [ return [
"INSERT OR IGNORE INTO [" + self.providerID + "] (name, season, episodes, indexerid, url, time, quality, release_group, version) VALUES (?,?,?,?,?,?,?,?,?)", 'INSERT OR IGNORE INTO [' + self.providerID + '] (name, season, episodes, indexerid, url, time, quality, release_group, version) VALUES (?,?,?,?,?,?,?,?,?)',
[name, season, episodeText, parse_result.show.indexerid, url, curTimestamp, quality, release_group, version]] [name, season, episodeText, parse_result.show.indexerid, url, curTimestamp, quality, release_group, version]]
@ -285,12 +288,12 @@ class TVCache():
else: else:
return [] return []
def listPropers(self, date=None, delimiter="."): def listPropers(self, date=None, delimiter='.'):
myDB = self._getDB() myDB = self._getDB()
sql = "SELECT * FROM [" + self.providerID + "] WHERE name LIKE '%.PROPER.%' OR name LIKE '%.REPACK.%'" sql = "SELECT * FROM [" + self.providerID + "] WHERE name LIKE '%.PROPER.%' OR name LIKE '%.REPACK.%'"
if date != None: if date != None:
sql += " AND time >= " + str(int(time.mktime(date.timetuple()))) sql += ' AND time >= ' + str(int(time.mktime(date.timetuple())))
return filter(lambda x: x['indexerid'] != 0, myDB.select(sql)) return filter(lambda x: x['indexerid'] != 0, myDB.select(sql))
@ -302,14 +305,14 @@ class TVCache():
myDB = self._getDB() myDB = self._getDB()
if type(episode) != list: if type(episode) != list:
sqlResults = myDB.select( sqlResults = myDB.select(
"SELECT * FROM [" + self.providerID + "] WHERE indexerid = ? AND season = ? AND episodes LIKE ?", 'SELECT * FROM [' + self.providerID + '] WHERE indexerid = ? AND season = ? AND episodes LIKE ?',
[episode.show.indexerid, episode.season, "%|" + str(episode.episode) + "|%"]) [episode.show.indexerid, episode.season, '%|' + str(episode.episode) + '|%'])
else: else:
for epObj in episode: for epObj in episode:
cl.append([ cl.append([
"SELECT * FROM [" + self.providerID + "] WHERE indexerid = ? AND season = ? AND episodes LIKE ? " 'SELECT * FROM [' + self.providerID + '] WHERE indexerid = ? AND season = ? AND episodes LIKE ? '
"AND quality IN (" + ",".join([str(x) for x in epObj.wantedQuality]) + ")", 'AND quality IN (' + ','.join([str(x) for x in epObj.wantedQuality]) + ')',
[epObj.show.indexerid, epObj.season, "%|" + str(epObj.episode) + "|%"]]) [epObj.show.indexerid, epObj.season, '%|' + str(epObj.episode) + '|%']])
sqlResults = myDB.mass_action(cl, fetchall=True) sqlResults = myDB.mass_action(cl, fetchall=True)
sqlResults = list(itertools.chain(*sqlResults)) sqlResults = list(itertools.chain(*sqlResults))
@ -318,45 +321,45 @@ class TVCache():
for curResult in sqlResults: for curResult in sqlResults:
# skip non-tv crap # skip non-tv crap
if not show_name_helpers.filterBadReleases(curResult["name"], parse=False): if not show_name_helpers.filterBadReleases(curResult['name'], parse=False):
continue continue
# get the show object, or if it's not one of our shows then ignore it # get the show object, or if it's not one of our shows then ignore it
showObj = helpers.findCertainShow(sickbeard.showList, int(curResult["indexerid"])) showObj = helpers.findCertainShow(sickbeard.showList, int(curResult['indexerid']))
if not showObj: if not showObj:
continue continue
# skip if provider is anime only and show is not anime # skip if provider is anime only and show is not anime
if self.provider.anime_only and not showObj.is_anime: if self.provider.anime_only and not showObj.is_anime:
logger.log(u"" + str(showObj.name) + " is not an anime, skiping", logger.DEBUG) logger.log(u'' + str(showObj.name) + ' is not an anime, skiping', logger.DEBUG)
continue continue
# get season and ep data (ignoring multi-eps for now) # get season and ep data (ignoring multi-eps for now)
curSeason = int(curResult["season"]) curSeason = int(curResult['season'])
if curSeason == -1: if curSeason == -1:
continue continue
curEp = curResult["episodes"].split("|")[1] curEp = curResult['episodes'].split('|')[1]
if not curEp: if not curEp:
continue continue
curEp = int(curEp) curEp = int(curEp)
curQuality = int(curResult["quality"]) curQuality = int(curResult['quality'])
curReleaseGroup = curResult["release_group"] curReleaseGroup = curResult['release_group']
curVersion = curResult["version"] curVersion = curResult['version']
# if the show says we want that episode then add it to the list # if the show says we want that episode then add it to the list
if not showObj.wantEpisode(curSeason, curEp, curQuality, manualSearch): if not showObj.wantEpisode(curSeason, curEp, curQuality, manualSearch):
logger.log(u"Skipping " + curResult["name"] + " because we don't want an episode that's " + logger.log(u'Skipping ' + curResult['name'] + ' because we don\'t want an episode that\'s ' +
Quality.qualityStrings[curQuality], logger.DEBUG) Quality.qualityStrings[curQuality], logger.DEBUG)
continue continue
epObj = showObj.getEpisode(curSeason, curEp) epObj = showObj.getEpisode(curSeason, curEp)
# build a result object # build a result object
title = curResult["name"] title = curResult['name']
url = curResult["url"] url = curResult['url']
logger.log(u"Found result " + title + " at " + url) logger.log(u'Found result ' + title + ' at ' + url)
result = self.provider.getResult([epObj]) result = self.provider.getResult([epObj])
result.show = showObj result.show = showObj