Improved find propers code.

Backlog and manual searches now check cache first for search results.
This commit is contained in:
echel0n 2014-05-30 00:36:47 -07:00
parent 1abef89b29
commit f01c5852d4
13 changed files with 114 additions and 162 deletions

View file

@ -76,6 +76,7 @@ class FailedProcessor(object):
raise exceptions.FailedProcessingFailed()
segment = {parsed.season_number:[]}
for episode in parsed.episode_numbers:
epObj = parsed.show.getEpisode(parsed.season_number, episode)
segment[parsed.season_number].append(epObj)

View file

@ -116,11 +116,10 @@ class ProperFinder():
sortedPropers = sorted(propers.values(), key=operator.attrgetter('date'), reverse=True)
finalPropers = []
for curProper in sortedPropers:
in_cache = False
try:
myParser = NameParser(False)
parse_result = myParser.parse(curProper.name)
parse_result = myParser.parse(curProper.name).convert()
except InvalidNameException:
logger.log(u"Unable to parse the filename " + curProper.name + " into a valid episode", logger.DEBUG)
continue
@ -128,37 +127,26 @@ class ProperFinder():
if not parse_result.series_name:
continue
cacheResult = sickbeard.name_cache.retrieveNameFromCache(parse_result.series_name)
if cacheResult:
in_cache = True
curProper.indexerid = int(cacheResult)
elif cacheResult == 0:
return None
if not curProper.indexerid:
showResult = helpers.searchDBForShow(parse_result.series_name)
if showResult:
curProper.indexerid = int(showResult[0])
if not curProper.indexerid:
for curShow in sickbeard.showList:
if show_name_helpers.isGoodResult(curProper.name, curShow, False):
curProper.indexerid = curShow.indexerid
break
if not parse_result.show:
sickbeard.name_cache.addNameToCache(parse_result.series_name, 0)
continue
if not in_cache:
sickbeard.name_cache.addNameToCache(parse_result.series_name, parse_result.show.indexerid)
if not parse_result.episode_numbers:
logger.log(
u"Ignoring " + curProper.name + " because it's for a full season rather than specific episode",
logger.DEBUG)
continue
showObj = parse_result.show
logger.log(
u"Successful match! Result " + parse_result.series_name + " matched to show " + showObj.name,
logger.DEBUG)
# set the indexerid in the db to the show's indexerid
curProper.indexerid = showObj.indexerid
# set the indexer in the db to the show's indexer
curProper.indexer = showObj.indexer
# populate our Proper instance
if parse_result.air_by_date or parse_result.sports:
curProper.season = -1
@ -167,53 +155,17 @@ class ProperFinder():
if parse_result.is_anime:
logger.log(u"I am sorry '"+curProper.name+"' seams to be an anime proper seach is not yet suported", logger.DEBUG)
continue
curProper.episode = parse_result.ab_episode_numbers[0]
else:
curProper.season = parse_result.season_number if parse_result.season_number != None else 1
curProper.episode = parse_result.episode_numbers[0]
curProper.quality = Quality.nameQuality(curProper.name, parse_result.is_anime)
# for each show in our list
for curShow in sickbeard.showList:
genericName = self._genericName(parse_result.series_name)
# get the scene name masks
sceneNames = set(show_name_helpers.makeSceneShowSearchStrings(curShow))
# for each scene name mask
for curSceneName in sceneNames:
# if it matches
if genericName == self._genericName(curSceneName):
logger.log(
u"Successful match! Result " + parse_result.series_name + " matched to show " + curShow.name,
logger.DEBUG)
# set the indexerid in the db to the show's indexerid
curProper.indexerid = curShow.indexerid
# set the indexer in the db to the show's indexer
curProper.indexer = curShow.indexer
# since we found it, break out
break
# if we found something in the inner for loop break out of this one
if curProper.indexerid != -1:
break
if not show_name_helpers.filterBadReleases(curProper.name):
logger.log(u"Proper " + curProper.name + " isn't a valid scene release that we want, igoring it",
logger.DEBUG)
continue
showObj = helpers.findCertainShow(sickbeard.showList, curProper.indexerid)
if not showObj:
logger.log(u"Unable to find the show with indexerID " + str(curProper.indexerid), logger.ERROR)
continue
if showObj.rls_ignore_words and search.filter_release_name(curProper.name, showObj.rls_ignore_words):
logger.log(u"Ignoring " + curProper.name + " based on ignored words filter: " + showObj.rls_ignore_words,
logger.MESSAGE)

View file

@ -248,21 +248,15 @@ class GenericProvider:
# mark season searched for season pack searches so we can skip later on
searched_scene_season = epObj.scene_season
if not epObj.show.air_by_date:
if epObj.scene_season == 0 or epObj.scene_episode == 0:
logger.log(
u"Incomplete Indexer <-> Scene mapping detected for " + epObj.prettyName() + ", skipping search!")
continue
# cacheResult = self.cache.searchCache([epObj], manualSearch)
#if len(cacheResult):
# results.update({epObj.episode:cacheResult[epObj]})
# continue
if search_mode == 'sponly':
for curString in self._get_season_search_strings(epObj):
itemList += self._doSearch(curString, len(episodes))
else:
cacheResult = self.cache.searchCache([epObj], manualSearch)
if len(cacheResult):
results.update({epObj.episode: cacheResult[epObj]})
continue
for curString in self._get_episode_search_strings(epObj):
itemList += self._doSearch(curString, len(episodes))
@ -271,12 +265,11 @@ class GenericProvider:
continue
# remove duplicate items
#itemList = [i for n, i in enumerate(itemList) if i not in itemList[n + 1:]]
searchItems[epObj] = itemList
# if we have cached results return them.
# if len(results):
# return results
#if we have cached results return them.
if len(results):
return results
for ep_obj in searchItems:
for item in searchItems[ep_obj]:

View file

@ -261,14 +261,14 @@ class IPTorrentsProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -397,15 +397,15 @@ class KATProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -309,14 +309,14 @@ class NextGenProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -288,15 +288,15 @@ class PublicHDProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -303,15 +303,15 @@ class SCCProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -11,7 +11,7 @@
# SickRage is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickRage. If not, see <http://www.gnu.org/licenses/>.
@ -37,14 +37,14 @@ from lib import requests
from lib.requests import exceptions
from sickbeard.helpers import sanitizeSceneName
class SpeedCDProvider(generic.TorrentProvider):
class SpeedCDProvider(generic.TorrentProvider):
urls = {'base_url': 'http://speed.cd/',
'login': 'http://speed.cd/takelogin.php',
'detail': 'http://speed.cd/t/%s',
'search': 'http://speed.cd/V3/API/API.php',
'download': 'http://speed.cd/download.php?torrent=%s',
}
}
def __init__(self):
@ -64,7 +64,7 @@ class SpeedCDProvider(generic.TorrentProvider):
self.url = self.urls['base_url']
self.categories = {'Season': {'c14':1}, 'Episode': {'c2':1, 'c49':1}, 'RSS': {'c14':1, 'c2':1, 'c49':1}}
self.categories = {'Season': {'c14': 1}, 'Episode': {'c2': 1, 'c49': 1}, 'RSS': {'c14': 1, 'c2': 1, 'c49': 1}}
def isEnabled(self):
return self.enabled
@ -81,7 +81,7 @@ class SpeedCDProvider(generic.TorrentProvider):
login_params = {'username': self.username,
'password': self.password
}
}
try:
response = self.session.post(self.urls['login'], data=login_params, timeout=30, verify=False)
@ -90,7 +90,7 @@ class SpeedCDProvider(generic.TorrentProvider):
return False
if re.search('Incorrect username or Password. Please try again.', response.text) \
or response.status_code == 401:
or response.status_code == 401:
logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR)
return False
@ -104,7 +104,7 @@ class SpeedCDProvider(generic.TorrentProvider):
if ep_obj.show.air_by_date or ep_obj.show.sports:
ep_string = show_name + str(ep_obj.airdate).split('-')[0]
else:
ep_string = show_name +' S%02d' % int(ep_obj.scene_season) #1) showName SXX
ep_string = show_name + ' S%02d' % int(ep_obj.scene_season) #1) showName SXX
search_string['Season'].append(ep_string)
@ -130,8 +130,9 @@ class SpeedCDProvider(generic.TorrentProvider):
search_string['Episode'].append(ep_string)
else:
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
ep_string = show_name_helpers.sanitizeSceneName(show_name) +' '+ \
sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season, 'episodenumber': ep_obj.scene_episode}
ep_string = show_name_helpers.sanitizeSceneName(show_name) + ' ' + \
sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
'episodenumber': ep_obj.scene_episode}
search_string['Episode'].append(re.sub('\s+', ' ', ep_string))
@ -152,7 +153,8 @@ class SpeedCDProvider(generic.TorrentProvider):
search_string = '+'.join(search_string.split())
post_data = dict({'/browse.php?' : None, 'cata': 'yes','jxt': 4,'jxw': 'b','search': search_string}, **self.categories[mode])
post_data = dict({'/browse.php?': None, 'cata': 'yes', 'jxt': 4, 'jxw': 'b', 'search': search_string},
**self.categories[mode])
data = self.session.post(self.urls['search'], data=post_data).json()
@ -167,7 +169,7 @@ class SpeedCDProvider(generic.TorrentProvider):
continue
title = re.sub('<[^>]*>', '', torrent['name'])
url = self.urls['download'] %(torrent['id'])
url = self.urls['download'] % (torrent['id'])
seeders = int(torrent['seed'])
leechers = int(torrent['leech'])
@ -192,7 +194,7 @@ class SpeedCDProvider(generic.TorrentProvider):
title, url, seeders, leechers = item
if url:
url = str(url).replace('&amp;','&')
url = str(url).replace('&amp;', '&')
return (title, url)
@ -216,11 +218,12 @@ class SpeedCDProvider(generic.TorrentProvider):
else:
r = self.session.get(url, verify=False)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
logger.log(u"Error loading "+self.name+" URL: " + ex(e), logger.ERROR)
logger.log(u"Error loading " + self.name + " URL: " + ex(e), logger.ERROR)
return None
if r.status_code != 200:
logger.log(self.name + u" page requested with url " + url +" returned status code is " + str(r.status_code) + ': ' + clients.http_error_code[r.status_code], logger.WARNING)
logger.log(self.name + u" page requested with url " + url + " returned status code is " + str(
r.status_code) + ': ' + clients.http_error_code[r.status_code], logger.WARNING)
return None
return r.content
@ -229,33 +232,34 @@ class SpeedCDProvider(generic.TorrentProvider):
results = []
sqlResults = db.DBConnection().select('SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
' WHERE e.airdate >= ' + str(search_date.toordinal()) +
' AND (e.status IN (' + ','.join([str(x) for x in Quality.DOWNLOADED]) + ')' +
' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))'
)
sqlResults = db.DBConnection().select(
'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
' WHERE e.airdate >= ' + str(search_date.toordinal()) +
' AND (e.status IN (' + ','.join([str(x) for x in Quality.DOWNLOADED]) + ')' +
' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))'
)
if not sqlResults:
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results
def seedRatio(self):
return self.ratio
class SpeedCDCache(tvcache.TVCache):
class SpeedCDCache(tvcache.TVCache):
def __init__(self, provider):
tvcache.TVCache.__init__(self, provider)
@ -298,9 +302,10 @@ class SpeedCDCache(tvcache.TVCache):
if not title or not url:
return None
logger.log(u"Attempting to cache item:[" + title +"]", logger.DEBUG)
logger.log(u"Attempting to cache item:[" + title + "]", logger.DEBUG)
return self._addCacheEntry(title, url)
provider = SpeedCDProvider()

View file

@ -377,15 +377,16 @@ class ThePirateBayProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -266,15 +266,15 @@ class TorrentDayProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -262,15 +262,15 @@ class TorrentLeechProvider(generic.TorrentProvider):
return []
for sqlshow in sqlResults:
self.show = curshow = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if not self.show: continue
curEp = curshow.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
for item in self._doSearch(searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today()))
return results

View file

@ -195,10 +195,10 @@ class TVShow(object):
return ep_list
def getEpisode(self, season=None, episode=None, file=None, noCreate=False, absolute_number=None):
def getEpisode(self, season=None, episode=None, file=None, noCreate=False, absolute_number=None, forceUpdate=False):
# Load XEM data to DB for show
sickbeard.scene_numbering.xem_refresh(self.indexerid, self.indexer)
sickbeard.scene_numbering.xem_refresh(self.indexerid, self.indexer, force=forceUpdate)
if not season in self.episodes:
self.episodes[season] = {}