monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-12-01 00:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix for daily searches and high cpu usage plus increases search speed

Browse source
This commit is contained in:
echel0n 2014-09-15 01:28:11 -07:00
parent 442631105b
commit 67bd1a9e98
6 changed files with 108 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
lib/tvdb_api/tvdb_api.py
View file

@ -815,7 +815,7 @@ class Tvdb:
self._setShowData(sid, '_actors', cur_actors) self._setShowData(sid, '_actors', cur_actors)
def _getShowData(self, sid, language, seriesSearch=False): def _getShowData(self, sid, language, getEpInfo=False):
"""Takes a series ID, gets the epInfo URL and parses the TVDB """Takes a series ID, gets the epInfo URL and parses the TVDB
XML file into the shows dict in layout: XML file into the shows dict in layout:
shows[series_id][season_number][episode_number] shows[series_id][season_number][episode_number]
@ -854,62 +854,60 @@ class Tvdb:
self._setShowData(sid, k, v) self._setShowData(sid, k, v)
if seriesSearch: if getEpInfo:
return True # Parse banners
if self.config['banners_enabled']:
self._parseBanners(sid)
# Parse banners # Parse actors
if self.config['banners_enabled']: if self.config['actors_enabled']:
self._parseBanners(sid) self._parseActors(sid)
# Parse actors # Parse episode data
if self.config['actors_enabled']: log().debug('Getting all episodes of %s' % (sid))
self._parseActors(sid)
# Parse episode data if self.config['useZip']:
log().debug('Getting all episodes of %s' % (sid)) url = self.config['url_epInfo_zip'] % (sid, language)
if self.config['useZip']:
url = self.config['url_epInfo_zip'] % (sid, language)
else:
url = self.config['url_epInfo'] % (sid, language)
epsEt = self._getetsrc(url, language=language)
episodes = epsEt["episode"]
if not isinstance(episodes, list):
episodes = [episodes]
for cur_ep in episodes:
if self.config['dvdorder']:
log().debug('Using DVD ordering.')
use_dvd = cur_ep['dvd_season'] != None and cur_ep['dvd_episodenumber'] != None
else: else:
use_dvd = False url = self.config['url_epInfo'] % (sid, language)
if use_dvd: epsEt = self._getetsrc(url, language=language)
seasnum, epno = cur_ep['dvd_season'], cur_ep['dvd_episodenumber']
else:
seasnum, epno = cur_ep['seasonnumber'], cur_ep['episodenumber']
if seasnum is None or epno is None: episodes = epsEt["episode"]
log().warning("An episode has incomplete season/episode number (season: %r, episode: %r)" % ( if not isinstance(episodes, list):
seasnum, epno)) episodes = [episodes]
continue # Skip to next episode
# float() is because https://github.com/dbr/tvnamer/issues/95 - should probably be fixed in TVDB data for cur_ep in episodes:
seas_no = int(float(seasnum)) if self.config['dvdorder']:
ep_no = int(float(epno)) log().debug('Using DVD ordering.')
use_dvd = cur_ep['dvd_season'] != None and cur_ep['dvd_episodenumber'] != None
else:
use_dvd = False
for k, v in cur_ep.items(): if use_dvd:
k = k.lower() seasnum, epno = cur_ep['dvd_season'], cur_ep['dvd_episodenumber']
else:
seasnum, epno = cur_ep['seasonnumber'], cur_ep['episodenumber']
if v is not None: if seasnum is None or epno is None:
if k == 'filename': log().warning("An episode has incomplete season/episode number (season: %r, episode: %r)" % (
v = self.config['url_artworkPrefix'] % (v) seasnum, epno))
else: continue # Skip to next episode
v = self._cleanData(v)
self._setItem(sid, seas_no, ep_no, k, v) # float() is because https://github.com/dbr/tvnamer/issues/95 - should probably be fixed in TVDB data
seas_no = int(float(seasnum))
ep_no = int(float(epno))
for k, v in cur_ep.items():
k = k.lower()
if v is not None:
if k == 'filename':
v = self.config['url_artworkPrefix'] % (v)
else:
v = self._cleanData(v)
self._setItem(sid, seas_no, ep_no, k, v)
return True return True
@ -927,7 +925,7 @@ class Tvdb:
if isinstance(selected_series, dict): if isinstance(selected_series, dict):
selected_series = [selected_series] selected_series = [selected_series]
sids = list(int(x['id']) for x in selected_series if sids = list(int(x['id']) for x in selected_series if
self._getShowData(int(x['id']), self.config['language'], seriesSearch=True)) self._getShowData(int(x['id']), self.config['language']))
self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series)) self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series))
return sids return sids
@ -938,7 +936,7 @@ class Tvdb:
if isinstance(key, (int, long)): if isinstance(key, (int, long)):
# Item is integer, treat as show id # Item is integer, treat as show id
if key not in self.shows: if key not in self.shows:
self._getShowData(key, self.config['language']) self._getShowData(key, self.config['language'], True)
return self.shows[key] return self.shows[key]
key = str(key).lower() key = str(key).lower()

62
lib/tvrage_api/tvrage_api.py
View file

@ -577,7 +577,7 @@ class TVRage:
return ui.selectSeries(allSeries) return ui.selectSeries(allSeries)
def _getShowData(self, sid, seriesSearch=False): def _getShowData(self, sid, getEpInfo=False):
"""Takes a series ID, gets the epInfo URL and parses the TVRAGE """Takes a series ID, gets the epInfo URL and parses the TVRAGE
XML file into the shows dict in layout: XML file into the shows dict in layout:
shows[series_id][season_number][episode_number] shows[series_id][season_number][episode_number]
@ -602,41 +602,39 @@ class TVRage:
self._setShowData(sid, k, v) self._setShowData(sid, k, v)
# series search ends here # series search ends here
if seriesSearch: if getEpInfo:
return True # Parse episode data
log().debug('Getting all episodes of %s' % (sid))
# Parse episode data self.config['params_epInfo']['sid'] = sid
log().debug('Getting all episodes of %s' % (sid)) epsEt = self._getetsrc(self.config['url_epInfo'], self.config['params_epInfo'])
self.config['params_epInfo']['sid'] = sid seasons = epsEt['episodelist']['season']
epsEt = self._getetsrc(self.config['url_epInfo'], self.config['params_epInfo']) if not isinstance(seasons, list):
seasons = [seasons]
seasons = epsEt['episodelist']['season'] for season in seasons:
if not isinstance(seasons, list): seas_no = int(season['@no'])
seasons = [seasons] episodes = season['episode']
if not isinstance(episodes, list):
episodes = [episodes]
for season in seasons: for episode in episodes:
seas_no = int(season['@no']) ep_no = int(episode['episodenumber'])
episodes = season['episode'] self._setItem(sid, seas_no, ep_no, 'seasonnumber', seas_no)
if not isinstance(episodes, list):
episodes = [episodes]
for episode in episodes: for k, v in episode.items():
ep_no = int(episode['episodenumber']) try:
self._setItem(sid, seas_no, ep_no, 'seasonnumber', seas_no) k = k.lower()
if v is not None:
if k == 'link':
v = v.rsplit('/', 1)[1]
k = 'id'
v = self._cleanData(v)
for k, v in episode.items(): self._setItem(sid, seas_no, ep_no, k, v)
try: except:
k = k.lower() continue
if v is not None:
if k == 'link':
v = v.rsplit('/', 1)[1]
k = 'id'
v = self._cleanData(v)
self._setItem(sid, seas_no, ep_no, k, v)
except:
continue
return True return True
def _nameToSid(self, name): def _nameToSid(self, name):
@ -652,7 +650,7 @@ class TVRage:
selected_series = self._getSeries(name) selected_series = self._getSeries(name)
if isinstance(selected_series, dict): if isinstance(selected_series, dict):
selected_series = [selected_series] selected_series = [selected_series]
sids = list(int(x['id']) for x in selected_series if self._getShowData(int(x['id']), seriesSearch=True)) sids = list(int(x['id']) for x in selected_series if self._getShowData(int(x['id'])))
self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series)) self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series))
return sids return sids
@ -663,7 +661,7 @@ class TVRage:
if isinstance(key, (int, long)): if isinstance(key, (int, long)):
# Item is integer, treat as show id # Item is integer, treat as show id
if key not in self.shows: if key not in self.shows:
self._getShowData(key) self._getShowData(key, True)
return self.shows[key] return self.shows[key]
key = str(key).lower() key = str(key).lower()

16
sickbeard/dailysearcher.py
View file

@ -50,15 +50,13 @@ class DailySearcher():
sql_l = [] sql_l = []
show = None show = None
wantedEp = {}
for sqlEp in sqlResults: for sqlEp in sqlResults:
try: try:
if not show or (show and int(sqlEp["showid"]) != show.indexerid): if not show or int(sqlEp["showid"]) != show.indexerid:
show = helpers.findCertainShow(sickbeard.showList, int(sqlEp["showid"])) show = helpers.findCertainShow(sickbeard.showList, int(sqlEp["showid"]))
wantedEp[show] = []
# build name cache for show
sickbeard.name_cache.buildNameCache(show)
except exceptions.MultipleShowObjectsException: except exceptions.MultipleShowObjectsException:
logger.log(u"ERROR: expected to find a single show matching " + sqlEp["showid"]) logger.log(u"ERROR: expected to find a single show matching " + sqlEp["showid"])
@ -70,6 +68,7 @@ class DailySearcher():
ep.status = common.SKIPPED ep.status = common.SKIPPED
else: else:
ep.status = common.WANTED ep.status = common.WANTED
wantedEp[show].append(ep)
sql_l.append(ep.get_sql()) sql_l.append(ep.get_sql())
@ -77,8 +76,9 @@ class DailySearcher():
myDB = db.DBConnection() myDB = db.DBConnection()
myDB.mass_action(sql_l) myDB.mass_action(sql_l)
# queue episode for daily search for show, episode in wantedEp.items():
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem() # queue episode for daily search
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, episode)
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
self.amActive = False self.amActive = False

9
sickbeard/providers/generic.py
View file

@ -203,8 +203,13 @@ class GenericProvider:
return True return True
def searchRSS(self): def searchRSS(self, episode):
return self.cache.findNeededEpisodes() results = {}
for ep in episode:
results += self.cache.findNeededEpisodes(ep)
return results
def getQuality(self, item, anime=False): def getQuality(self, item, anime=False):
""" """

4
sickbeard/search.py
View file

@ -319,7 +319,7 @@ def isFirstBestMatch(result):
return False return False
def searchForNeededEpisodes(): def searchForNeededEpisodes(episode):
foundResults = {} foundResults = {}
didSearch = False didSearch = False
@ -333,7 +333,7 @@ def searchForNeededEpisodes():
try: try:
curProvider.cache.updateCache() curProvider.cache.updateCache()
curFoundResults = curProvider.searchRSS() curFoundResults = curProvider.searchRSS(episode)
except exceptions.AuthException, e: except exceptions.AuthException, e:
logger.log(u"Authentication error: " + ex(e), logger.ERROR) logger.log(u"Authentication error: " + ex(e), logger.ERROR)
continue continue

29
sickbeard/search_queue.py
View file

@ -92,14 +92,11 @@ class SearchQueue(generic_queue.GenericQueue):
def add_item(self, item): def add_item(self, item):
if isinstance(item, DailySearchQueueItem): if isinstance(item, (DailySearchQueueItem, BacklogQueueItem)) and not self.is_in_queue(item.show, item.segment):
# daily searches
generic_queue.GenericQueue.add_item(self, item)
elif isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment):
# build name cache for show # build name cache for show
sickbeard.name_cache.buildNameCache(item.show) sickbeard.name_cache.buildNameCache(item.show)
# backlog searches # daily and backlog searches
generic_queue.GenericQueue.add_item(self, item) generic_queue.GenericQueue.add_item(self, item)
elif isinstance(item, (ManualSearchQueueItem, FailedQueueItem)) and not self.is_ep_in_queue(item.segment): elif isinstance(item, (ManualSearchQueueItem, FailedQueueItem)) and not self.is_ep_in_queue(item.segment):
# build name cache for show # build name cache for show
@ -111,18 +108,20 @@ class SearchQueue(generic_queue.GenericQueue):
logger.log(u"Not adding item, it's already in the queue", logger.DEBUG) logger.log(u"Not adding item, it's already in the queue", logger.DEBUG)
class DailySearchQueueItem(generic_queue.QueueItem): class DailySearchQueueItem(generic_queue.QueueItem):
def __init__(self): def __init__(self, show, segment):
generic_queue.QueueItem.__init__(self, 'Daily Search', DAILY_SEARCH) generic_queue.QueueItem.__init__(self, 'Daily Search', DAILY_SEARCH)
self.show = show
self.segment = segment
def run(self): def run(self):
generic_queue.QueueItem.run(self) generic_queue.QueueItem.run(self)
try: try:
logger.log("Beginning daily search for new episodes") logger.log("Beginning daily search for: [" + self.show.name + "]")
foundResults = search.searchForNeededEpisodes() foundResults = search.searchForNeededEpisodes(segment)
if not len(foundResults): if not len(foundResults):
logger.log(u"No needed episodes found") logger.log(u"No needed episodes found during daily search for: [" + self.show.name + "]")
else: else:
for result in foundResults: for result in foundResults:
# just use the first result for now # just use the first result for now
@ -152,7 +151,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
generic_queue.QueueItem.run(self) generic_queue.QueueItem.run(self)
try: try:
logger.log("Beginning manual search for [" + self.segment.prettyName() + "]") logger.log("Beginning manual search for: [" + self.segment.prettyName() + "]")
searchResult = search.searchProviders(self.show, [self.segment], True) searchResult = search.searchProviders(self.show, [self.segment], True)
if searchResult: if searchResult:
@ -167,7 +166,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
ui.notifications.message('No downloads were found', ui.notifications.message('No downloads were found',
"Couldn't find a download for <i>%s</i>" % self.segment.prettyName()) "Couldn't find a download for <i>%s</i>" % self.segment.prettyName())
logger.log(u"Unable to find a download for " + self.segment.prettyName()) logger.log(u"Unable to find a download for: [" + self.segment.prettyName() + "]")
except Exception: except Exception:
logger.log(traceback.format_exc(), logger.DEBUG) logger.log(traceback.format_exc(), logger.DEBUG)
@ -191,7 +190,7 @@ class BacklogQueueItem(generic_queue.QueueItem):
generic_queue.QueueItem.run(self) generic_queue.QueueItem.run(self)
try: try:
logger.log("Beginning backlog search for [" + self.show.name + "]") logger.log("Beginning backlog search for: [" + self.show.name + "]")
searchResult = search.searchProviders(self.show, self.segment, False) searchResult = search.searchProviders(self.show, self.segment, False)
if searchResult: if searchResult:
@ -203,7 +202,7 @@ class BacklogQueueItem(generic_queue.QueueItem):
# give the CPU a break # give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET]) time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
else: else:
logger.log(u"No needed episodes found during backlog search for [" + self.show.name + "]") logger.log(u"No needed episodes found during backlog search for: [" + self.show.name + "]")
except Exception: except Exception:
logger.log(traceback.format_exc(), logger.DEBUG) logger.log(traceback.format_exc(), logger.DEBUG)
@ -232,7 +231,7 @@ class FailedQueueItem(generic_queue.QueueItem):
history.logFailed(self.segment, release, provider) history.logFailed(self.segment, release, provider)
failed_history.revertEpisode(self.segment) failed_history.revertEpisode(self.segment)
logger.log("Beginning failed download search for [" + self.segment.prettyName() + "]") logger.log("Beginning failed download search for: [" + self.segment.prettyName() + "]")
searchResult = search.searchProviders(self.show, [self.segment], True) searchResult = search.searchProviders(self.show, [self.segment], True)
@ -245,7 +244,7 @@ class FailedQueueItem(generic_queue.QueueItem):
# give the CPU a break # give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET]) time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
else: else:
logger.log(u"No valid episode found to retry for [" + self.segment.prettyName() + "]") logger.log(u"No valid episode found to retry for: [" + self.segment.prettyName() + "]")
except Exception: except Exception:
logger.log(traceback.format_exc(), logger.DEBUG) logger.log(traceback.format_exc(), logger.DEBUG)