From 67bd1a9e983d174912e378690e359ac1c070c403 Mon Sep 17 00:00:00 2001 From: echel0n Date: Mon, 15 Sep 2014 01:28:11 -0700 Subject: [PATCH] Fix for daily searches and high cpu usage plus increases search speed --- lib/tvdb_api/tvdb_api.py | 96 +++++++++++++++++----------------- lib/tvrage_api/tvrage_api.py | 62 +++++++++++----------- sickbeard/dailysearcher.py | 16 +++--- sickbeard/providers/generic.py | 9 +++- sickbeard/search.py | 4 +- sickbeard/search_queue.py | 29 +++++----- 6 files changed, 108 insertions(+), 108 deletions(-) diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index ed4c0d34..6b64c82d 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -815,7 +815,7 @@ class Tvdb: self._setShowData(sid, '_actors', cur_actors) - def _getShowData(self, sid, language, seriesSearch=False): + def _getShowData(self, sid, language, getEpInfo=False): """Takes a series ID, gets the epInfo URL and parses the TVDB XML file into the shows dict in layout: shows[series_id][season_number][episode_number] @@ -854,62 +854,60 @@ class Tvdb: self._setShowData(sid, k, v) - if seriesSearch: - return True + if getEpInfo: + # Parse banners + if self.config['banners_enabled']: + self._parseBanners(sid) - # Parse banners - if self.config['banners_enabled']: - self._parseBanners(sid) + # Parse actors + if self.config['actors_enabled']: + self._parseActors(sid) - # Parse actors - if self.config['actors_enabled']: - self._parseActors(sid) + # Parse episode data + log().debug('Getting all episodes of %s' % (sid)) - # Parse episode data - log().debug('Getting all episodes of %s' % (sid)) - - if self.config['useZip']: - url = self.config['url_epInfo_zip'] % (sid, language) - else: - url = self.config['url_epInfo'] % (sid, language) - - epsEt = self._getetsrc(url, language=language) - - episodes = epsEt["episode"] - if not isinstance(episodes, list): - episodes = [episodes] - - for cur_ep in episodes: - if self.config['dvdorder']: - log().debug('Using DVD ordering.') - use_dvd = cur_ep['dvd_season'] != None and cur_ep['dvd_episodenumber'] != None + if self.config['useZip']: + url = self.config['url_epInfo_zip'] % (sid, language) else: - use_dvd = False + url = self.config['url_epInfo'] % (sid, language) - if use_dvd: - seasnum, epno = cur_ep['dvd_season'], cur_ep['dvd_episodenumber'] - else: - seasnum, epno = cur_ep['seasonnumber'], cur_ep['episodenumber'] + epsEt = self._getetsrc(url, language=language) - if seasnum is None or epno is None: - log().warning("An episode has incomplete season/episode number (season: %r, episode: %r)" % ( - seasnum, epno)) - continue # Skip to next episode + episodes = epsEt["episode"] + if not isinstance(episodes, list): + episodes = [episodes] - # float() is because https://github.com/dbr/tvnamer/issues/95 - should probably be fixed in TVDB data - seas_no = int(float(seasnum)) - ep_no = int(float(epno)) + for cur_ep in episodes: + if self.config['dvdorder']: + log().debug('Using DVD ordering.') + use_dvd = cur_ep['dvd_season'] != None and cur_ep['dvd_episodenumber'] != None + else: + use_dvd = False - for k, v in cur_ep.items(): - k = k.lower() + if use_dvd: + seasnum, epno = cur_ep['dvd_season'], cur_ep['dvd_episodenumber'] + else: + seasnum, epno = cur_ep['seasonnumber'], cur_ep['episodenumber'] - if v is not None: - if k == 'filename': - v = self.config['url_artworkPrefix'] % (v) - else: - v = self._cleanData(v) + if seasnum is None or epno is None: + log().warning("An episode has incomplete season/episode number (season: %r, episode: %r)" % ( + seasnum, epno)) + continue # Skip to next episode - self._setItem(sid, seas_no, ep_no, k, v) + # float() is because https://github.com/dbr/tvnamer/issues/95 - should probably be fixed in TVDB data + seas_no = int(float(seasnum)) + ep_no = int(float(epno)) + + for k, v in cur_ep.items(): + k = k.lower() + + if v is not None: + if k == 'filename': + v = self.config['url_artworkPrefix'] % (v) + else: + v = self._cleanData(v) + + self._setItem(sid, seas_no, ep_no, k, v) return True @@ -927,7 +925,7 @@ class Tvdb: if isinstance(selected_series, dict): selected_series = [selected_series] sids = list(int(x['id']) for x in selected_series if - self._getShowData(int(x['id']), self.config['language'], seriesSearch=True)) + self._getShowData(int(x['id']), self.config['language'])) self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series)) return sids @@ -938,7 +936,7 @@ class Tvdb: if isinstance(key, (int, long)): # Item is integer, treat as show id if key not in self.shows: - self._getShowData(key, self.config['language']) + self._getShowData(key, self.config['language'], True) return self.shows[key] key = str(key).lower() diff --git a/lib/tvrage_api/tvrage_api.py b/lib/tvrage_api/tvrage_api.py index e5fcbf10..86d82b8d 100644 --- a/lib/tvrage_api/tvrage_api.py +++ b/lib/tvrage_api/tvrage_api.py @@ -577,7 +577,7 @@ class TVRage: return ui.selectSeries(allSeries) - def _getShowData(self, sid, seriesSearch=False): + def _getShowData(self, sid, getEpInfo=False): """Takes a series ID, gets the epInfo URL and parses the TVRAGE XML file into the shows dict in layout: shows[series_id][season_number][episode_number] @@ -602,41 +602,39 @@ class TVRage: self._setShowData(sid, k, v) # series search ends here - if seriesSearch: - return True + if getEpInfo: + # Parse episode data + log().debug('Getting all episodes of %s' % (sid)) - # Parse episode data - log().debug('Getting all episodes of %s' % (sid)) + self.config['params_epInfo']['sid'] = sid + epsEt = self._getetsrc(self.config['url_epInfo'], self.config['params_epInfo']) - self.config['params_epInfo']['sid'] = sid - epsEt = self._getetsrc(self.config['url_epInfo'], self.config['params_epInfo']) + seasons = epsEt['episodelist']['season'] + if not isinstance(seasons, list): + seasons = [seasons] - seasons = epsEt['episodelist']['season'] - if not isinstance(seasons, list): - seasons = [seasons] + for season in seasons: + seas_no = int(season['@no']) + episodes = season['episode'] + if not isinstance(episodes, list): + episodes = [episodes] - for season in seasons: - seas_no = int(season['@no']) - episodes = season['episode'] - if not isinstance(episodes, list): - episodes = [episodes] + for episode in episodes: + ep_no = int(episode['episodenumber']) + self._setItem(sid, seas_no, ep_no, 'seasonnumber', seas_no) - for episode in episodes: - ep_no = int(episode['episodenumber']) - self._setItem(sid, seas_no, ep_no, 'seasonnumber', seas_no) + for k, v in episode.items(): + try: + k = k.lower() + if v is not None: + if k == 'link': + v = v.rsplit('/', 1)[1] + k = 'id' + v = self._cleanData(v) - for k, v in episode.items(): - try: - k = k.lower() - if v is not None: - if k == 'link': - v = v.rsplit('/', 1)[1] - k = 'id' - v = self._cleanData(v) - - self._setItem(sid, seas_no, ep_no, k, v) - except: - continue + self._setItem(sid, seas_no, ep_no, k, v) + except: + continue return True def _nameToSid(self, name): @@ -652,7 +650,7 @@ class TVRage: selected_series = self._getSeries(name) if isinstance(selected_series, dict): selected_series = [selected_series] - sids = list(int(x['id']) for x in selected_series if self._getShowData(int(x['id']), seriesSearch=True)) + sids = list(int(x['id']) for x in selected_series if self._getShowData(int(x['id']))) self.corrections.update(dict((x['seriesname'], int(x['id'])) for x in selected_series)) return sids @@ -663,7 +661,7 @@ class TVRage: if isinstance(key, (int, long)): # Item is integer, treat as show id if key not in self.shows: - self._getShowData(key) + self._getShowData(key, True) return self.shows[key] key = str(key).lower() diff --git a/sickbeard/dailysearcher.py b/sickbeard/dailysearcher.py index f64766d9..e4cf3bb1 100644 --- a/sickbeard/dailysearcher.py +++ b/sickbeard/dailysearcher.py @@ -50,15 +50,13 @@ class DailySearcher(): sql_l = [] show = None + wantedEp = {} for sqlEp in sqlResults: - try: - if not show or (show and int(sqlEp["showid"]) != show.indexerid): + if not show or int(sqlEp["showid"]) != show.indexerid: show = helpers.findCertainShow(sickbeard.showList, int(sqlEp["showid"])) - - # build name cache for show - sickbeard.name_cache.buildNameCache(show) + wantedEp[show] = [] except exceptions.MultipleShowObjectsException: logger.log(u"ERROR: expected to find a single show matching " + sqlEp["showid"]) @@ -70,6 +68,7 @@ class DailySearcher(): ep.status = common.SKIPPED else: ep.status = common.WANTED + wantedEp[show].append(ep) sql_l.append(ep.get_sql()) @@ -77,8 +76,9 @@ class DailySearcher(): myDB = db.DBConnection() myDB.mass_action(sql_l) - # queue episode for daily search - dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem() - sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) + for show, episode in wantedEp.items(): + # queue episode for daily search + dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, episode) + sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) self.amActive = False \ No newline at end of file diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index 18c9c324..59b1decb 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -203,8 +203,13 @@ class GenericProvider: return True - def searchRSS(self): - return self.cache.findNeededEpisodes() + def searchRSS(self, episode): + results = {} + + for ep in episode: + results += self.cache.findNeededEpisodes(ep) + + return results def getQuality(self, item, anime=False): """ diff --git a/sickbeard/search.py b/sickbeard/search.py index 86b665aa..b0a23951 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -319,7 +319,7 @@ def isFirstBestMatch(result): return False -def searchForNeededEpisodes(): +def searchForNeededEpisodes(episode): foundResults = {} didSearch = False @@ -333,7 +333,7 @@ def searchForNeededEpisodes(): try: curProvider.cache.updateCache() - curFoundResults = curProvider.searchRSS() + curFoundResults = curProvider.searchRSS(episode) except exceptions.AuthException, e: logger.log(u"Authentication error: " + ex(e), logger.ERROR) continue diff --git a/sickbeard/search_queue.py b/sickbeard/search_queue.py index 1703c17e..49f07de4 100644 --- a/sickbeard/search_queue.py +++ b/sickbeard/search_queue.py @@ -92,14 +92,11 @@ class SearchQueue(generic_queue.GenericQueue): def add_item(self, item): - if isinstance(item, DailySearchQueueItem): - # daily searches - generic_queue.GenericQueue.add_item(self, item) - elif isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment): + if isinstance(item, (DailySearchQueueItem, BacklogQueueItem)) and not self.is_in_queue(item.show, item.segment): # build name cache for show sickbeard.name_cache.buildNameCache(item.show) - # backlog searches + # daily and backlog searches generic_queue.GenericQueue.add_item(self, item) elif isinstance(item, (ManualSearchQueueItem, FailedQueueItem)) and not self.is_ep_in_queue(item.segment): # build name cache for show @@ -111,18 +108,20 @@ class SearchQueue(generic_queue.GenericQueue): logger.log(u"Not adding item, it's already in the queue", logger.DEBUG) class DailySearchQueueItem(generic_queue.QueueItem): - def __init__(self): + def __init__(self, show, segment): generic_queue.QueueItem.__init__(self, 'Daily Search', DAILY_SEARCH) + self.show = show + self.segment = segment def run(self): generic_queue.QueueItem.run(self) try: - logger.log("Beginning daily search for new episodes") - foundResults = search.searchForNeededEpisodes() + logger.log("Beginning daily search for: [" + self.show.name + "]") + foundResults = search.searchForNeededEpisodes(segment) if not len(foundResults): - logger.log(u"No needed episodes found") + logger.log(u"No needed episodes found during daily search for: [" + self.show.name + "]") else: for result in foundResults: # just use the first result for now @@ -152,7 +151,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem): generic_queue.QueueItem.run(self) try: - logger.log("Beginning manual search for [" + self.segment.prettyName() + "]") + logger.log("Beginning manual search for: [" + self.segment.prettyName() + "]") searchResult = search.searchProviders(self.show, [self.segment], True) if searchResult: @@ -167,7 +166,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem): ui.notifications.message('No downloads were found', "Couldn't find a download for %s" % self.segment.prettyName()) - logger.log(u"Unable to find a download for " + self.segment.prettyName()) + logger.log(u"Unable to find a download for: [" + self.segment.prettyName() + "]") except Exception: logger.log(traceback.format_exc(), logger.DEBUG) @@ -191,7 +190,7 @@ class BacklogQueueItem(generic_queue.QueueItem): generic_queue.QueueItem.run(self) try: - logger.log("Beginning backlog search for [" + self.show.name + "]") + logger.log("Beginning backlog search for: [" + self.show.name + "]") searchResult = search.searchProviders(self.show, self.segment, False) if searchResult: @@ -203,7 +202,7 @@ class BacklogQueueItem(generic_queue.QueueItem): # give the CPU a break time.sleep(common.cpu_presets[sickbeard.CPU_PRESET]) else: - logger.log(u"No needed episodes found during backlog search for [" + self.show.name + "]") + logger.log(u"No needed episodes found during backlog search for: [" + self.show.name + "]") except Exception: logger.log(traceback.format_exc(), logger.DEBUG) @@ -232,7 +231,7 @@ class FailedQueueItem(generic_queue.QueueItem): history.logFailed(self.segment, release, provider) failed_history.revertEpisode(self.segment) - logger.log("Beginning failed download search for [" + self.segment.prettyName() + "]") + logger.log("Beginning failed download search for: [" + self.segment.prettyName() + "]") searchResult = search.searchProviders(self.show, [self.segment], True) @@ -245,7 +244,7 @@ class FailedQueueItem(generic_queue.QueueItem): # give the CPU a break time.sleep(common.cpu_presets[sickbeard.CPU_PRESET]) else: - logger.log(u"No valid episode found to retry for [" + self.segment.prettyName() + "]") + logger.log(u"No valid episode found to retry for: [" + self.segment.prettyName() + "]") except Exception: logger.log(traceback.format_exc(), logger.DEBUG)