From 83bcc6368305ba1bc8d1181a765de4d23d086e2a Mon Sep 17 00:00:00 2001 From: echel0n Date: Fri, 25 Apr 2014 20:42:35 -0700 Subject: [PATCH] More RSS cache fixes --- sickbeard/helpers.py | 5 ++- sickbeard/providers/ezrss.py | 8 ++--- sickbeard/providers/generic.py | 20 +++--------- sickbeard/providers/hdbits.py | 52 ++++++++++--------------------- sickbeard/providers/newzbin.py | 2 +- sickbeard/providers/newznab.py | 23 +++----------- sickbeard/providers/nzbsrus.py | 30 ++++++++---------- sickbeard/providers/omgwtfnzbs.py | 15 +++------ sickbeard/providers/tvtorrents.py | 2 +- 9 files changed, 50 insertions(+), 107 deletions(-) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 8069413c..9be243d2 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -167,7 +167,7 @@ def sanitizeFileName(name): return name -def getURL(url, post_data=None, headers=None, params=None, timeout=None): +def getURL(url, post_data=None, headers=None, params=None, json=False): """ Returns a byte-string retrieved from the url provider. """ @@ -206,6 +206,9 @@ Returns a byte-string retrieved from the url provider. logger.log(u"Connection timed out " + str(e.message) + " while loading URL " + url, logger.WARNING) return None + if json: + return resp.json() if resp.ok else None + return resp.content if resp.ok else None diff --git a/sickbeard/providers/ezrss.py b/sickbeard/providers/ezrss.py index 046e9f10..f141e302 100644 --- a/sickbeard/providers/ezrss.py +++ b/sickbeard/providers/ezrss.py @@ -52,8 +52,7 @@ class EZRSSProvider(generic.TorrentProvider): def getQuality(self, item): - filename = helpers.get_xml_text( - item.find('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName')) + filename = item.filename quality = Quality.nameQuality(filename) return quality @@ -121,7 +120,6 @@ class EZRSSProvider(generic.TorrentProvider): items = data.entries results = [] - for curItem in items: (title, url) = self._get_title_and_url(curItem) @@ -139,9 +137,7 @@ class EZRSSProvider(generic.TorrentProvider): def _get_title_and_url(self, item): (title, url) = generic.TorrentProvider._get_title_and_url(self, item) - filename = helpers.get_xml_text( - item.find('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName')) - + filename = item.filename if filename: new_title = self._extract_name_from_filename(filename) if new_title: diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index a05f334b..a5cce3b2 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -103,7 +103,7 @@ class GenericProvider: return result - def getURL(self, url, post_data=None, headers=None): + def getURL(self, url, post_data=None, headers=None, json=False): """ By default this is just a simple urlopen call but this method should be overridden for providers with special URL requirements (like cookies) @@ -112,7 +112,7 @@ class GenericProvider: if not headers: headers = [] - data = helpers.getURL(url, post_data, headers) + data = helpers.getURL(url, post_data, headers, json=json) if not data: logger.log(u"Error loading " + self.name + " URL: " + url, logger.ERROR) @@ -244,7 +244,7 @@ class GenericProvider: self._checkAuth() # XEM episode scene numbering - sceneEpisode = copy.copy(episode) + sceneEpisode = copy.deepcopy(episode) sceneEpisode.convertToSceneNumbering() logger.log(u'Searching "%s" for "%s" as "%s"' @@ -439,16 +439,4 @@ class TorrentProvider(GenericProvider): def __init__(self, name): GenericProvider.__init__(self, name) - self.providerType = GenericProvider.TORRENT - -# self.option = {SEED_POLICY_TIME : '', -# SEED_POLICY_RATIO: '', -# 'PROCESS_METHOD': '' -# } - -# def get_provider_options(self): -# pass -# -# def set_provider_options(self): -# self.option[SEED_POLICY_TIME] + '|' + self.option[SEED_POLICY_RATIO] + '|' + self.option['PROCESS_METHOD'] - + self.providerType = GenericProvider.TORRENT \ No newline at end of file diff --git a/sickbeard/providers/hdbits.py b/sickbeard/providers/hdbits.py index 9473fe77..183e564f 100644 --- a/sickbeard/providers/hdbits.py +++ b/sickbeard/providers/hdbits.py @@ -53,17 +53,16 @@ class HDBitsProvider(generic.TorrentProvider): return True - def _checkAuthFromData(self, parsedJSON): + def _checkAuthFromData(self, data): - if parsedJSON is None: + if data is None: return self._checkAuth() - if 'status' in parsedJSON and 'message' in parsedJSON: - if parsedJSON.get('status') == 5: - logger.log(u"Incorrect authentication credentials for " + self.name + " : " + parsedJSON['message'], - logger.DEBUG) - raise AuthException( - "Your authentication credentials for " + self.name + " are incorrect, check your config.") + if data.status == 5: + logger.log(u"Incorrect authentication credentials for " + self.name + " : " + data.feed.title, + logger.DEBUG) + raise AuthException( + "Your authentication credentials for " + self.name + " are incorrect, check your config.") return True @@ -81,26 +80,19 @@ class HDBitsProvider(generic.TorrentProvider): if results or not manualSearch: return results - data = self.getURL(self.search_url, post_data=self._make_post_data_JSON(show=episode.show, episode=episode)) + data = self.getRSSFeed(self.search_url, post_data=self._make_post_data_JSON(show=episode.show, episode=episode)) if not data: logger.log(u"No data returned from " + self.search_url, logger.ERROR) return [] - parsedJSON = helpers.parse_json(data) - - if parsedJSON is None: - logger.log(u"Error trying to load " + self.name + " JSON data", logger.ERROR) - return [] - - if self._checkAuthFromData(parsedJSON): + if self._checkAuthFromData(data): results = [] - if parsedJSON and 'data' in parsedJSON: - items = parsedJSON['data'] - else: + items = data.entries + if not len(items) > 0: logger.log(u"Resulting JSON from " + self.name + " isn't correct, not parsing it", logger.ERROR) - items = [] + return [] for item in items: @@ -144,8 +136,8 @@ class HDBitsProvider(generic.TorrentProvider): def _get_title_and_url(self, item): - title = item['name'] - url = self.download_url + urllib.urlencode({'id': item['id'], 'passkey': sickbeard.HDBITS_PASSKEY}) + title = item.title + url = self.download_url + urllib.urlencode({'id': item.id, 'passkey': sickbeard.HDBITS_PASSKEY}) return (title, url) @@ -199,18 +191,9 @@ class HDBitsCache(tvcache.TVCache): logger.log(u"Clearing " + self.provider.name + " cache and updating with new information") self._clearCache() - parsedJSON = helpers.parse_json(data) - - if parsedJSON is None: - logger.log(u"Error trying to load " + self.provider.name + " JSON feed", logger.ERROR) - return [] - - if self._checkAuth(parsedJSON): - if parsedJSON and 'data' in parsedJSON: - items = parsedJSON['data'] - else: - logger.log(u"Resulting JSON from " + self.provider.name + " isn't correct, not parsing it", - logger.ERROR) + if self._checkAuth(data): + items = data.entries + if not len(items) > 0: return [] cl = [] @@ -248,5 +231,4 @@ class HDBitsCache(tvcache.TVCache): def _checkAuth(self, data): return self.provider._checkAuthFromData(data) - provider = HDBitsProvider() diff --git a/sickbeard/providers/newzbin.py b/sickbeard/providers/newzbin.py index 05c84b81..d93eeace 100644 --- a/sickbeard/providers/newzbin.py +++ b/sickbeard/providers/newzbin.py @@ -81,7 +81,7 @@ class NewzbinProvider(generic.NZBProvider): return sickbeard.NEWZBIN def getQuality(self, item): - attributes = item.getElementsByTagName('report:attributes')[0] + attributes = item.report[0] attr_dict = {} for attribute in attributes.getElementsByTagName('report:attribute'): diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index f627d72a..e2c13ecc 100644 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -178,7 +178,7 @@ class NewznabProvider(generic.NZBProvider): raise AuthException( "Your account isn't allowed to use the API on " + self.name + ", contact the administrator") else: - logger.log(u"Unknown error given from " + self.name + ": " + data.description, + logger.log(u"Unknown error given from " + self.name + ": " + data.feed.title, logger.ERROR) return False @@ -224,7 +224,7 @@ class NewznabProvider(generic.NZBProvider): results.append(curItem) else: logger.log( - u"The XML returned from the " + self.name + " RSS feed is incomplete, this result is unusable", + u"The data returned from the " + self.name + " RSS feed is incomplete, this result is unusable", logger.DEBUG) return results @@ -244,22 +244,11 @@ class NewznabProvider(generic.NZBProvider): (title, url) = self._get_title_and_url(item) - description_node = item.find('pubDate') - description_text = helpers.get_xml_text(description_node) - - try: - # we could probably do dateStr = descriptionStr but we want date in this format - date_text = re.search('(\w{3}, \d{1,2} \w{3} \d{4} \d\d:\d\d:\d\d) [\+\-]\d{4}', - description_text).group(1) - except: - date_text = None - - if not date_text: + if not item.published_parsed: logger.log(u"Unable to figure out the date for entry " + title + ", skipping it") continue else: - - result_date = email.utils.parsedate(date_text) + result_date = item.published_parsed if result_date: result_date = datetime.datetime(*result_date[0:6]) @@ -296,10 +285,6 @@ class NewznabCache(tvcache.TVCache): logger.log(u"No data returned from " + rss_url, logger.ERROR) return None - # hack this in until it's fixed server side - #if data and not data.startswith('' + data - return data def _checkAuth(self, data): diff --git a/sickbeard/providers/nzbsrus.py b/sickbeard/providers/nzbsrus.py index 48a7d1ad..f8b56508 100644 --- a/sickbeard/providers/nzbsrus.py +++ b/sickbeard/providers/nzbsrus.py @@ -28,7 +28,6 @@ except ImportError: from sickbeard import exceptions, logger from sickbeard import tvcache, show_name_helpers - class NZBsRUSProvider(generic.NZBProvider): def __init__(self): generic.NZBProvider.__init__(self, "NZBs'R'US") @@ -68,29 +67,26 @@ class NZBsRUSProvider(generic.NZBProvider): searchURL = self.url + 'api.php?' + urllib.urlencode(params) logger.log(u"NZBS'R'US search url: " + searchURL, logger.DEBUG) - data = self.getURL(searchURL) + data = self.getRSSFeed(searchURL) if not data: return [] - if not data.startswith(' 0: logger.log(u"Error trying to parse NZBS'R'US XML data.", logger.ERROR) logger.log(u"RSS data: " + data, logger.DEBUG) return [] - return root.findall('./results/result') - def _get_title_and_url(self, element): - if element.find('title'): # RSS feed - title = element.find('title').text - url = element.find('link').text.replace('&', '&') + return items + + def _get_title_and_url(self, item): + if item.title: # RSS feed + title = item.title + url = item.link else: # API item - title = element.find('name').text - nzbID = element.find('id').text - key = element.find('key').text + title = item.name + nzbID = item.id + key = item.key url = self.url + 'nzbdownload_rss.php' + '/' + \ nzbID + '/' + sickbeard.NZBSRUS_UID + '/' + key + '/' return (title, url) @@ -111,7 +107,7 @@ class NZBsRUSCache(tvcache.TVCache): url += urllib.urlencode(urlArgs) logger.log(u"NZBs'R'US cache update URL: " + url, logger.DEBUG) - data = self.provider.getURL(url) + data = self.provider.getRSSFeed(url) return data def _checkAuth(self, data): diff --git a/sickbeard/providers/omgwtfnzbs.py b/sickbeard/providers/omgwtfnzbs.py index cbf0ea77..15c881cc 100644 --- a/sickbeard/providers/omgwtfnzbs.py +++ b/sickbeard/providers/omgwtfnzbs.py @@ -64,7 +64,6 @@ class OmgwtfnzbsProvider(generic.NZBProvider): if is_XML: # provider doesn't return xml on error return True - else: parsedJSON = parsed_data @@ -112,23 +111,17 @@ class OmgwtfnzbsProvider(generic.NZBProvider): search_url = 'https://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params) logger.log(u"Search url: " + search_url, logger.DEBUG) - data = self.getURL(search_url) + data = self.getURL(search_url, json=True) if not data: logger.log(u"No data returned from " + search_url, logger.ERROR) return [] - parsedJSON = helpers.parse_json(data) - - if parsedJSON is None: - logger.log(u"Error trying to load " + self.name + " JSON data", logger.ERROR) - return [] - - if self._checkAuthFromData(parsedJSON, is_XML=False): + if self._checkAuthFromData(data, is_XML=False): results = [] - for item in parsedJSON: + for item in data: if 'release' in item and 'getnzb' in item: results.append(item) @@ -171,7 +164,7 @@ class OmgwtfnzbsCache(tvcache.TVCache): logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) - data = self.provider.getURL(rss_url) + data = self.provider.getRSSFeed(rss_url) if not data: logger.log(u"No data returned from " + rss_url, logger.ERROR) diff --git a/sickbeard/providers/tvtorrents.py b/sickbeard/providers/tvtorrents.py index e0885a67..9ae88fec 100644 --- a/sickbeard/providers/tvtorrents.py +++ b/sickbeard/providers/tvtorrents.py @@ -59,7 +59,7 @@ class TvTorrentsProvider(generic.TorrentProvider): if data is None: return self._checkAuth() - description_text = data.description + description_text = data.feed.title if "User can't be found" in description_text or "Invalid Hash" in description_text: logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(description_text),