More RSS cache fixes

sickbeard/helpers.py

@@ -167,7 +167,7 @@ def sanitizeFileName(name):
     return name
 
 
-def getURL(url, post_data=None, headers=None, params=None, timeout=None):
+def getURL(url, post_data=None, headers=None, params=None, json=False):
     """
 Returns a byte-string retrieved from the url provider.
 """
@@ -206,6 +206,9 @@ Returns a byte-string retrieved from the url provider.
         logger.log(u"Connection timed out " + str(e.message) + " while loading URL " + url, logger.WARNING)
         return None
 
+    if json:
+        return resp.json() if resp.ok else None
+
     return resp.content if resp.ok else None
 
 

sickbeard/providers/ezrss.py

@@ -52,8 +52,7 @@ class EZRSSProvider(generic.TorrentProvider):
 
     def getQuality(self, item):
 
-        filename = helpers.get_xml_text(
+        filename = item.filename
-            item.find('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName'))
         quality = Quality.nameQuality(filename)
 
         return quality
@@ -121,7 +120,6 @@ class EZRSSProvider(generic.TorrentProvider):
         items = data.entries
 
         results = []
-
         for curItem in items:
 
             (title, url) = self._get_title_and_url(curItem)
@@ -139,9 +137,7 @@ class EZRSSProvider(generic.TorrentProvider):
     def _get_title_and_url(self, item):
         (title, url) = generic.TorrentProvider._get_title_and_url(self, item)
 
-        filename = helpers.get_xml_text(
+        filename = item.filename
-            item.find('{http://xmlns.ezrss.it/0.1/}torrent/{http://xmlns.ezrss.it/0.1/}fileName'))
-
         if filename:
             new_title = self._extract_name_from_filename(filename)
             if new_title:

sickbeard/providers/generic.py

@@ -103,7 +103,7 @@ class GenericProvider:
 
         return result
 
-    def getURL(self, url, post_data=None, headers=None):
+    def getURL(self, url, post_data=None, headers=None, json=False):
         """
         By default this is just a simple urlopen call but this method should be overridden
         for providers with special URL requirements (like cookies)
@@ -112,7 +112,7 @@ class GenericProvider:
         if not headers:
             headers = []
 
-        data = helpers.getURL(url, post_data, headers)
+        data = helpers.getURL(url, post_data, headers, json=json)
 
         if not data:
             logger.log(u"Error loading " + self.name + " URL: " + url, logger.ERROR)
@@ -244,7 +244,7 @@ class GenericProvider:
         self._checkAuth()
 
         # XEM episode scene numbering
-        sceneEpisode = copy.copy(episode)
+        sceneEpisode = copy.deepcopy(episode)
         sceneEpisode.convertToSceneNumbering()
 
         logger.log(u'Searching "%s" for "%s" as "%s"'
@@ -439,16 +439,4 @@ class TorrentProvider(GenericProvider):
     def __init__(self, name):
         GenericProvider.__init__(self, name)
 
-        self.providerType = GenericProvider.TORRENT
+        self.providerType = GenericProvider.TORRENT
-
-#        self.option = {SEED_POLICY_TIME : '',
-#                       SEED_POLICY_RATIO: '',
-#                       'PROCESS_METHOD': ''
-#                       }
-
-#    def get_provider_options(self):
-#        pass
-#    
-#    def set_provider_options(self):
-#        self.option[SEED_POLICY_TIME] + '|' + self.option[SEED_POLICY_RATIO] + '|' + self.option['PROCESS_METHOD']
-        

sickbeard/providers/hdbits.py

@@ -53,17 +53,16 @@ class HDBitsProvider(generic.TorrentProvider):
 
         return True
 
-    def _checkAuthFromData(self, parsedJSON):
+    def _checkAuthFromData(self, data):
 
-        if parsedJSON is None:
+        if data is None:
             return self._checkAuth()
 
-        if 'status' in parsedJSON and 'message' in parsedJSON:
+        if data.status == 5:
-            if parsedJSON.get('status') == 5:
+            logger.log(u"Incorrect authentication credentials for " + self.name + " : " + data.feed.title,
-                logger.log(u"Incorrect authentication credentials for " + self.name + " : " + parsedJSON['message'],
+                       logger.DEBUG)
-                           logger.DEBUG)
+            raise AuthException(
-                raise AuthException(
+                "Your authentication credentials for " + self.name + " are incorrect, check your config.")
-                    "Your authentication credentials for " + self.name + " are incorrect, check your config.")
 
         return True
 
@@ -81,26 +80,19 @@ class HDBitsProvider(generic.TorrentProvider):
         if results or not manualSearch:
             return results
 
-        data = self.getURL(self.search_url, post_data=self._make_post_data_JSON(show=episode.show, episode=episode))
+        data = self.getRSSFeed(self.search_url, post_data=self._make_post_data_JSON(show=episode.show, episode=episode))
 
         if not data:
             logger.log(u"No data returned from " + self.search_url, logger.ERROR)
             return []
 
-        parsedJSON = helpers.parse_json(data)
+        if self._checkAuthFromData(data):
-
-        if parsedJSON is None:
-            logger.log(u"Error trying to load " + self.name + " JSON data", logger.ERROR)
-            return []
-
-        if self._checkAuthFromData(parsedJSON):
             results = []
 
-            if parsedJSON and 'data' in parsedJSON:
+            items = data.entries
-                items = parsedJSON['data']
+            if not len(items) > 0:
-            else:
                 logger.log(u"Resulting JSON from " + self.name + " isn't correct, not parsing it", logger.ERROR)
-                items = []
+                return []
 
             for item in items:
 
@@ -144,8 +136,8 @@ class HDBitsProvider(generic.TorrentProvider):
 
     def _get_title_and_url(self, item):
 
-        title = item['name']
+        title = item.title
-        url = self.download_url + urllib.urlencode({'id': item['id'], 'passkey': sickbeard.HDBITS_PASSKEY})
+        url = self.download_url + urllib.urlencode({'id': item.id, 'passkey': sickbeard.HDBITS_PASSKEY})
 
         return (title, url)
 
@@ -199,18 +191,9 @@ class HDBitsCache(tvcache.TVCache):
             logger.log(u"Clearing " + self.provider.name + " cache and updating with new information")
             self._clearCache()
 
-            parsedJSON = helpers.parse_json(data)
+            if self._checkAuth(data):
-
+                items = data.entries
-            if parsedJSON is None:
+                if not len(items) > 0:
-                logger.log(u"Error trying to load " + self.provider.name + " JSON feed", logger.ERROR)
-                return []
-
-            if self._checkAuth(parsedJSON):
-                if parsedJSON and 'data' in parsedJSON:
-                    items = parsedJSON['data']
-                else:
-                    logger.log(u"Resulting JSON from " + self.provider.name + " isn't correct, not parsing it",
-                               logger.ERROR)
                     return []
 
                 cl = []
@@ -248,5 +231,4 @@ class HDBitsCache(tvcache.TVCache):
     def _checkAuth(self, data):
         return self.provider._checkAuthFromData(data)
 
-
 provider = HDBitsProvider()

sickbeard/providers/newzbin.py

@@ -81,7 +81,7 @@ class NewzbinProvider(generic.NZBProvider):
         return sickbeard.NEWZBIN
 
     def getQuality(self, item):
-        attributes = item.getElementsByTagName('report:attributes')[0]
+        attributes = item.report[0]
         attr_dict = {}
 
         for attribute in attributes.getElementsByTagName('report:attribute'):

sickbeard/providers/newznab.py

@@ -178,7 +178,7 @@ class NewznabProvider(generic.NZBProvider):
                 raise AuthException(
                     "Your account isn't allowed to use the API on " + self.name + ", contact the administrator")
             else:
-                logger.log(u"Unknown error given from " + self.name + ": " + data.description,
+                logger.log(u"Unknown error given from " + self.name + ": " + data.feed.title,
                            logger.ERROR)
                 return False
 
@@ -224,7 +224,7 @@ class NewznabProvider(generic.NZBProvider):
                     results.append(curItem)
                 else:
                     logger.log(
-                        u"The XML returned from the " + self.name + " RSS feed is incomplete, this result is unusable",
+                        u"The data returned from the " + self.name + " RSS feed is incomplete, this result is unusable",
                         logger.DEBUG)
 
             return results
@@ -244,22 +244,11 @@ class NewznabProvider(generic.NZBProvider):
 
                 (title, url) = self._get_title_and_url(item)
 
-                description_node = item.find('pubDate')
+                if not item.published_parsed:
-                description_text = helpers.get_xml_text(description_node)
-
-                try:
-                    # we could probably do dateStr = descriptionStr but we want date in this format
-                    date_text = re.search('(\w{3}, \d{1,2} \w{3} \d{4} \d\d:\d\d:\d\d) [\+\-]\d{4}',
-                                          description_text).group(1)
-                except:
-                    date_text = None
-
-                if not date_text:
                     logger.log(u"Unable to figure out the date for entry " + title + ", skipping it")
                     continue
                 else:
-
+                    result_date = item.published_parsed
-                    result_date = email.utils.parsedate(date_text)
                     if result_date:
                         result_date = datetime.datetime(*result_date[0:6])
 
@@ -296,10 +285,6 @@ class NewznabCache(tvcache.TVCache):
             logger.log(u"No data returned from " + rss_url, logger.ERROR)
             return None
 
-        # hack this in until it's fixed server side
-        #if data and not data.startswith('<?xml'):
-        #    data = '<?xml version="1.0" encoding="ISO-8859-1" ?>' + data
-
         return data
 
     def _checkAuth(self, data):

sickbeard/providers/nzbsrus.py

@@ -28,7 +28,6 @@ except ImportError:
 from sickbeard import exceptions, logger
 from sickbeard import tvcache, show_name_helpers
 
-
 class NZBsRUSProvider(generic.NZBProvider):
     def __init__(self):
         generic.NZBProvider.__init__(self, "NZBs'R'US")
@@ -68,29 +67,26 @@ class NZBsRUSProvider(generic.NZBProvider):
         searchURL = self.url + 'api.php?' + urllib.urlencode(params)
         logger.log(u"NZBS'R'US search url: " + searchURL, logger.DEBUG)
 
-        data = self.getURL(searchURL)
+        data = self.getRSSFeed(searchURL)
         if not data:
             return []
 
-        if not data.startswith('<?xml'):  # Error will be a single line of text
+        items = data.entries
-            logger.log(u"NZBs'R'US error: " + data, logger.ERROR)
+        if not len(items) > 0:
-            return []
-
-        root = etree.fromstring(data)
-        if root is None:
             logger.log(u"Error trying to parse NZBS'R'US XML data.", logger.ERROR)
             logger.log(u"RSS data: " + data, logger.DEBUG)
             return []
-        return root.findall('./results/result')
 
-    def _get_title_and_url(self, element):
+        return items
-        if element.find('title'):  # RSS feed
+
-            title = element.find('title').text
+    def _get_title_and_url(self, item):
-            url = element.find('link').text.replace('&amp;', '&')
+        if item.title:  # RSS feed
+            title = item.title
+            url = item.link
         else:  # API item
-            title = element.find('name').text
+            title = item.name
-            nzbID = element.find('id').text
+            nzbID = item.id
-            key = element.find('key').text
+            key = item.key
             url = self.url + 'nzbdownload_rss.php' + '/' + \
                   nzbID + '/' + sickbeard.NZBSRUS_UID + '/' + key + '/'
         return (title, url)
@@ -111,7 +107,7 @@ class NZBsRUSCache(tvcache.TVCache):
         url += urllib.urlencode(urlArgs)
         logger.log(u"NZBs'R'US cache update URL: " + url, logger.DEBUG)
 
-        data = self.provider.getURL(url)
+        data = self.provider.getRSSFeed(url)
         return data
 
     def _checkAuth(self, data):

sickbeard/providers/omgwtfnzbs.py

@@ -64,7 +64,6 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
         if is_XML:
             # provider doesn't return xml on error
             return True
-
         else:
             parsedJSON = parsed_data
 
@@ -112,23 +111,17 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
         search_url = 'https://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params)
         logger.log(u"Search url: " + search_url, logger.DEBUG)
 
-        data = self.getURL(search_url)
+        data = self.getURL(search_url, json=True)
 
         if not data:
             logger.log(u"No data returned from " + search_url, logger.ERROR)
             return []
 
-        parsedJSON = helpers.parse_json(data)
+        if self._checkAuthFromData(data, is_XML=False):
-
-        if parsedJSON is None:
-            logger.log(u"Error trying to load " + self.name + " JSON data", logger.ERROR)
-            return []
-
-        if self._checkAuthFromData(parsedJSON, is_XML=False):
 
             results = []
 
-            for item in parsedJSON:
+            for item in data:
                 if 'release' in item and 'getnzb' in item:
                     results.append(item)
 
@@ -171,7 +164,7 @@ class OmgwtfnzbsCache(tvcache.TVCache):
 
         logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
 
-        data = self.provider.getURL(rss_url)
+        data = self.provider.getRSSFeed(rss_url)
 
         if not data:
             logger.log(u"No data returned from " + rss_url, logger.ERROR)

sickbeard/providers/tvtorrents.py

@@ -59,7 +59,7 @@ class TvTorrentsProvider(generic.TorrentProvider):
         if data is None:
             return self._checkAuth()
 
-        description_text = data.description
+        description_text = data.feed.title
 
         if "User can't be found" in description_text or "Invalid Hash" in description_text:
             logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(description_text),