# # This file is part of SickGear. # # SickGear is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # SickGear is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . from __future__ import division # from collections import OrderedDict from io import BytesIO from math import ceil import datetime import re import time import sickgear from . import generic from .. import classes, db, helpers, logger, tvcache from ..classes import SearchResult from ..common import NeededQualities, Quality from ..helpers import remove_non_release_groups from ..indexers.indexer_config import * from ..network_timezones import SG_TIMEZONE from ..sgdatetime import SGDatetime from ..search import get_aired_in_season, get_wanted_qualities from ..show_name_helpers import get_show_names from ..scene_exceptions import has_season_exceptions from ..tv import TVEpisode, TVShow from lib.dateutil import parser from lib.sg_helpers import md5_for_text, try_int from exceptions_helper import AuthException, ex, MultipleShowObjectsException from lxml_etree import etree from six import iteritems, itervalues, string_types from _23 import urlencode # noinspection PyUnreachableCode if False: from typing import Any, AnyStr, Dict, List, Tuple, Union class NewznabConstants(object): SEARCH_TEXT = -100 SEARCH_SEASON = -101 SEARCH_EPISODE = -102 CAT_SD = -200 CAT_HD = -201 CAT_UHD = -202 CAT_HEVC = -203 CAT_ANIME = -204 CAT_SPORT = -205 CAT_WEBDL = -206 catSearchStrings = {r'^Anime$': CAT_ANIME, r'^Sport$': CAT_SPORT, r'^SD$': CAT_SD, r'^BoxSD$': CAT_SD, r'^HD$': CAT_HD, r'^BoxHD$': CAT_HD, r'^UHD$': CAT_UHD, r'^4K$': CAT_UHD, # r'^HEVC$': CAT_HEVC, r'^WEB.?DL$': CAT_WEBDL} providerToIndexerMapping = {'tvdbid': TVINFO_TVDB, 'rageid': TVINFO_TVRAGE, 'tvmazeid': TVINFO_TVMAZE, 'imdbid': TVINFO_IMDB, 'tmdbid': TVINFO_TMDB, 'traktid': TVINFO_TRAKT} indexer_priority_list = [TVINFO_TVDB, TVINFO_TVMAZE, TVINFO_TVRAGE, TVINFO_TRAKT, TVINFO_TMDB, TVINFO_TMDB] searchTypes = {'rid': TVINFO_TVRAGE, 'tvdbid': TVINFO_TVDB, 'tvmazeid': TVINFO_TVMAZE, 'imdbid': TVINFO_IMDB, 'tmdbid': TVINFO_TMDB, 'traktid': TVINFO_TRAKT, 'q': SEARCH_TEXT, 'season': SEARCH_SEASON, 'ep': SEARCH_EPISODE} SERVER_DEFAULT = 0 SERVER_SPOTWEB = 1 SERVER_HYDRA1 = 2 SERVER_HYDRA2 = 3 server_types = {SERVER_DEFAULT: 'newznab', SERVER_SPOTWEB: 'spotweb', SERVER_HYDRA1: 'NZBHydra', SERVER_HYDRA2: 'NZBHydra 2'} # md5_for_text values custom_server_types = {SERVER_SPOTWEB: ('720db6d19d02339e9c320dd788291136',)} def __init__(self): pass class NewznabProvider(generic.NZBProvider): def __init__(self, name, # type: AnyStr url, # type: AnyStr key='', # type: AnyStr cat_ids=None, # type: Union[List, AnyStr] search_mode=None, # type: AnyStr search_fallback=False, # type: bool enable_recentsearch=False, # type: bool enable_backlog=False, # type: bool enable_scheduled_backlog=False, # type: bool server_type=None # type: Union[int, None] ): """ :param name: provider name :param url: url :param key: :param cat_ids: :param search_mode: :param search_fallback: :param enable_recentsearch: :param enable_backlog: :param enable_scheduled_backlog: :param server_type: """ generic.NZBProvider.__init__(self, name, True, False) self.url = url self.key = key self.server_type = try_int(server_type, None) or NewznabConstants.SERVER_DEFAULT self._exclude = set() self.cat_ids = cat_ids or '' self.search_mode = search_mode or 'eponly' self.search_fallback = bool(try_int(search_fallback)) self.enable_recentsearch = bool(try_int(enable_recentsearch)) self.enable_backlog = bool(try_int(enable_backlog)) self.enable_scheduled_backlog = bool(try_int(enable_scheduled_backlog, 1)) self.needs_auth = '0' != self.key.strip() # '0' in the key setting indicates that api_key is not needed self.default = False self._caps = {} self._caps_cats = {} self._caps_all_cats = [] self._caps_need_apikey = {'need': False, 'date': datetime.date.fromordinal(1)} self._limits = 100 self._last_recent_search = None self._caps_last_updated = datetime.datetime.fromordinal(1) self.cache = NewznabCache(self) # filters # deprecated; kept here as bookmark for new haspretime:0|1 + nuked:0|1 can be used here instead # if super(NewznabProvider, self).get_id() in ('nzbs_org',): # self.filter = [] # if 'nzbs_org' == super(NewznabProvider, self).get_id(): # self.may_filter = OrderedDict([ # ('so', ('scene only', False)), ('snn', ('scene not nuked', False))]) @property def cat_ids(self): return self._cat_ids @cat_ids.setter def cat_ids(self, cats): self._cat_ids = self.clean_newznab_categories(cats) @property def caps(self): self.check_cap_update() return self._caps @property def cats(self): self.check_cap_update() return self._caps_cats @property def excludes(self): self.check_cap_update() return self._exclude @property def all_cats(self): self.check_cap_update() return self._caps_all_cats @property def limits(self): self.check_cap_update() return self._limits @property def last_recent_search(self): if not self._last_recent_search: try: my_db = db.DBConnection('cache.db') res = my_db.select('SELECT' + ' "datetime" FROM "lastrecentsearch" WHERE "name"=?', [self.get_id()]) if res: self._last_recent_search = SGDatetime.from_timestamp(int(res[0]['datetime'])) except (BaseException, Exception): pass return self._last_recent_search @last_recent_search.setter def last_recent_search(self, value): try: my_db = db.DBConnection('cache.db') if isinstance(value, datetime.datetime): save_value = SGDatetime.timestamp_near(value) else: save_value = SGDatetime.timestamp_far(value, default=0) my_db.action('INSERT OR REPLACE INTO "lastrecentsearch" (name, datetime) VALUES (?,?)', [self.get_id(), save_value]) except (BaseException, Exception): pass self._last_recent_search = value def image_name(self): """ :rtype: AnyStr """ if self.server_type not in (NewznabConstants.SERVER_DEFAULT, NewznabConstants.SERVER_SPOTWEB): return 'warning16.png' return generic.GenericProvider.image_name( self, ('newznab', 'spotweb')[self.server_type == NewznabConstants.SERVER_SPOTWEB]) def check_cap_update(self): if self.enabled and \ (not self._caps or (datetime.datetime.now() - self._caps_last_updated) >= datetime.timedelta(days=1)): self.get_caps() def _get_caps_data(self): xml_caps = None if self.enabled: if datetime.date.today() - self._caps_need_apikey['date'] > datetime.timedelta(days=30) or \ not self._caps_need_apikey['need']: self._caps_need_apikey['need'] = False data = self.get_url('%s/api?t=caps' % self.url) if data: xml_caps = helpers.parse_xml(data) if None is xml_caps or not hasattr(xml_caps, 'tag') or 'error' == xml_caps.tag or 'caps' != xml_caps.tag: api_key = self.maybe_apikey() if isinstance(api_key, string_types) and api_key not in ('0', ''): data = self.get_url('%s/api?t=caps&apikey=%s' % (self.url, api_key)) if data: xml_caps = helpers.parse_xml(data) if None is not xml_caps and 'caps' == getattr(xml_caps, 'tag', ''): self._caps_need_apikey = {'need': True, 'date': datetime.date.today()} return xml_caps def _check_excludes(self, cats # type: Union[Dict, List] ): if isinstance(cats, dict): c = [] for v in itervalues(cats): c.extend(v) self._exclude = set(c) else: self._exclude = set([v for v in cats]) def get_caps(self): caps = {} cats = {} all_cats = [] xml_caps = self._get_caps_data() old_api = False if None is not xml_caps: server_node = xml_caps.find('.//server') if None is not server_node: if NewznabConstants.server_types.get(NewznabConstants.SERVER_SPOTWEB) in \ (server_node.get('type', '') or server_node.get('title', '')).lower() \ or (md5_for_text(server_node.get('type', '')) in NewznabConstants.custom_server_types.get(NewznabConstants.SERVER_SPOTWEB)): self.server_type = NewznabConstants.SERVER_SPOTWEB elif 'nzbhydra 2' in server_node.get('title', '').lower() or \ 'nzbhydra2' in server_node.get('url', '').lower(): self.server_type = NewznabConstants.SERVER_HYDRA2 elif 'nzbhydra' == server_node.get('title', '').lower().strip() or \ server_node.get('url', '').lower().strip().endswith('nzbhydra'): self.server_type = NewznabConstants.SERVER_HYDRA1 else: self.server_type = NewznabConstants.SERVER_DEFAULT tv_search = xml_caps.find('.//tv-search') if None is not tv_search: for c in [i for i in tv_search.get('supportedParams', '').split(',')]: k = NewznabConstants.searchTypes.get(c) if k: caps[k] = c # for very old newznab, just add text search as only option if not tv_search.get('supportedParams'): caps[NewznabConstants.searchTypes['q']] = 'q' old_api = True limit = xml_caps.find('.//limits') if None is not limit: lim = try_int(limit.get('max'), 100) self._limits = (100, lim)[100 <= lim] try: for category in xml_caps.iter('category'): if 'TV' == category.get('name'): for subcat in category.findall('subcat'): try: cat_name = subcat.attrib['name'] cat_id = subcat.attrib['id'] all_cats.append({'id': cat_id, 'name': cat_name}) for s, v in iteritems(NewznabConstants.catSearchStrings): if None is not re.search(s, cat_name, re.IGNORECASE): cats.setdefault(v, []).append(cat_id) except (BaseException, Exception): continue elif category.get('name', '').upper() in ['XXX', 'OTHER', 'MISC']: for subcat in category.findall('subcat'): try: if None is not re.search(r'^Anime$', subcat.attrib['name'], re.IGNORECASE): cats.setdefault(NewznabConstants.CAT_ANIME, []).append(subcat.attrib['id']) break except (BaseException, Exception): continue except (BaseException, Exception): logger.debug('Error parsing result for [%s]' % self.name) if not caps and self._caps and not all_cats and self._caps_all_cats and not cats and self._caps_cats: self._check_excludes(cats) return if self.enabled: self._caps_last_updated = datetime.datetime.now() if not old_api and (self.url and 'nzbplanet.net' not in self.url.lower()): if not caps: caps[TVINFO_TVDB] = 'tvdbid' if NewznabConstants.SEARCH_SEASON not in caps or not caps.get(NewznabConstants.SEARCH_SEASON): caps[NewznabConstants.SEARCH_SEASON] = 'season' if NewznabConstants.SEARCH_EPISODE not in caps or not caps.get(NewznabConstants.SEARCH_EPISODE): caps[NewznabConstants.SEARCH_TEXT] = 'ep' if TVINFO_TVRAGE not in caps or not caps.get(TVINFO_TVRAGE): caps[TVINFO_TVRAGE] = 'rid' if NewznabConstants.SEARCH_TEXT not in caps or not caps.get(NewznabConstants.SEARCH_TEXT): caps[NewznabConstants.SEARCH_TEXT] = 'q' if NewznabConstants.CAT_HD not in cats or not cats.get(NewznabConstants.CAT_HD): cats[NewznabConstants.CAT_HD] = ['5040'] # (['5040'], ['5040', '5090'])['nzbs_org' == self.get_id()] if NewznabConstants.CAT_SD not in cats or not cats.get(NewznabConstants.CAT_SD): cats[NewznabConstants.CAT_SD] = ['5030'] # (['5030'], ['5030', '5070'])['nzbs_org' == self.get_id()] if NewznabConstants.CAT_ANIME not in cats or not cats.get(NewznabConstants.CAT_ANIME): cats[NewznabConstants.CAT_ANIME] = ['5070'] # (['5070'], ['6070', '7040'])['nzbs_org' == self.get_id()] if NewznabConstants.CAT_SPORT not in cats or not cats.get(NewznabConstants.CAT_SPORT): cats[NewznabConstants.CAT_SPORT] = ['5060'] self._check_excludes(cats) self._caps = caps self._caps_cats = cats self._caps_all_cats = all_cats def clean_newznab_categories(self, cats # type: Union[List, AnyStr] ): # type: (...) -> Union[List, AnyStr] """ Removes automatically mapped categories from the list :param cats: :return: """ if isinstance(cats, list): return [x for x in cats if x['id'] not in self.excludes] return ','.join(set(cats.split(',')) - self.excludes) def _check_auth(self, is_required=None): if self.should_skip(): return False return super(NewznabProvider, self)._check_auth(is_required) def _check_auth_from_data(self, data, # type: etree.Element url # type: AnyStr ): # type: (...) -> bool """ :param data: :param url: :return: """ if None is data or not hasattr(data, 'tag'): return False if 'error' == data.tag: code = try_int(data.get('code')) description = data.get('description') or '' msg = ('', ': %s' % description)[bool(description)] if code not in (100, 101, 102, 401, 429, 500, 910): logger.warning('Unknown error given from %s%s' % (self.name, msg)) else: if 100 == code: raise AuthException('API key for %s is incorrect, check your config' % self.name) elif 101 == code: raise AuthException('Account suspended on %s, contact the admin' % self.name) elif 401 == code: logger.warning('Error code 401 (Unauthorized) from provider%s' % msg) raise AuthException('Account disabled on %s (code 401), contact the admin%s' % (self.name, msg)) elif code in (102, 429, 500): try: retry_time, unit = re.findall(r'(?i)(?:Try again|Retry) in (\d+)\W+([a-z]+)', description)[0] except IndexError: retry_time, unit = None, None handle_fail = True if 102 == code: handle_fail = ('limit' in description.lower()) or (retry_time and unit) if not handle_fail: raise AuthException( 'Your account isn\'t allowed to use the %s API, contact the admin%s' % (self.name, ('', '. Provider message: %s' % description)[bool(description)])) if handle_fail: self.tmr_limit_update(retry_time, unit, description) self.log_failure_url(url) elif 910 == code: logger.warning('%s %s, please check with provider' % (self.name, ('currently has their API disabled', description)[bool(description)])) return False self.tmr_limit_count = 0 return True def config_str(self): # type: (...) -> AnyStr return '%s|%s|%s|%s|%i|%s|%i|%i|%i|%i|%i' \ % (self.name or '', self.url or '', self.maybe_apikey() or '', self.cat_ids or '', self.enabled, self.search_mode or '', self.search_fallback, getattr(self, 'enable_recentsearch', False), getattr(self, 'enable_backlog', False), getattr(self, 'enable_scheduled_backlog', False), self.server_type) def _season_strings(self, ep_obj # type: TVEpisode ): # type: (...) -> List[Dict[AnyStr, List[AnyStr]]] """ :param ep_obj: episode object :return: """ search_params = [] base_params = {} # season ep_detail = None if ep_obj.show_obj.air_by_date or ep_obj.show_obj.is_sports: airdate = str(ep_obj.airdate).split('-')[0] base_params['season'] = airdate base_params['q'] = airdate if ep_obj.show_obj.air_by_date: ep_detail = '+"%s"' % airdate elif ep_obj.show_obj.is_anime: base_params['season'] = '%d' % ep_obj.scene_absolute_number else: base_params['season'] = str((ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show_obj.is_scene)]) ep_detail = 'S%02d' % try_int(base_params['season'], 1) # id search params = base_params.copy() use_id = False if not has_season_exceptions(ep_obj.show_obj.tvid, ep_obj.show_obj.prodid, ep_obj.season): for i in sickgear.TVInfoAPI().all_sources: if i in ep_obj.show_obj.ids and 0 < ep_obj.show_obj.ids[i]['id'] and i in self.caps: params[self.caps[i]] = ep_obj.show_obj.ids[i]['id'] use_id = True use_id and search_params.append(params) spacer = 'nzbgeek.info' in self.url.lower() and ' ' or '.' # query search and exceptions name_exceptions = get_show_names(ep_obj, spacer) for cur_exception in name_exceptions: params = base_params.copy() if 'q' in params: params['q'] = '%s%s%s' % (cur_exception, spacer, params['q']) search_params.append(params) if ep_detail: params = base_params.copy() params['q'] = '%s%s%s' % (cur_exception, spacer, ep_detail) 'season' in params and params.pop('season') 'ep' in params and params.pop('ep') search_params.append(params) return [{'Season': search_params}] def _episode_strings(self, ep_obj # type: TVEpisode ): # type: (...) -> List[Dict[AnyStr, List[AnyStr]]] """ :param ep_obj: episode object :return: """ search_params = [] base_params = {} if not ep_obj: return [base_params] ep_detail = None if ep_obj.show_obj.air_by_date or ep_obj.show_obj.is_sports: airdate = str(ep_obj.airdate).split('-') base_params['season'] = airdate[0] if ep_obj.show_obj.air_by_date: base_params['ep'] = '/'.join(airdate[1:]) ep_detail = '+"%s.%s"' % (base_params['season'], '.'.join(airdate[1:])) elif ep_obj.show_obj.is_anime: base_params['ep'] = '%i' % (try_int(ep_obj.scene_absolute_number) or try_int(ep_obj.scene_episode)) ep_detail = '%02d' % try_int(base_params['ep']) else: base_params['season'], base_params['ep'] = ( (ep_obj.season, ep_obj.episode), (ep_obj.scene_season, ep_obj.scene_episode))[ep_obj.show_obj.is_scene] ep_detail = sickgear.config.naming_ep_type[2] % { 'seasonnumber': try_int(base_params['season'], 1), 'episodenumber': try_int(base_params['ep'], 1)} # id search params = base_params.copy() use_id = False if not has_season_exceptions(ep_obj.show_obj.tvid, ep_obj.show_obj.prodid, ep_obj.season): for i in sickgear.TVInfoAPI().all_sources: if i in ep_obj.show_obj.ids and 0 < ep_obj.show_obj.ids[i]['id'] and i in self.caps: params[self.caps[i]] = ep_obj.show_obj.ids[i]['id'] use_id = True use_id and search_params.append(params) spacer = 'nzbgeek.info' in self.url.lower() and ' ' or '.' # query search and exceptions name_exceptions = get_show_names(ep_obj, spacer) if sickgear.scene_exceptions.has_abs_episodes(ep_obj): search_params.append({'q': '%s%s%s' % (ep_obj.show_obj.name, spacer, base_params['ep'])}) for cur_exception in name_exceptions: params = base_params.copy() params['q'] = cur_exception search_params.append(params) if ep_detail: params = base_params.copy() params['q'] = '%s%s%s' % (cur_exception, spacer, ep_detail) 'season' in params and params.pop('season') 'ep' in params and params.pop('ep') search_params.append(params) return [{'Episode': search_params}] def _title_and_url(self, item # type: etree.Element ): # type: (...) -> Union[Tuple[AnyStr, AnyStr], Tuple[None, None]] """ :param item: :return: :rtype: Tuple[AnyStr, AnyStr] or Tuple[None, None] """ title, url = None, None try: url = str(item.findtext('link')).replace('&', '&') title = ('%s' % item.findtext('title')).strip() title = re.sub(r'\s+', '.', title) # remove indexer specific release name parts r_found = True while r_found: r_found = False for pattern, repl in ((r'(?i)-Scrambled$', ''), (r'(?i)-BUYMORE$', ''), (r'(?i)-Obfuscated$', ''), (r'(?i)-postbot$', ''), (r'(?i)[-.]English$', '')): if re.search(pattern, title): r_found = True title = re.sub(pattern, repl, title) parts = re.findall('(.*(?:(?:h.?|x)26[45]|vp9|av1|hevc|xvid|divx)[^-]*)(.*)', title, re.I)[0] title = '%s-%s' % (parts[0], remove_non_release_groups(parts[1].split('-')[1])) except (BaseException, Exception): pass return title, url def get_size_uid(self, item, # type: etree.Element **kwargs ): # type: (...) -> Tuple[int, Union[AnyStr, None]] """ :param item: :param kwargs: :return: """ size = -1 uid = None if 'name_space' in kwargs and 'newznab' in kwargs['name_space']: size, uid = self._parse_size_uid(item, kwargs['name_space']) return size, uid def get_show(self, item, # type: etree.Element **kwargs ): # type: (...) -> Union[TVShow, None] """ :param item: :param kwargs: :return: """ show_obj = None if 'name_space' in kwargs and 'newznab' in kwargs['name_space']: tvid_prodid = self.cache.parse_ids(item, kwargs['name_space']) if tvid_prodid: try: show_obj = helpers.find_show_by_id(tvid_prodid, no_mapped_ids=False, check_multishow=True) except MultipleShowObjectsException: return None return show_obj def choose_search_mode(self, ep_obj_list, # type: List[TVEpisode] ep_obj, # type: TVEpisode hits_per_page=100 # type: int ): # type: (...) -> Tuple[bool, NeededQualities, int] """ :param ep_obj_list: episode object list :param ep_obj: episode object :param hits_per_page: :return: season search, needed qualities class, max hits """ searches = [eo for eo in ep_obj_list if (not ep_obj.show_obj.is_scene and eo.season == ep_obj.season) or (ep_obj.show_obj.is_scene and eo.scene_season == ep_obj.scene_season)] needed = NeededQualities() needed.check_needed_types(ep_obj.show_obj) for s in searches: if needed.all_qualities_needed: break if not s.show_obj.is_anime and not s.show_obj.is_sports: if not getattr(s, 'wanted_quality', None): # this should not happen, the creation is missing for the search in this case logger.warning('wanted_quality property was missing for search, creating it') ep_status, ep_quality = Quality.split_composite_status(ep_obj.status) s.wanted_quality = get_wanted_qualities(ep_obj, ep_status, ep_quality, unaired=True) needed.check_needed_qualities(s.wanted_quality) if not hasattr(ep_obj, 'eps_aired_in_season'): # this should not happen, the creation is missing for the search in this case logger.warning('eps_aired_in_season property was missing for search, creating it') ep_count, ep_count_scene = get_aired_in_season(ep_obj.show_obj) ep_obj.eps_aired_in_season = ep_count.get(ep_obj.season, 0) ep_obj.eps_aired_in_scene_season = ep_count_scene.get(ep_obj.scene_season, 0) if ep_obj.show_obj.is_scene \ else ep_obj.eps_aired_in_season per_ep, limit_per_ep = 0, 0 if needed.need_sd and not needed.need_hd: per_ep, limit_per_ep = 10, 25 if needed.need_hd: if not needed.need_sd: per_ep, limit_per_ep = 30, 90 else: per_ep, limit_per_ep = 40, 120 if needed.need_uhd or (needed.need_hd and not self.cats.get(NewznabConstants.CAT_UHD)): per_ep += 4 limit_per_ep += 10 if ep_obj.show_obj.is_anime or ep_obj.show_obj.is_sports or ep_obj.show_obj.air_by_date: rel_per_ep, limit_per_ep = 5, 10 else: rel_per_ep = per_ep rel = max(1, int(ceil((ep_obj.eps_aired_in_scene_season if ep_obj.show_obj.is_scene else ep_obj.eps_aired_in_season * rel_per_ep) / hits_per_page))) rel_limit = max(1, int(ceil((ep_obj.eps_aired_in_scene_season if ep_obj.show_obj.is_scene else ep_obj.eps_aired_in_season * limit_per_ep) / hits_per_page))) season_search = rel < (len(searches) * 100 // hits_per_page) if not season_search: needed = NeededQualities() needed.check_needed_types(ep_obj.show_obj) if not ep_obj.show_obj.is_anime and not ep_obj.show_obj.is_sports: if not getattr(ep_obj, 'wanted_quality', None): ep_status, ep_quality = Quality.split_composite_status(ep_obj.status) ep_obj.wanted_quality = get_wanted_qualities(ep_obj, ep_status, ep_quality, unaired=True) needed.check_needed_qualities(ep_obj.wanted_quality) else: if not ep_obj.show_obj.is_anime and not ep_obj.show_obj.is_sports: for cur_ep_obj in ep_obj_list: if not getattr(cur_ep_obj, 'wanted_quality', None): ep_status, ep_quality = Quality.split_composite_status(cur_ep_obj.status) cur_ep_obj.wanted_quality = get_wanted_qualities(cur_ep_obj, ep_status, ep_quality, unaired=True) needed.check_needed_qualities(cur_ep_obj.wanted_quality) return (season_search, needed, (hits_per_page * 100 // hits_per_page * 2, hits_per_page * int(ceil(rel_limit * 1.5)))[season_search]) def find_search_results(self, show_obj, # type: TVShow ep_obj_list, # type: List[TVEpisode] search_mode, # type: AnyStr manual_search=False, # type: bool try_other_searches=False, # type: bool **kwargs ): # type: (...) -> Union[Dict[TVEpisode, Dict[TVEpisode, SearchResult]], Dict] """ :param show_obj: show object :param ep_obj_list: episode list :param search_mode: search mode :param manual_search: maunal search :param try_other_searches: :param kwargs: :return: """ check = self._check_auth() results = {} # type: Dict[int, List] if (isinstance(check, bool) and not check) or self.should_skip(): return results self.show_obj = show_obj item_list = [] name_space = {} searched_scene_season = s_mode = None search_list = [] for ep_obj in ep_obj_list: # skip if season already searched if (s_mode or 'sponly' == search_mode) and 1 < len(ep_obj_list) \ and searched_scene_season == ep_obj.scene_season: continue # search cache for episode result cache_result = self.cache.search_cache(ep_obj, manual_search) if cache_result: if ep_obj.episode not in results: results[ep_obj.episode] = cache_result else: results[ep_obj.episode].extend(cache_result) # found result, search next episode continue s_mode, needed, max_items = self.choose_search_mode(ep_obj_list, ep_obj, hits_per_page=self.limits) needed.check_needed_types(self.show_obj) if 'sponly' == search_mode: searched_scene_season = ep_obj.scene_season # get season search params search_params = self._season_strings(ep_obj) else: # get single episode search params if s_mode and 1 < len(ep_obj_list): searched_scene_season = ep_obj.scene_season search_params = self._season_strings(ep_obj) else: search_params = self._episode_strings(ep_obj) search_list += [(search_params, needed, max_items)] search_done = [] for (search_params, needed, max_items) in search_list: if self.should_skip(log_warning=False): break for cur_param in search_params: if cur_param in search_done: continue search_done += [cur_param] items, n_space = self._search_provider(cur_param, search_mode=search_mode, epcount=len(ep_obj_list), needed=needed, max_items=max_items, try_all_searches=try_other_searches) item_list += items name_space.update(n_space) if self.should_skip(): break return self.finish_find_search_results( show_obj, ep_obj_list, search_mode, manual_search, results, item_list, name_space=name_space) @staticmethod def _parse_pub_date(item, default=None): # type: (etree.Element, Union[int, None]) -> Union[datetime.datetime, None] """ :param item: :param default: :return: """ parsed_date = default try: p = item.findtext('pubDate') if p: p = parser.parse(p, fuzzy=True) try: p = p.astimezone(SG_TIMEZONE) except (BaseException, Exception): pass if isinstance(p, datetime.datetime): parsed_date = p.replace(tzinfo=None) except (BaseException, Exception): pass return parsed_date @staticmethod def _parse_size_uid(item, # type: etree.Element ns, # type: Dict default=-1 # type: int ): # type: (...) -> Tuple[int, Union[AnyStr, None]] """ :param item: :param ns: :param default: :return: """ parsed_size = default uid = None try: if ns and 'newznab' in ns: for attr in item.findall('%sattr' % ns['newznab']): if 'size' == attr.get('name', ''): parsed_size = try_int(attr.get('value'), -1) elif 'guid' == attr.get('name', ''): uid = attr.get('value') except (BaseException, Exception): pass return parsed_size, uid def _search_provider(self, search_params, # type: Dict[AnyStr, List[Dict[AnyStr, List[AnyStr]]]] needed=NeededQualities(need_all=True), # type: NeededQualities max_items=400, # type: int try_all_searches=False, # type: bool **kwargs ): # type: (...) -> Tuple[List, Dict] """ :param search_params: :param needed: :param max_items: :param try_all_searches: :param kwargs: :return: """ results, n_spaces = [], {} if self.should_skip(): return results, n_spaces api_key = self._check_auth() if isinstance(api_key, bool) and not api_key: return results, n_spaces base_params = {'t': 'tvsearch', 'maxage': sickgear.USENET_RETENTION or 0, 'limit': self.limits, 'attrs': ','.join([k for k, v in iteritems(NewznabConstants.providerToIndexerMapping) if v in self.caps]), 'offset': 0} base_params_rss = {'num': self.limits, 'dl': '1'} rss_fallback = False if isinstance(api_key, string_types) and api_key not in ('0', ''): base_params['apikey'] = api_key base_params_rss['r'] = api_key args = re.findall(r'.*?([ir])\s*=\s*([^\s&;]*)', api_key) if 1 <= len(args): for (cur_key, cur_value) in args: base_params_rss[cur_key] = cur_value if 'r' == cur_key: rss_fallback = True base_params['apikey'] = cur_value if not rss_fallback: logger.warning('Invalid API key config for API to RSS fallback,' ' not found: i=num&r=key or i=&r=key or &r=key') return results, n_spaces for mode in search_params: params = dict(needed=needed, max_items=max_items, try_all_searches=try_all_searches, base_params=base_params, base_params_rss=base_params_rss) results, n_spaces = self._search_core(search_params, **params) if not self.should_skip() and not results and rss_fallback and 'Cache' == mode: results, n_spaces = self._search_core(search_params, use_rss=True, **params) break return results, n_spaces def _search_core(self, search_params, needed=None, max_items=400, try_all_searches=False, use_rss=False, base_params=None, base_params_rss=None): results, n_spaces = [], {} total, cnt, search_url, exit_log = 0, len(results), '', True cat_sport = self.cats.get(NewznabConstants.CAT_SPORT, ['5060']) cat_anime = self.cats.get(NewznabConstants.CAT_ANIME, ['5070']) cat_hd = self.cats.get(NewznabConstants.CAT_HD, ['5040']) cat_sd = self.cats.get(NewznabConstants.CAT_SD, ['5030']) cat_uhd = self.cats.get(NewznabConstants.CAT_UHD) cat_webdl = self.cats.get(NewznabConstants.CAT_WEBDL) for mode in search_params: if self.should_skip(log_warning=False): break for i, params in enumerate(search_params[mode]): # type: int, List[Dict[AnyStr, List[AnyStr]]] if self.should_skip(log_warning=False): break # category ids cat = [] if 'Episode' == mode or 'Season' == mode: if not (any(x in params for x in [v for c, v in iteritems(self.caps) if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON]])): logger.log('Show is missing either an id or search term for search') continue if needed.need_anime: cat.extend(cat_anime) if needed.need_sports: cat.extend(cat_sport) if needed.need_hd: cat.extend(cat_hd) if needed.need_sd: cat.extend(cat_sd) if needed.need_uhd and None is not cat_uhd: cat.extend(cat_uhd) if needed.need_webdl and None is not cat_webdl: cat.extend(cat_webdl) if self.cat_ids or len(cat): base_params['cat'] = ','.join(sorted(set((self.cat_ids.split(',') if self.cat_ids else []) + cat))) base_params_rss['t'] = base_params['cat'] request_params = base_params.copy() # if ('Propers' == mode or 'nzbs_org' == self.get_id()) \ if 'Propers' == mode \ and 'q' in params and not (any(x in params for x in ['season', 'ep'])): request_params['t'] = 'search' request_params.update(params) # deprecated; kept here as bookmark for new haspretime:0|1 + nuked:0|1 can be used here instead # if hasattr(self, 'filter'): # if 'nzbs_org' == self.get_id(): # request_params['rls'] = ((0, 1)['so' in self.filter], 2)['snn' in self.filter] # workaround a strange glitch if sum([ord(i) for i in self.get_id()]) in [383] and 5 == 14 - request_params['maxage']: request_params['maxage'] += 1 offset = 0 batch_count = not 0 first_date = last_date = None # hardcoded to stop after a max of 4 hits (400 items) per query while (offset <= total) and (offset < max_items) and batch_count: cnt = len(results) if 'Cache' == mode and use_rss: search_url = '%srss?%s' % (self.url, urlencode(base_params_rss)) else: search_url = '%sapi?%s' % (self.url, urlencode(request_params)) i and time.sleep(2.1) data = self.get_url(search_url) if self.should_skip() or not data: break # hack this in until it's fixed server side if not data.startswith('%s' % data try: parsed_xml, n_spaces = self.cache.parse_and_get_ns(data) items = parsed_xml.findall('channel/item') except (BaseException, Exception): logger.warning('Error trying to load %s RSS feed' % self.name) break if not self._check_auth_from_data(parsed_xml, search_url): break if 'rss' != parsed_xml.tag: logger.warning('Resulting XML from %s isn\'t RSS, not parsing it' % self.name) break i and time.sleep(2.1) for item in items: title, url = self._title_and_url(item) if title and url: results.append(item) else: logger.debug('The data returned from %s is incomplete, this result is unusable' % self.name) # get total and offset attributes try: if 0 == total: total = (try_int(parsed_xml.find( './/%sresponse' % n_spaces['newznab']).get('total', 0)), 1000)['Cache' == mode] hits = (total // self.limits + int(0 < (total % self.limits))) hits += int(0 == hits) offset = try_int(parsed_xml.find('.//%sresponse' % n_spaces['newznab']).get('offset', 0)) except (AttributeError, KeyError): if not use_rss: break total = len(items) # No items found, prevent from doing another search if 0 == total: break # Cache mode, prevent from doing another search if 'Cache' == mode: if items and len(items): if not first_date: first_date = self._parse_pub_date(items[0]) last_date = self._parse_pub_date(items[-1]) if not first_date or not last_date or not self.last_recent_search or \ last_date <= self.last_recent_search or use_rss: break if offset != request_params['offset']: logger.log('Ask your newznab provider to fix their newznab responses') break request_params['offset'] += request_params['limit'] if total <= request_params['offset']: break # there are more items available than the amount given in one call, grab some more items = total - request_params['offset'] logger.debug(f'{items} more item{helpers.maybe_plural(items)} to fetch from a batch of up to' f' {request_params["limit"]} items.') batch_count = self._log_result(results, mode, cnt, search_url) exit_log = False if 'Cache' == mode and first_date: self.last_recent_search = first_date if exit_log: self._log_search(mode, len(results), search_url) if not try_all_searches and any(x in request_params for x in [ v for c, v in iteritems(self.caps) if c not in [NewznabConstants.SEARCH_EPISODE, NewznabConstants.SEARCH_SEASON, NewznabConstants.SEARCH_TEXT]]) and len(results): break return results, n_spaces def find_propers(self, search_date=None, # type: Union[datetime.date, None] shows=None, # type: Union[List[int, int], None] anime=None, # type: Union[List[int, int], None] **kwargs ): # type: (...) -> List[classes.Proper] """ :param search_date: :param shows: :param anime: :param kwargs: :return: """ cache_results = self.cache.list_propers(search_date) results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show_obj) for x in cache_results] check = self._check_auth() if isinstance(check, bool) and not check: return results index = 0 # alt_search = ('nzbs_org' == self.get_id()) # do_search_alt = False search_terms = [] regex = [] if shows: search_terms += ['.proper.', '.repack.', '.real.'] regex += ['proper|repack', Quality.real_check] proper_check = re.compile(r'(?i)(\b%s\b)' % '|'.join(regex)) if anime: terms = 'v2|v3|v4|v5|v6|v7|v8|v9' search_terms += [terms] regex += [terms] proper_check = re.compile(r'(?i)(%s)' % '|'.join(regex)) urls = [] while index < len(search_terms): if self.should_skip(log_warning=False): break search_params = {'q': search_terms[index], 'maxage': sickgear.BACKLOG_LIMITED_PERIOD + 2} # if alt_search: # # if do_search_alt: # search_params['t'] = 'search' # index += 1 # # do_search_alt = not do_search_alt # # else: # index += 1 index += 1 items, n_space = self._search_provider({'Propers': [search_params]}) for item in items: (title, url) = self._title_and_url(item) # noinspection PyUnboundLocalVariable if not proper_check.search(title) or url in urls: continue urls.append(url) result_date = self._parse_pub_date(item) if not result_date: logger.log(f'Unable to figure out the date for entry {title}, skipping it') continue result_size, result_uid = self._parse_size_uid(item, ns=n_space) if not search_date or search_date < result_date: show_obj = self.get_show(item, name_space=n_space) search_result = classes.Proper(title, url, result_date, self.show_obj, parsed_show_obj=show_obj, size=result_size, puid=result_uid) results.append(search_result) time.sleep(0.5) return results def _log_result(self, results, mode, cnt, url): count = len(results) - cnt if count: self._log_search(mode, count, url) return count def __str__(self): return 'NewznabProvider: %s (%s); server type: %s; enabled searches: %s' % \ (self.name, ('disabled', 'enabled')[self.enabled in (True, 1)], NewznabConstants.server_types.get(self.server_type, 'unknown'), ','.join(en[1] for en in ((getattr(self, 'enable_recentsearch', False), 'recent'), (getattr(self, 'enable_backlog', False), 'backlog'), (getattr(self, 'enable_scheduled_backlog', False), 'scheduled')) if en[0]) or 'None') def __repr__(self): return self.__str__() class NewznabCache(tvcache.TVCache): def __init__(self, provider): tvcache.TVCache.__init__(self, provider, interval=5) # helper method to read the namespaces from xml @staticmethod def parse_and_get_ns(data): # type: (AnyStr) -> Tuple[etree.Element, Dict] """ :param data: :return: """ events = 'start', 'start-ns' root = None ns = {} for event, elem in etree.iterparse(BytesIO(data.encode('utf-8')), events): if 'start-ns' == event: ns[elem[0]] = '{%s}' % elem[1] elif 'start' == event: if None is root: root = elem return root, ns def update_cache(self, needed=NeededQualities(need_all=True), # type: NeededQualities **kwargs ): """ :param needed: needed qualites class :param kwargs: """ if 4489 != sickgear.RECENTSEARCH_INTERVAL or self.should_update(): n_spaces = {} try: check = self.check_auth() if isinstance(check, bool) and not check: items = None else: (items, n_spaces) = self.provider.cache_data(needed=needed) except (BaseException, Exception) as e: logger.error('Error updating Cache: %s' % ex(e)) items = None if items: self.clear_cache() # parse data cl = [] for item in items: ci = self.parse_item(n_spaces, item) if None is not ci: cl.append(ci) if 0 < len(cl): my_db = self.get_db() my_db.mass_action(cl) # set updated as time the attempt to fetch data is self.set_last_update() @staticmethod def parse_ids(item, ns): # type: (etree.Element, Dict) -> Dict[int, int] """ :param item: :param ns: :return: """ ids = {} if 'newznab' in ns: for attr in item.findall('%sattr' % ns['newznab']): if attr.get('name', '') in NewznabConstants.providerToIndexerMapping: v = try_int(attr.get('value')) if 0 < v: ids[NewznabConstants.providerToIndexerMapping[attr.get('name')]] = v return ids # overwrite method with that parses the rageid from the newznab feed def parse_item(self, ns, # type: Dict item # type: etree.Element ): # type: (...) -> Union[List[AnyStr, List[Any]], None] """ :param ns: :param item: :return: """ title, url = self._title_and_url(item) ids = self.parse_ids(item, ns) if title and url: return self.add_cache_entry(title, url, tvid_prodid=ids) logger.debug('Data returned from the %s feed is incomplete, this result is unusable' % self.provider.name)