SickGear/sickgear/scene_exceptions.py

#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict

import io
import os
import re
import sys
import traceback

import sickgear
from exceptions_helper import ex
from json_helper import json_load
from . import db, helpers, logger, name_cache
from .anime import create_anidb_obj
from .classes import OrderedDefaultdict
from .indexers.indexer_config import TVINFO_TVDB
from .scheduler import Job
from .sgdatetime import SGDatetime

import lib.rarfile.rarfile as rarfile

from _23 import list_range
from six import iteritems

# noinspection PyUnreachableCode
if False:
    # noinspection PyUnresolvedReferences
    from typing import AnyStr, List, Tuple, Optional, Union
    from .tv import TVShow

MEMCACHE = {}


class ReleaseMap(Job):
    def __init__(self):
        super(ReleaseMap, self).__init__(self.job_run, thread_lock=True, kwargs={})

        MEMCACHE.setdefault('release_map', {})
        MEMCACHE.setdefault('release_map_season', {})
        MEMCACHE.setdefault('release_map_xem', defaultdict(list))

    def job_run(self):

        # update xem id lists
        self.fetch_xem_ids()

        # update release exceptions
        self.fetch_exceptions()

    def fetch_xem_ids(self):

        for cur_tvid, cur_name in iteritems(sickgear.TVInfoAPI().xem_supported_sources):
            xem_ids = self._get_xem_ids(cur_name, sickgear.TVInfoAPI(cur_tvid).config['xem_origin'])
            if len(xem_ids):
                MEMCACHE['release_map_xem'][cur_tvid] = xem_ids

    @staticmethod
    def _get_xem_ids(infosrc_name, xem_origin):
        # type: (AnyStr, AnyStr) -> List
        """

        :param infosrc_name:
        :param xem_origin:
        """
        result = []

        url = 'https://thexem.info/map/havemap?origin=%s' % xem_origin

        task = 'Fetching show ids with%s xem scene mapping%s for origin'
        logger.log(f'{task % ("", "s")} {infosrc_name}')
        parsed_json = helpers.get_url(url, parse_json=True, timeout=90)
        if not isinstance(parsed_json, dict) or not parsed_json:
            logger.error(f'Failed {task.lower() % ("", "s")} {infosrc_name},'
                         f' Unable to get URL: {url}')
        else:
            if 'success' == parsed_json.get('result', '') and 'data' in parsed_json:
                result = list(set(filter(lambda prodid: 0 < prodid,
                                         map(lambda pid: helpers.try_int(pid), parsed_json['data']))))
                if 0 == len(result):
                    logger.warning(f'Failed {task.lower() % ("", "s")} {infosrc_name},'
                                   f' no data items parsed from URL: {url}')

        logger.log(f'Finished {task.lower() % (f" {len(result)}", helpers.maybe_plural(result))} {infosrc_name}')
        return result

    def _xem_exceptions_fetcher(self):

        result = {}

        xem_list = 'xem_us'
        for cur_show_obj in sickgear.showList:
            if cur_show_obj.is_anime and not cur_show_obj.paused:
                xem_list = 'xem'
                break

        if self._should_refresh(xem_list):
            for cur_tvid in [_i for _i in sickgear.TVInfoAPI().sources
                             if 'xem_origin' in sickgear.TVInfoAPI(_i).config]:
                logger.log(f'Checking for XEM scene exception updates for {sickgear.TVInfoAPI(cur_tvid).name}')

                url = 'https://thexem.info/map/allNames?origin=%s%s&seasonNumbers=1'\
                      % (sickgear.TVInfoAPI(cur_tvid).config['xem_origin'], ('&language=us', '')['xem' == xem_list])

                parsed_json = helpers.get_url(url, parse_json=True, timeout=90)
                if not parsed_json:
                    logger.error(f'Check scene exceptions update failed for {sickgear.TVInfoAPI(cur_tvid).name},'
                                 f' Unable to get URL: {url}')
                    continue

                if 'failure' == parsed_json['result']:
                    continue

                for cur_prodid, cur_names in iteritems(parsed_json['data']):
                    try:
                        result[(cur_tvid, int(cur_prodid))] = cur_names
                    except (BaseException, Exception):
                        continue

            self._set_last_refresh(xem_list)

        return result

    def _anidb_exceptions_fetcher(self):

        result = {}

        if self._should_refresh('anidb'):
            logger.log('Checking for AniDB scene exception updates')
            for cur_show_obj in filter(lambda _s: _s.is_anime and TVINFO_TVDB == _s.tvid, sickgear.showList):
                try:
                    anime = create_anidb_obj(name=cur_show_obj.name, tvdbid=cur_show_obj.prodid, autoCorrectName=True)
                except (BaseException, Exception):
                    continue
                if anime.name and anime.name != cur_show_obj.name:
                    result[(cur_show_obj.tvid, cur_show_obj.prodid)] = [{anime.name: -1}]

            self._set_last_refresh('anidb')

        return result

    def fetch_exceptions(self):
        """
        Looks up release exceptions on GitHub, Xem, and Anidb, parses them into a dict, and inserts them into the
        scene_exceptions table in cache.db. Finally, clears the scene name cache.
        """
        def _merge_exceptions(source, dest):
            for cur_ex in source:
                dest[cur_ex] = source[cur_ex] + ([] if cur_ex not in dest else dest[cur_ex])

        exceptions = self._xem_exceptions_fetcher()  # XEM scene exceptions
        _merge_exceptions(self._anidb_exceptions_fetcher(), exceptions)  # AniDB scene exceptions
        _merge_exceptions(self._github_exceptions_fetcher(), exceptions)  # GitHub stored release exceptions

        exceptions_custom, count_updated_numbers, min_remain_iv = self._custom_exceptions_fetcher()
        _merge_exceptions(exceptions_custom, exceptions)  # Custom exceptions

        is_changed_exceptions = False

        # write all the exceptions we got off the net into the database
        my_db = db.DBConnection()
        cl = []
        for cur_tvid_prodid in exceptions:

            # get a list of the existing exceptions for this ID
            existing_exceptions = [{_x['show_name']: _x['season']} for _x in
                                   my_db.select('SELECT show_name, season'
                                                ' FROM [scene_exceptions]'
                                                ' WHERE indexer = ? AND indexer_id = ?',
                                                list(cur_tvid_prodid))]

            # if this exception isn't already in the DB then add it
            for cur_ex_dict in filter(lambda e: e not in existing_exceptions, exceptions[cur_tvid_prodid]):
                try:
                    exception, season = next(iteritems(cur_ex_dict))
                except (BaseException, Exception):
                    logger.error('release exception error')
                    logger.error(traceback.format_exc())
                    continue

                cl.append(['INSERT INTO [scene_exceptions]'
                           ' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?)',
                           list(cur_tvid_prodid) + [exception, season]])
                is_changed_exceptions = True

        if cl:
            my_db.mass_action(cl)
            name_cache.build_name_cache(update_only_scene=True)

        # since this could invalidate the results of the cache we clear it out after updating
        if is_changed_exceptions:
            logger.log('Updated release exceptions')
        else:
            logger.log('No release exceptions update needed')

        # cleanup
        exceptions.clear()

        return is_changed_exceptions, count_updated_numbers, min_remain_iv

    def _github_exceptions_fetcher(self):
        """
        Looks up the exceptions on GitHub
        """

        # global exception_dict
        result = {}

        # exceptions are stored on GitHub pages
        for cur_tvid in sickgear.TVInfoAPI().sources:
            if self._should_refresh(sickgear.TVInfoAPI(cur_tvid).name):
                url = sickgear.TVInfoAPI(cur_tvid).config.get('scene_url')
                if not url:
                    continue

                logger.log(f'Checking for release exception updates for {sickgear.TVInfoAPI(cur_tvid).name}')

                url_data = helpers.get_url(url)
                if None is url_data:
                    # When None is urlData, trouble connecting to GitHub
                    logger.error(f'Check release exceptions update failed. Unable to get URL: {url}')
                    continue

                else:
                    self._set_last_refresh(sickgear.TVInfoAPI(cur_tvid).name)

                    # each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
                    for cur_line in url_data.splitlines():
                        prodid, sep, aliases = cur_line.partition(':')

                        if not aliases:
                            continue

                        prodid = int(prodid)

                        # regex out the list of shows, taking \' into account
                        alias_list = [{re.sub(r'\\(.)', r'\1', _x): -1} for _x in
                                      re.findall(r"'(.*?)(?<!\\)',?", aliases)]
                        result[(cur_tvid, prodid)] = alias_list
                        del alias_list
                    del url_data

        return result

    def _custom_exceptions_fetcher(self):

        src_id = 'GHSG'
        logger.log(f'Checking to update custom alternatives from {src_id}')

        dirpath = os.path.join(sickgear.CACHE_DIR, 'alts')
        tmppath = os.path.join(dirpath, 'tmp')
        file_rar = os.path.join(tmppath, 'alt.rar')
        file_cache = os.path.join(dirpath, 'alt.json')
        iv = 30 * 60  # min interval to fetch updates
        refresh = self._should_refresh(src_id, iv)
        fetch_data = not os.path.isfile(file_cache) or (not int(os.environ.get('NO_ALT_GET', 0)) and refresh)
        if fetch_data:
            if os.path.exists(tmppath):
                helpers.remove_file(tmppath, tree=True)
            helpers.make_path(tmppath)
            helpers.download_file(r'https://github.com/SickGear/sickgear.altdata/raw/main/alt.rar', file_rar)

            rar_handle = None
            if 'win32' == sys.platform:
                rarfile.UNRAR_TOOL = os.path.join(sickgear.PROG_DIR, 'lib', 'rarfile', 'UnRAR.exe')
            try:
                rar_handle = rarfile.RarFile(file_rar)
                rar_handle.extractall(path=dirpath, pwd='sickgear_alt')
            except(BaseException, Exception) as e:
                logger.error(f'Failed to unpack archive: {file_rar} with error: {ex(e)}')

            if rar_handle:
                rar_handle.close()
                del rar_handle

            helpers.remove_file(tmppath, tree=True)

        if refresh:
            self._set_last_refresh(src_id)

        result = {}
        count_updated_numbers = 0
        if fetch_data or os.path.isfile(file_cache):
            try:
                with io.open(file_cache) as fh:
                    data = json_load(fh)
                result, count_updated_numbers = self._parse_custom_exceptions(data)
            except(BaseException, Exception) as e:
                logger.error(f'Failed to unpack json data: {file_rar} with error: {ex(e)}')

        else:
            logger.debug(f'Unable to fetch custom exceptions, skipped: {file_rar}')

        return result, count_updated_numbers, self._should_refresh(src_id, iv, remaining=True)

    @staticmethod
    def _parse_custom_exceptions(data):
        # type: (AnyStr) -> tuple
        """

        :param data: json text
        """
        # handle data
        from .scene_numbering import find_scene_numbering, set_scene_numbering_helper
        from .tv import TVidProdid

        result = {}
        count_updated_numbers = 0
        for cur_tvid_prodid, cur_season_data in iteritems(data):
            show_obj = sickgear.helpers.find_show_by_id(cur_tvid_prodid, no_mapped_ids=True)
            if not show_obj:
                continue

            used = set()
            for cur_for_season, cur_data in iteritems(cur_season_data):
                cur_for_season = helpers.try_int(cur_for_season, None)
                tvid, prodid = TVidProdid(cur_tvid_prodid).tuple
                if cur_data.get('n'):  # alt names
                    result.setdefault((tvid, prodid), [])
                    result[(tvid, prodid)] += [{_name: cur_for_season} for _name in cur_data.get('n')]

                for cur_update in cur_data.get('se') or []:
                    for cur_for_episode, cur_se_range in iteritems(cur_update):  # scene episode alt numbers
                        cur_for_episode = helpers.try_int(cur_for_episode, None)

                        target_season, episode_range = cur_se_range.split('x')
                        scene_episodes = [int(_x) for _x in episode_range.split('-')
                                          if None is not helpers.try_int(_x, None)]

                        if 2 == len(scene_episodes):
                            desc = scene_episodes[0] > scene_episodes[1]
                            if desc:  # handle a descending range case
                                scene_episodes.reverse()
                            scene_episodes = list_range(*[scene_episodes[0], scene_episodes[1] + 1])
                            if desc:
                                scene_episodes.reverse()

                        target_season = helpers.try_int(target_season, None)
                        for cur_target_episode in scene_episodes:
                            sn = find_scene_numbering(tvid, prodid, cur_for_season, cur_for_episode)
                            used.add((cur_for_season, cur_for_episode, target_season, cur_target_episode))
                            if sn and ((cur_for_season, cur_for_episode) + sn) not in used \
                                    and (cur_for_season, cur_for_episode) not in used:
                                logger.debug(f'Skipped setting "{show_obj.unique_name}"'
                                             f' episode {cur_for_season}x{cur_for_episode}'
                                             f' to target a release {target_season}x{cur_target_episode}'
                                             f' because set to {sn[0]}x{sn[1]}')
                            else:
                                used.add((cur_for_season, cur_for_episode))
                                if not sn or sn != (target_season, cur_target_episode):  # not already set
                                    result = set_scene_numbering_helper(
                                        tvid, prodid, for_season=cur_for_season, for_episode=cur_for_episode,
                                        scene_season=target_season, scene_episode=cur_target_episode)
                                    if result.get('success'):
                                        count_updated_numbers += 1

                            cur_for_episode += 1

        return result, count_updated_numbers

    @staticmethod
    def _should_refresh(name, max_refresh_age_secs=86400, remaining=False):
        # type: (AnyStr, int, bool) -> Union[bool, int]
        """

        :param name: name
        :param max_refresh_age_secs:
        :param remaining: True to return remaining seconds
        :return:
        """
        my_db = db.DBConnection()
        rows = my_db.select('SELECT last_refreshed FROM [scene_exceptions_refresh] WHERE list = ?', [name])
        if rows:
            last_refresh = int(rows[0]['last_refreshed'])
            if remaining:
                time_left = (last_refresh + max_refresh_age_secs - SGDatetime.timestamp_near())
                return (0, time_left)[time_left > 0]
            return SGDatetime.timestamp_near() > last_refresh + max_refresh_age_secs
        return True

    @staticmethod
    def _set_last_refresh(name):
        # type: (AnyStr) -> None
        """

        :param name: name
        :type name: AnyStr
        """
        my_db = db.DBConnection()
        my_db.upsert('scene_exceptions_refresh',
                     {'last_refreshed': SGDatetime.timestamp_near()},
                     {'list': name})

    @staticmethod
    def update_exceptions(show_obj, release_exceptions):
        # type: (TVShow, list) -> None
        """
        Given a show object and a list of alternative names,
        update MEMCACHE['release_map'], the db, and rebuild name_cache.
        """
        logger.log(f'Updating release exceptions for {show_obj.unique_name or show_obj.name}')

        my_db = db.DBConnection()
        my_db.action('DELETE FROM [scene_exceptions]'
                     ' WHERE indexer = ? AND indexer_id = ?',
                     [show_obj.tvid, show_obj.prodid])

        # A change has been made to the scene exception list. Clear the cache, to make this visible
        MEMCACHE['release_map'][(show_obj.tvid, show_obj.prodid)] = defaultdict(list)

        for cur_ex in release_exceptions:

            season, alt_name = cur_ex.split('|', 1)
            try:
                season = int(season)
            except (BaseException, Exception):
                logger.error(f'invalid season for release exception: {show_obj.tvid_prodid} - {season}:{alt_name}')
                continue

            MEMCACHE['release_map'][(show_obj.tvid, show_obj.prodid)][season].append(alt_name)

            my_db.action('INSERT INTO [scene_exceptions]'
                         ' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?)',
                         [show_obj.tvid, show_obj.prodid, alt_name, season])

        sickgear.name_cache.build_name_cache(update_only_scene=True)

    def has_season_exceptions(self, tvid, prodid, season):
        # type: (int, int, int) -> bool

        self.get_alt_names(tvid, prodid, season)
        return (-1 < season) and season in MEMCACHE['release_map'].get((tvid, prodid), {})

    def get_alt_names(self, tvid, prodid, season=-1):
        # type: (int, int, Optional[int]) -> List
        """
        Return a list and update MEMCACHE['release_map'] of alternative show names from db
        for all seasons, or a specific show season.

        :param tvid: show tvid
        :param prodid: show prodid
        :param season: optional season number
        """
        alt_names = MEMCACHE['release_map'].get((tvid, prodid), {}).get(season, [])

        if not alt_names:
            my_db = db.DBConnection()
            exceptions = my_db.select('SELECT show_name'
                                      ' FROM [scene_exceptions]'
                                      ' WHERE indexer = ? AND indexer_id = ?'
                                      ' AND season = ?',
                                      [tvid, prodid, season])
            if exceptions:
                alt_names = list(set([_ex['show_name'] for _ex in exceptions]))

                if (tvid, prodid) not in MEMCACHE['release_map']:
                    MEMCACHE['release_map'][(tvid, prodid)] = {}
                MEMCACHE['release_map'][(tvid, prodid)][season] = alt_names

        if 1 == season:  # if we were looking for season 1 we can add generic names
            alt_names += self.get_alt_names(tvid, prodid)

        return alt_names

    @staticmethod
    def get_show_exceptions(tvid_prodid):
        # type: (AnyStr) -> OrderedDefaultdict
        """
        return a scene exceptions dict for a show

        :param tvid_prodid: a show tvid:prodid
        """
        exceptions_dict = OrderedDefaultdict(list)

        from .tv import TVidProdid

        my_db = db.DBConnection()
        exceptions = my_db.select('SELECT show_name, season'
                                  ' FROM [scene_exceptions]'
                                  ' WHERE indexer = ? AND indexer_id = ?'
                                  ' ORDER BY season DESC, show_name DESC',
                                  TVidProdid(tvid_prodid).list)

        exceptions_seasons = []
        if exceptions:
            for cur_ex in exceptions:
                # order as, s*, and then season desc, show_name also desc (so years in names fall the newest on top)
                if -1 == cur_ex['season']:
                    exceptions_dict[-1].append(cur_ex['show_name'])
                else:
                    exceptions_seasons += [cur_ex]

            for cur_ex in exceptions_seasons:
                exceptions_dict[cur_ex['season']].append(cur_ex['show_name'])

        return exceptions_dict

    @staticmethod
    def get_exception_seasons(tvid, prodid):
        # type: (int, int) -> List[int]
        """
        return a list of season numbers that have alternative names
        :param tvid: show tvid
        :param prodid: show prodid
        """
        exception_seasons = MEMCACHE['release_map_season'].get((tvid, prodid), [])

        if not exception_seasons:
            my_db = db.DBConnection()
            sql_result = my_db.select('SELECT DISTINCT(season) AS season'
                                      ' FROM [scene_exceptions]'
                                      ' WHERE indexer = ? AND indexer_id = ?',
                                      [tvid, prodid])
            if sql_result:
                exception_seasons = list(set([int(_x['season']) for _x in sql_result]))

                if (tvid, prodid) not in MEMCACHE['release_map_season']:
                    MEMCACHE['release_map_season'][(tvid, prodid)] = {}

                MEMCACHE['release_map_season'][(tvid, prodid)] = exception_seasons

        return exception_seasons


def get_scene_exception_by_name(show_name):
    # type: (AnyStr) -> List[None, None, None] or List[int, int, int]
    """

    :param show_name: show name
    """
    return _get_scene_exception_by_name_multiple(show_name)[0]


def _get_scene_exception_by_name_multiple(show_name):
    # type: (AnyStr) -> List[List[None, None, None] or List[int, int, int]]
    """

    :param show_name: show name
    :return:  (tvid, prodid, season) of the exception, None if no exception is present.
    """
    try:
        exception_result = name_cache.sceneNameCache[helpers.full_sanitize_scene_name(show_name)]
    except (BaseException, Exception):
        return [[None, None, None]]

    return [exception_result]


def has_abs_episodes(ep_obj=None, name=None):
    # type: (Optional[sickgear.tv.TVEpisode], Optional[AnyStr]) -> bool
    """

    :param ep_obj: episode object
    :param name: name
    """
    return any((name or ep_obj.show_obj.name or '').lower().startswith(_x.lower()) for _x in [
        'The Eighties', 'The Making of the Mob', 'The Night Of', 'Roots 2016', 'Trepalium'
    ])