SickGear/sickbeard/search_queue.py

# Author: Nic Wolfe <nic@wolfeden.ca>
# URL: http://code.google.com/p/sickbeard/
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import with_statement

import traceback
import threading
import datetime
import re

import sickbeard
from sickbeard import db, logger, common, exceptions, helpers, network_timezones, generic_queue, search, \
    failed_history, history, ui, properFinder
from sickbeard.search import wanted_episodes, get_aired_in_season, set_wanted_aired
from sickbeard.classes import Proper
from sickbeard.indexers.indexer_config import INDEXER_TVDB


search_queue_lock = threading.Lock()

BACKLOG_SEARCH = 10
RECENT_SEARCH = 20
FAILED_SEARCH = 30
MANUAL_SEARCH = 40
PROPER_SEARCH = 50

MANUAL_SEARCH_HISTORY = []
MANUAL_SEARCH_HISTORY_SIZE = 100


class SearchQueue(generic_queue.GenericQueue):
    def __init__(self):
        generic_queue.GenericQueue.__init__(self)
        self.queue_name = 'SEARCHQUEUE'

    def is_in_queue(self, show, segment):
        with self.lock:
            for cur_item in self.queue:
                if isinstance(cur_item, BacklogQueueItem) and cur_item.show == show and cur_item.segment == segment:
                    return True
            return False

    def is_ep_in_queue(self, segment):
        with self.lock:
            for cur_item in self.queue:
                if isinstance(cur_item, (ManualSearchQueueItem, FailedQueueItem)) and cur_item.segment == segment:
                    return True
            return False

    def is_show_in_queue(self, show):
        with self.lock:
            for cur_item in self.queue:
                if isinstance(cur_item, (ManualSearchQueueItem, FailedQueueItem)) and cur_item.show.indexerid == show:
                    return True
            return False

    def get_all_ep_from_queue(self, show):
        with self.lock:
            ep_obj_list = []
            for cur_item in self.queue:
                if (isinstance(cur_item, (ManualSearchQueueItem, FailedQueueItem)) and
                        show == str(cur_item.show.indexerid)):
                    ep_obj_list.append(cur_item)

            if ep_obj_list:
                return ep_obj_list
            return False

    def pause_backlog(self):
        with self.lock:
            self.min_priority = generic_queue.QueuePriorities.HIGH

    def unpause_backlog(self):
        with self.lock:
            self.min_priority = 0

    def is_backlog_paused(self):
        # backlog priorities are NORMAL, this should be done properly somewhere
        with self.lock:
            return self.min_priority >= generic_queue.QueuePriorities.NORMAL

    def _is_in_progress(self, item_type):
        with self.lock:
            for cur_item in self.queue + [self.currentItem]:
                if isinstance(cur_item, item_type):
                    return True
            return False

    def is_manualsearch_in_progress(self):
        # Only referenced in webserve.py, only current running manualsearch or failedsearch is needed!!
        return self._is_in_progress((ManualSearchQueueItem, FailedQueueItem))

    def is_backlog_in_progress(self):
        return self._is_in_progress(BacklogQueueItem)

    def is_recentsearch_in_progress(self):
        return self._is_in_progress(RecentSearchQueueItem)

    def is_propersearch_in_progress(self):
        with self.lock:
            for cur_item in self.queue + [self.currentItem]:
                if isinstance(cur_item, ProperSearchQueueItem) and None is cur_item.propers:
                    return True
            return False

    def is_standard_backlog_in_progress(self):
        with self.lock:
            for cur_item in self.queue + [self.currentItem]:
                if isinstance(cur_item, BacklogQueueItem) and cur_item.standard_backlog:
                    return True
            return False

    def type_of_backlog_in_progress(self):
        limited = full = other = False
        with self.lock:
            for cur_item in self.queue + [self.currentItem]:
                if isinstance(cur_item, BacklogQueueItem):
                    if cur_item.standard_backlog:
                        if cur_item.limited_backlog:
                            limited = True
                        else:
                            full = True
                    else:
                        other = True

            types = []
            for msg, variant in ['Limited', limited], ['Full', full], ['On Demand', other]:
                if variant:
                    types.append(msg)
            message = 'None'
            if types:
                message = ', '.join(types)
            return message

    def queue_length(self):
        length = {'backlog': [], 'recent': 0, 'manual': [], 'failed': [], 'proper': []}
        with self.lock:
            for cur_item in [self.currentItem] + self.queue:
                if isinstance(cur_item, RecentSearchQueueItem):
                    length['recent'] += 1
                elif isinstance(cur_item, BacklogQueueItem):
                    length['backlog'] += [dict(indexerid=cur_item.show.indexerid, indexer=cur_item.show.indexer,
                                               name=cur_item.show.name, segment=cur_item.segment,
                                               standard_backlog=cur_item.standard_backlog,
                                               limited_backlog=cur_item.limited_backlog, forced=cur_item.forced,
                                               torrent_only=cur_item.torrent_only)]
                elif isinstance(cur_item, ProperSearchQueueItem):
                    length['proper'] += [dict(recent=None is not cur_item.propers)]
                elif isinstance(cur_item, ManualSearchQueueItem):
                    length['manual'] += [dict(indexerid=cur_item.show.indexerid, indexer=cur_item.show.indexer,
                                              name=cur_item.show.name, segment=cur_item.segment)]
                elif isinstance(cur_item, FailedQueueItem):
                    length['failed'] += [dict(indexerid=cur_item.show.indexerid, indexer=cur_item.show.indexer,
                                              name=cur_item.show.name, segment=cur_item.segment)]
            return length

    def add_item(self, item):
        if isinstance(item, (RecentSearchQueueItem, ProperSearchQueueItem)):
            # recent and proper searches
            generic_queue.GenericQueue.add_item(self, item)
        elif isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment):
            # backlog searches
            generic_queue.GenericQueue.add_item(self, item)
        elif isinstance(item, (ManualSearchQueueItem, FailedQueueItem)) and not self.is_ep_in_queue(item.segment):
            # manual and failed searches
            generic_queue.GenericQueue.add_item(self, item)
        else:
            logger.log(u'Not adding item, it\'s already in the queue', logger.DEBUG)


class RecentSearchQueueItem(generic_queue.QueueItem):
    def __init__(self):
        self.success = None
        self.episodes = []
        generic_queue.QueueItem.__init__(self, 'Recent Search', RECENT_SEARCH)

    def run(self):
        generic_queue.QueueItem.run(self)

        try:
            self._change_missing_episodes()

            show_list = sickbeard.showList
            from_date = datetime.date.fromordinal(1)
            needed = common.neededQualities()
            for curShow in show_list:
                if curShow.paused:
                    continue

                wanted_eps = wanted_episodes(curShow, from_date, unaired=sickbeard.SEARCH_UNAIRED)

                if wanted_eps:
                    if not needed.all_needed:
                        if not needed.all_types_needed:
                            needed.check_needed_types(curShow)
                        if not needed.all_qualities_needed:
                            for w in wanted_eps:
                                if needed.all_qualities_needed:
                                    break
                                if not w.show.is_anime and not w.show.is_sports:
                                    needed.check_needed_qualities(w.wantedQuality)

                    self.episodes.extend(wanted_eps)

            if sickbeard.DOWNLOAD_PROPERS:
                properFinder.get_needed_qualites(needed)

            self.update_providers(needed=needed)
            self._check_for_propers(needed)

            if not self.episodes:
                logger.log(u'No search of cache for episodes required')
                self.success = True
            else:
                num_shows = len(set([ep.show.name for ep in self.episodes]))
                logger.log(u'Found %d needed episode%s spanning %d show%s'
                           % (len(self.episodes), helpers.maybe_plural(len(self.episodes)),
                              num_shows, helpers.maybe_plural(num_shows)))

                try:
                    logger.log(u'Beginning recent search for episodes')
                    found_results = search.search_for_needed_episodes(self.episodes)

                    if not len(found_results):
                        logger.log(u'No needed episodes found')
                    else:
                        for result in found_results:
                            # just use the first result for now
                            logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
                            self.success = search.snatch_episode(result)

                            helpers.cpu_sleep()

                except (StandardError, Exception):
                    logger.log(traceback.format_exc(), logger.ERROR)

                if None is self.success:
                    self.success = False

        finally:
            self.finish()

    @staticmethod
    def _check_for_propers(needed):
        if not sickbeard.DOWNLOAD_PROPERS:
            return

        propers = {}
        my_db = db.DBConnection('cache.db')
        sql_results = my_db.select('SELECT * FROM provider_cache')
        re_p = (r'\brepack|proper|real\b', r'\brepack|proper|real|v[1-5]\b')[needed.need_anime]

        proper_regex = re.compile(re_p, flags=re.I)

        for s in sql_results:
            if proper_regex.search(s['name']):
                try:
                    show = helpers.find_show_by_id(sickbeard.showList, {INDEXER_TVDB: int(s['indexerid'])})
                except (StandardError, Exception):
                    continue
                if show:
                    propers.setdefault(s['provider'], []).append(
                        Proper(s['name'], s['url'], datetime.datetime.fromtimestamp(s['time']), show, parsed_show=show))

        if propers:
            logger.log('Found Proper/Repack/Real in recent search, sending data to properfinder')
            propersearch_queue_item = sickbeard.search_queue.ProperSearchQueueItem(propers=propers)
            sickbeard.searchQueueScheduler.action.add_item(propersearch_queue_item)

    @staticmethod
    def _change_missing_episodes():
        if not network_timezones.network_dict:
            network_timezones.update_network_dict()

        if network_timezones.network_dict:
            cur_date = (datetime.date.today() + datetime.timedelta(days=1)).toordinal()
        else:
            cur_date = (datetime.date.today() - datetime.timedelta(days=2)).toordinal()

        cur_time = datetime.datetime.now(network_timezones.sb_timezone)

        my_db = db.DBConnection()
        sql_results = my_db.select(
            'SELECT * FROM tv_episodes WHERE status = ? AND season > 0 AND airdate <= ? AND airdate > 1',
            [common.UNAIRED, cur_date])

        sql_l = []
        show = None
        wanted = False

        for sqlEp in sql_results:
            try:
                if not show or show.indexerid != int(sqlEp['showid']):
                    show = helpers.findCertainShow(sickbeard.showList, int(sqlEp['showid']))

                # for when there is orphaned series in the database but not loaded into our showlist
                if not show:
                    continue

            except exceptions.MultipleShowObjectsException:
                logger.log(u'ERROR: expected to find a single show matching %s' % sqlEp['showid'])
                continue

            try:
                end_time = (network_timezones.parse_date_time(sqlEp['airdate'], show.airs, show.network) +
                            datetime.timedelta(minutes=helpers.tryInt(show.runtime, 60)))
                # filter out any episodes that haven't aired yet
                if end_time > cur_time:
                    continue
            except (StandardError, Exception):
                # if an error occurred assume the episode hasn't aired yet
                continue

            ep = show.getEpisode(int(sqlEp['season']), int(sqlEp['episode']))
            with ep.lock:
                # Now that it is time, change state of UNAIRED show into expected or skipped
                ep.status = (common.WANTED, common.SKIPPED)[ep.show.paused]
                result = ep.get_sql()
                if None is not result:
                    sql_l.append(result)
                    wanted |= (False, True)[common.WANTED == ep.status]
        else:
            logger.log(u'No unaired episodes marked wanted')

        if 0 < len(sql_l):
            my_db = db.DBConnection()
            my_db.mass_action(sql_l)
            if wanted:
                logger.log(u'Found new episodes marked wanted')

    @staticmethod
    def update_providers(needed=common.neededQualities(need_all=True)):
        orig_thread_name = threading.currentThread().name
        threads = []

        providers = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_recentsearch]
        for cur_provider in providers:
            if not cur_provider.cache.should_update():
                continue

            if not threads:
                logger.log('Updating provider caches with recent upload data')

            # spawn a thread for each provider to save time waiting for slow response providers
            threads.append(threading.Thread(target=cur_provider.cache.updateCache,
                                            kwargs={'needed': needed},
                                            name='%s :: [%s]' % (orig_thread_name, cur_provider.name)))
            # start the thread we just created
            threads[-1].start()

        if not len(providers):
            logger.log('No NZB/Torrent providers in Media Providers/Options are enabled to match recent episodes',
                       logger.WARNING)

        if threads:
            # wait for all threads to finish
            for t in threads:
                t.join()

            logger.log('Finished updating provider caches')


class ProperSearchQueueItem(generic_queue.QueueItem):
    def __init__(self, propers=None):
        generic_queue.QueueItem.__init__(self, 'Proper Search', PROPER_SEARCH)
        self.priority = (generic_queue.QueuePriorities.VERYHIGH, generic_queue.QueuePriorities.HIGH)[None is propers]
        self.propers = propers
        self.success = None

    def run(self):
        generic_queue.QueueItem.run(self)

        try:
            properFinder.search_propers(self.propers)
        finally:
            self.finish()


class ManualSearchQueueItem(generic_queue.QueueItem):
    def __init__(self, show, segment):
        generic_queue.QueueItem.__init__(self, 'Manual Search', MANUAL_SEARCH)
        self.priority = generic_queue.QueuePriorities.HIGH
        self.name = 'MANUAL-%s' % show.indexerid
        self.success = None
        self.show = show
        self.segment = segment
        self.started = None

    def run(self):
        generic_queue.QueueItem.run(self)

        try:
            logger.log(u'Beginning manual search for: [%s]' % self.segment.prettyName())
            self.started = True

            ep_count, ep_count_scene = get_aired_in_season(self.show)
            set_wanted_aired(self.segment, True, ep_count, ep_count_scene, manual=True)

            search_result = search.search_providers(self.show, [self.segment], True, try_other_searches=True)

            if search_result:
                # just use the first result for now
                logger.log(u'Downloading %s from %s' % (search_result[0].name, search_result[0].provider.name))
                self.success = search.snatch_episode(search_result[0])

                helpers.cpu_sleep()

            else:
                ui.notifications.message('No downloads found',
                                         u'Could not find a download for <i>%s</i>' % self.segment.prettyName())

                logger.log(u'Unable to find a download for: [%s]' % self.segment.prettyName())

        except (StandardError, Exception):
            logger.log(traceback.format_exc(), logger.ERROR)

        finally:
            # Keep a list with the 100 last executed searches
            fifo(MANUAL_SEARCH_HISTORY, self, MANUAL_SEARCH_HISTORY_SIZE)

            if self.success is None:
                self.success = False

            self.finish()


class BacklogQueueItem(generic_queue.QueueItem):
    def __init__(self, show, segment, standard_backlog=False, limited_backlog=False, forced=False, torrent_only=False):
        generic_queue.QueueItem.__init__(self, 'Backlog', BACKLOG_SEARCH)
        self.priority = generic_queue.QueuePriorities.LOW
        self.name = 'BACKLOG-%s' % show.indexerid
        self.success = None
        self.show = show
        self.segment = segment
        self.standard_backlog = standard_backlog
        self.limited_backlog = limited_backlog
        self.forced = forced
        self.torrent_only = torrent_only

    def run(self):
        generic_queue.QueueItem.run(self)

        is_error = False
        try:
            if not self.standard_backlog:
                ep_count, ep_count_scene = get_aired_in_season(self.show)
                for ep_obj in self.segment:
                    set_wanted_aired(ep_obj, True, ep_count, ep_count_scene)

            logger.log(u'Beginning backlog search for: [%s]' % self.show.name)
            search_result = search.search_providers(
                self.show, self.segment, False,
                try_other_searches=(not self.standard_backlog or not self.limited_backlog),
                scheduled=self.standard_backlog)

            if search_result:
                for result in search_result:
                    # just use the first result for now
                    logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
                    search.snatch_episode(result)

                    helpers.cpu_sleep()
            else:
                logger.log(u'No needed episodes found during backlog search for: [%s]' % self.show.name)
        except (StandardError, Exception):
            is_error = True
            logger.log(traceback.format_exc(), logger.ERROR)

        finally:
            logger.log('Completed backlog search %sfor: [%s]' % (('', 'with a debug error ')[is_error], self.show.name))
            self.finish()


class FailedQueueItem(generic_queue.QueueItem):
    def __init__(self, show, segment):
        generic_queue.QueueItem.__init__(self, 'Retry', FAILED_SEARCH)
        self.priority = generic_queue.QueuePriorities.HIGH
        self.name = 'RETRY-%s' % show.indexerid
        self.show = show
        self.segment = segment
        self.success = None
        self.started = None

    def run(self):
        generic_queue.QueueItem.run(self)
        self.started = True

        try:
            ep_count, ep_count_scene = get_aired_in_season(self.show)
            for ep_obj in self.segment:

                logger.log(u'Marking episode as bad: [%s]' % ep_obj.prettyName())

                failed_history.set_episode_failed(ep_obj)
                (release, provider) = failed_history.find_release(ep_obj)
                failed_history.revert_episode(ep_obj)
                if release:
                    failed_history.add_failed(release)
                    history.log_failed(ep_obj, release, provider)

                logger.log(u'Beginning failed download search for: [%s]' % ep_obj.prettyName())

                set_wanted_aired(ep_obj, True, ep_count, ep_count_scene)

            search_result = search.search_providers(self.show, self.segment, True, try_other_searches=True)

            if search_result:
                for result in search_result:
                    # just use the first result for now
                    logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
                    search.snatch_episode(result)

                    helpers.cpu_sleep()
            else:
                pass
                # logger.log(u'No valid episode found to retry for: [%s]' % self.segment.prettyName())
        except (StandardError, Exception):
            logger.log(traceback.format_exc(), logger.ERROR)

        finally:
            # Keep a list with the 100 last executed searches
            fifo(MANUAL_SEARCH_HISTORY, self, MANUAL_SEARCH_HISTORY_SIZE)

            if self.success is None:
                self.success = False

            self.finish()


def fifo(my_list, item, max_size=100):
    if len(my_list) >= max_size:
        my_list.pop(0)
    my_list.append(item)