mirror of
https://github.com/SickGear/SickGear.git
synced 2024-12-01 08:53:37 +00:00
dd7d5bcfec
Add try to match .nzb or .torrent name in history for pp, fallback to dirname. Add showObj lookup to scheduled pp. Add a debug log message for snatched shows found in history during post processing. Add use filename of largest video file for history lookups.
914 lines
42 KiB
Python
914 lines
42 KiB
Python
# Author: Nic Wolfe <nic@wolfeden.ca>
|
|
# URL: http://code.google.com/p/sickbeard/
|
|
#
|
|
# This file is part of SickGear.
|
|
#
|
|
# SickGear is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# SickGear is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import with_statement
|
|
|
|
from functools import partial
|
|
import datetime
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
import time
|
|
|
|
import sickbeard
|
|
from sickbeard import postProcessor
|
|
from sickbeard import db, helpers, exceptions
|
|
from sickbeard import encodingKludge as ek
|
|
from sickbeard.exceptions import ex
|
|
from sickbeard import logger
|
|
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
|
|
from sickbeard import common
|
|
from sickbeard.common import SNATCHED, SNATCHED_PROPER, SNATCHED_BEST
|
|
from sickbeard.history import reset_status
|
|
from sickbeard.exceptions import MultipleShowObjectsException
|
|
|
|
from sickbeard import failedProcessor
|
|
|
|
import lib.rarfile.rarfile as rarfile
|
|
|
|
try:
|
|
import json
|
|
except ImportError:
|
|
from lib import simplejson as json
|
|
|
|
try:
|
|
from lib.send2trash import send2trash
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
# noinspection PyArgumentList
|
|
class ProcessTVShow(object):
|
|
""" Process a TV Show """
|
|
|
|
def __init__(self, webhandler=None):
|
|
self.files_passed = 0
|
|
self.files_failed = 0
|
|
self.fail_detected = False
|
|
self._output = []
|
|
self.webhandler = webhandler
|
|
|
|
@property
|
|
def any_vid_processed(self):
|
|
return 0 < self.files_passed
|
|
|
|
@property
|
|
def result(self, pre=True):
|
|
return (('<br />', u'\n')[pre]).join(self._output)
|
|
|
|
def _buffer(self, text=None):
|
|
if None is not text:
|
|
self._output.append(text)
|
|
if self.webhandler:
|
|
logger_msg = re.sub(r'(?i)<br(?:[\s/]+)>', '\n', text)
|
|
logger_msg = re.sub('(?i)<a[^>]+>([^<]+)<[/]a>', r'\1', logger_msg)
|
|
self.webhandler('%s%s' % (logger_msg, u'\n'))
|
|
|
|
def _log_helper(self, message, log_level=logger.DEBUG):
|
|
logger_msg = re.sub(r'(?i)<br(?:[\s/]+)>\.*', '', message)
|
|
logger_msg = re.sub('(?i)<a[^>]+>([^<]+)<[/]a>', r'\1', logger_msg)
|
|
logger.log(u'%s' % logger_msg, log_level)
|
|
self._buffer(message)
|
|
return
|
|
|
|
def _set_process_success(self, state=True, reset=False):
|
|
if state:
|
|
self.files_passed += 1
|
|
else:
|
|
self.files_failed += 1
|
|
if reset:
|
|
self.files_passed = 0
|
|
self.files_failed = 0
|
|
|
|
def _delete_folder(self, folder, check_empty=True):
|
|
|
|
# check if it's a folder
|
|
if not ek.ek(os.path.isdir, folder):
|
|
return False
|
|
|
|
# make sure it isn't TV_DOWNLOAD_DIR
|
|
if sickbeard.TV_DOWNLOAD_DIR and helpers.real_path(sickbeard.TV_DOWNLOAD_DIR) == helpers.real_path(folder):
|
|
return False
|
|
|
|
# check if it's empty folder when wanted checked
|
|
if check_empty and ek.ek(os.listdir, folder):
|
|
return False
|
|
|
|
# try deleting folder
|
|
try:
|
|
shutil.rmtree(folder)
|
|
except (OSError, IOError) as e:
|
|
logger.log(u'Warning: unable to delete folder: %s: %s' % (folder, ex(e)), logger.WARNING)
|
|
return False
|
|
|
|
if ek.ek(os.path.isdir, folder):
|
|
logger.log(u'Warning: unable to delete folder: %s' % folder, logger.WARNING)
|
|
return False
|
|
|
|
self._log_helper(u'Deleted folder ' + folder, logger.MESSAGE)
|
|
return True
|
|
|
|
def _delete_files(self, process_path, notwanted_files, use_trash=False, force=False):
|
|
|
|
if not self.any_vid_processed and not force:
|
|
return
|
|
|
|
result = True
|
|
# Delete all file not needed
|
|
for cur_file in notwanted_files:
|
|
|
|
cur_file_path = ek.ek(os.path.join, process_path, cur_file)
|
|
|
|
if not ek.ek(os.path.isfile, cur_file_path):
|
|
continue # Prevent error when a notwantedfiles is an associated files
|
|
|
|
# check first the read-only attribute
|
|
file_attribute = ek.ek(os.stat, cur_file_path)[0]
|
|
if not file_attribute & stat.S_IWRITE:
|
|
# File is read-only, so make it writeable
|
|
self._log_helper(u'Changing ReadOnly flag for file ' + cur_file)
|
|
try:
|
|
ek.ek(os.chmod, cur_file_path, stat.S_IWRITE)
|
|
except OSError as e:
|
|
self._log_helper(u'Cannot change permissions of %s: %s' % (cur_file_path, str(e.strerror)))
|
|
try:
|
|
if use_trash:
|
|
ek.ek(send2trash, cur_file_path)
|
|
else:
|
|
ek.ek(os.remove, cur_file_path)
|
|
except OSError as e:
|
|
self._log_helper(u'Unable to delete file %s: %s' % (cur_file, str(e.strerror)))
|
|
|
|
if ek.ek(os.path.isfile, cur_file_path):
|
|
result = False
|
|
else:
|
|
self._log_helper(u'Deleted file ' + cur_file)
|
|
|
|
return result
|
|
|
|
def check_name(self, name):
|
|
so = None
|
|
my_db = db.DBConnection()
|
|
sql_results = my_db.select('SELECT showid FROM history WHERE resource = ?'
|
|
'AND (' + ' OR '.join(
|
|
"action LIKE '%%%02d'" % x for x in (SNATCHED, SNATCHED_PROPER,
|
|
SNATCHED_BEST)) + ') ORDER BY rowid', [name])
|
|
if sql_results:
|
|
try:
|
|
so = helpers.findCertainShow(sickbeard.showList, int(sql_results[-1]['showid']))
|
|
if hasattr(so, 'name'):
|
|
logger.log('Found Show: %s in snatch history for: %s' % (so.name, name), logger.DEBUG)
|
|
except MultipleShowObjectsException:
|
|
so = None
|
|
return so
|
|
|
|
def showObj_helper(self, showObj, base_dir, dir_name, nzb_name, pp_type, alt_showObj=None):
|
|
if None is showObj and base_dir == sickbeard.TV_DOWNLOAD_DIR and not nzb_name or 'manual' == pp_type: # Scheduled Post Processing Active
|
|
return self.check_name(dir_name)
|
|
return (showObj, alt_showObj)[None is showObj and None is not alt_showObj]
|
|
|
|
def check_video_filenames(self, path, videofiles):
|
|
video_pick = None
|
|
video_size = 0
|
|
for cur_video_file in videofiles:
|
|
try:
|
|
cur_video_size = ek.ek(os.path.getsize, ek.ek(os.path.join, path, cur_video_file))
|
|
except (StandardError, Exception):
|
|
continue
|
|
|
|
if 0 == video_size or cur_video_size > video_size:
|
|
video_size = cur_video_size
|
|
video_pick = cur_video_file
|
|
|
|
if video_pick:
|
|
vid_filename = ek.ek(os.path.splitext, video_pick)[0]
|
|
# check if filename is garbage, disregard it
|
|
if re.search(r'^[a-zA-Z0-9]+$', vid_filename):
|
|
return None
|
|
|
|
return self.check_name(vid_filename)
|
|
|
|
return None
|
|
|
|
def process_dir(self, dir_name, nzb_name=None, process_method=None, force=False, force_replace=None, failed=False, pp_type='auto', cleanup=False, showObj=None):
|
|
"""
|
|
Scans through the files in dir_name and processes whatever media files it finds
|
|
|
|
dir_name: The folder name to look in
|
|
nzb_name: The NZB name which resulted in this folder being downloaded
|
|
force: True to postprocess already postprocessed files
|
|
failed: Boolean for whether or not the download failed
|
|
pp_type: Type of postprocessing auto or manual
|
|
"""
|
|
|
|
# if they passed us a real directory then assume it's the one we want
|
|
if dir_name and ek.ek(os.path.isdir, dir_name):
|
|
dir_name = ek.ek(os.path.realpath, dir_name)
|
|
|
|
# if the client and SickGear are not on the same machine translate the directory in a network directory
|
|
elif dir_name and sickbeard.TV_DOWNLOAD_DIR and ek.ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR)\
|
|
and ek.ek(os.path.normpath, dir_name) != ek.ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR):
|
|
dir_name = ek.ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek.ek(os.path.abspath, dir_name).split(os.path.sep)[-1])
|
|
self._log_helper(u'SickGear PP Config, completed TV downloads folder: ' + sickbeard.TV_DOWNLOAD_DIR)
|
|
|
|
if dir_name:
|
|
self._log_helper(u'Checking folder... ' + dir_name)
|
|
|
|
# if we didn't find a real directory then process "failed" or just quit
|
|
if not dir_name or not ek.ek(os.path.isdir, dir_name):
|
|
if nzb_name and failed:
|
|
self._process_failed(dir_name, nzb_name, showObj=showObj)
|
|
else:
|
|
self._log_helper(u'Unable to figure out what folder to process. ' +
|
|
u'If your downloader and SickGear aren\'t on the same PC then make sure ' +
|
|
u'you fill out your completed TV download folder in the PP config.')
|
|
return self.result
|
|
|
|
if None is showObj:
|
|
if isinstance(nzb_name, basestring):
|
|
showObj = self.check_name(re.sub(r'\.(nzb|torrent)$', '', nzb_name, flags=re.I))
|
|
|
|
if None is showObj and dir_name:
|
|
showObj = self.check_name(ek.ek(os.path.basename, dir_name))
|
|
|
|
path, dirs, files = self._get_path_dir_files(dir_name, nzb_name, pp_type)
|
|
|
|
if sickbeard.POSTPONE_IF_SYNC_FILES and any(filter(helpers.isSyncFile, files)):
|
|
self._log_helper(u'Found temporary sync files, skipping post process', logger.ERROR)
|
|
return self.result
|
|
|
|
if not process_method:
|
|
process_method = sickbeard.PROCESS_METHOD
|
|
|
|
self._log_helper(u'Processing folder... %s' % path)
|
|
|
|
work_files = []
|
|
joined = self.join(path)
|
|
if joined:
|
|
work_files += [joined]
|
|
|
|
rar_files, rarfile_history = self.unused_archives(
|
|
path, filter(helpers.is_first_rar_volume, files), pp_type, process_method)
|
|
rar_content = self._unrar(path, rar_files, force)
|
|
if self.fail_detected:
|
|
self._process_failed(dir_name, nzb_name, showObj=showObj)
|
|
return self.result
|
|
path, dirs, files = self._get_path_dir_files(dir_name, nzb_name, pp_type)
|
|
video_files = filter(helpers.has_media_ext, files)
|
|
video_in_rar = filter(helpers.has_media_ext, rar_content)
|
|
work_files += [ek.ek(os.path.join, path, item) for item in rar_content]
|
|
|
|
if 0 < len(files):
|
|
self._log_helper(u'Process file%s: %s' % (helpers.maybe_plural(files), str(files)))
|
|
if 0 < len(video_files):
|
|
self._log_helper(u'Process video file%s: %s' % (helpers.maybe_plural(video_files), str(video_files)))
|
|
if 0 < len(rar_content):
|
|
self._log_helper(u'Process rar content: ' + str(rar_content))
|
|
if 0 < len(video_in_rar):
|
|
self._log_helper(u'Process video%s in rar: %s' % (helpers.maybe_plural(video_in_rar), str(video_in_rar)))
|
|
|
|
# If nzb_name is set and there's more than one videofile in the folder, files will be lost (overwritten).
|
|
nzb_name_original = nzb_name
|
|
if 2 <= len(video_files):
|
|
nzb_name = None
|
|
|
|
if None is showObj and 0 < len(video_files):
|
|
showObj = self.check_video_filenames(path, video_files)
|
|
|
|
# self._set_process_success()
|
|
|
|
# Don't Link media when the media is extracted from a rar in the same path
|
|
if process_method in ('hardlink', 'symlink') and video_in_rar:
|
|
soh = showObj
|
|
if None is showObj:
|
|
soh = self.check_video_filenames(path, video_in_rar)
|
|
self._process_media(path, video_in_rar, nzb_name, 'move', force, force_replace, showObj=soh)
|
|
self._delete_files(path, [ek.ek(os.path.relpath, item, path) for item in work_files], force=True)
|
|
video_batch = set(video_files) - set(video_in_rar)
|
|
else:
|
|
video_batch = video_files
|
|
|
|
try:
|
|
while 0 < len(video_batch):
|
|
video_pick = ['']
|
|
video_size = 0
|
|
for cur_video_file in video_batch:
|
|
cur_video_size = ek.ek(os.path.getsize, ek.ek(os.path.join, path, cur_video_file))
|
|
if 0 == video_size or cur_video_size > video_size:
|
|
video_size = cur_video_size
|
|
video_pick = [cur_video_file]
|
|
|
|
video_batch = set(video_batch) - set(video_pick)
|
|
|
|
self._process_media(path, video_pick, nzb_name, process_method, force, force_replace, use_trash=cleanup, showObj=showObj)
|
|
|
|
except OSError as e:
|
|
logger.log('Batch skipped, %s%s' %
|
|
(ex(e), e.filename and (' (file %s)' % e.filename) or ''), logger.WARNING)
|
|
|
|
# Process video files in TV subdirectories
|
|
for directory in [x for x in dirs if self._validate_dir(path, x, nzb_name_original, failed, showObj=self.showObj_helper(showObj, dir_name, x, nzb_name, pp_type))]:
|
|
|
|
# self._set_process_success(reset=True)
|
|
|
|
for walk_path, walk_dir, files in ek.ek(os.walk, ek.ek(os.path.join, path, directory), topdown=False):
|
|
|
|
if sickbeard.POSTPONE_IF_SYNC_FILES and any(filter(helpers.isSyncFile, files)):
|
|
self._log_helper(u'Found temporary sync files, skipping post process', logger.ERROR)
|
|
return self.result
|
|
|
|
rar_files, rarfile_history = self.unused_archives(
|
|
walk_path, filter(helpers.is_first_rar_volume, files), pp_type, process_method, rarfile_history)
|
|
rar_content = self._unrar(walk_path, rar_files, force)
|
|
work_files += [ek.ek(os.path.join, walk_path, item) for item in rar_content]
|
|
if self.fail_detected:
|
|
self._process_failed(dir_name, nzb_name, showObj=self.showObj_helper(showObj, directory))
|
|
continue
|
|
files = list(set(files + rar_content))
|
|
video_files = filter(helpers.has_media_ext, files)
|
|
video_in_rar = filter(helpers.has_media_ext, rar_content)
|
|
notwanted_files = [x for x in files if x not in video_files]
|
|
|
|
# Don't Link media when the media is extracted from a rar in the same path
|
|
if process_method in ('hardlink', 'symlink') and video_in_rar:
|
|
self._process_media(walk_path, video_in_rar, nzb_name, 'move', force, force_replace, showObj=self.showObj_helper(showObj, dir_name, directory, nzb_name, pp_type, self.check_video_filenames(walk_dir, video_in_rar)))
|
|
video_batch = set(video_files) - set(video_in_rar)
|
|
else:
|
|
video_batch = video_files
|
|
|
|
try:
|
|
while 0 < len(video_batch):
|
|
video_pick = ['']
|
|
video_size = 0
|
|
for cur_video_file in video_batch:
|
|
cur_video_size = ek.ek(os.path.getsize, ek.ek(os.path.join, walk_path, cur_video_file))
|
|
|
|
if 0 == video_size or cur_video_size > video_size:
|
|
video_size = cur_video_size
|
|
video_pick = [cur_video_file]
|
|
|
|
video_batch = set(video_batch) - set(video_pick)
|
|
|
|
self._process_media(walk_path, video_pick, nzb_name, process_method, force, force_replace, use_trash=cleanup, showObj=self.showObj_helper(showObj, dir_name, directory, nzb_name, pp_type, self.check_video_filenames(walk_dir, video_pick)))
|
|
|
|
except OSError as e:
|
|
logger.log('Batch skipped, %s%s' %
|
|
(ex(e), e.filename and (' (file %s)' % e.filename) or ''), logger.WARNING)
|
|
|
|
if process_method in ('hardlink', 'symlink') and video_in_rar:
|
|
self._delete_files(walk_path, rar_content)
|
|
else:
|
|
# Delete all file not needed
|
|
if not self.any_vid_processed\
|
|
or 'move' != process_method\
|
|
or ('manual' == pp_type and not cleanup): # Avoid deleting files if Manual Postprocessing
|
|
continue
|
|
|
|
self._delete_files(walk_path, notwanted_files, use_trash=cleanup)
|
|
|
|
if 'move' == process_method\
|
|
and ek.ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR) != ek.ek(os.path.normpath, walk_path):
|
|
self._delete_folder(walk_path, check_empty=False)
|
|
|
|
if 'copy' == process_method and work_files:
|
|
self._delete_files(path, [ek.ek(os.path.relpath, item, path) for item in work_files], force=True)
|
|
for f in sorted(list(set([ek.ek(os.path.dirname, item) for item in work_files]) - {path}),
|
|
key=len, reverse=True):
|
|
self._delete_folder(f)
|
|
|
|
def _bottom_line(text, log_level=logger.DEBUG):
|
|
self._buffer('-' * len(text))
|
|
self._log_helper(text, log_level)
|
|
|
|
if self.any_vid_processed:
|
|
if not self.files_failed:
|
|
_bottom_line(u'Successfully processed.', logger.MESSAGE)
|
|
else:
|
|
_bottom_line(u'Successfully processed at least one video file%s.' %
|
|
(', others were skipped', ' and skipped another')[1 == self.files_failed], logger.MESSAGE)
|
|
else:
|
|
_bottom_line(u'Failed! Did not process any files.', logger.WARNING)
|
|
|
|
return self.result
|
|
|
|
@staticmethod
|
|
def unused_archives(path, archives, pp_type, process_method, archive_history=None):
|
|
|
|
archive_history = (archive_history, {})[not archive_history]
|
|
if ('auto' == pp_type and sickbeard.PROCESS_AUTOMATICALLY
|
|
and 'copy' == process_method and sickbeard.UNPACK):
|
|
|
|
archive_history_file = ek.ek(os.path.join, sickbeard.DATA_DIR, 'archive_history.txt')
|
|
|
|
if not archive_history:
|
|
try:
|
|
with open(archive_history_file, 'r') as fh:
|
|
archive_history = json.loads(fh.read(10 * 1024 * 1024))
|
|
except (IOError, ValueError, Exception):
|
|
pass
|
|
|
|
init_history_cnt = len(archive_history)
|
|
|
|
for archive in archive_history.keys():
|
|
if not ek.ek(os.path.isfile, archive):
|
|
del archive_history[archive]
|
|
|
|
unused_files = list(set([ek.ek(os.path.join, path, x) for x in archives]) - set(archive_history.keys()))
|
|
archives = [ek.ek(os.path.basename, x) for x in unused_files]
|
|
if unused_files:
|
|
for f in unused_files:
|
|
archive_history.setdefault(f, time.mktime(datetime.datetime.utcnow().timetuple()))
|
|
|
|
if init_history_cnt != len(archive_history):
|
|
try:
|
|
with open(archive_history_file, 'w') as fh:
|
|
fh.write(json.dumps(archive_history))
|
|
except (IOError, Exception):
|
|
pass
|
|
|
|
return archives, archive_history
|
|
|
|
def _validate_dir(self, path, dir_name, nzb_name_original, failed, showObj=None):
|
|
|
|
self._log_helper(u'Processing sub dir: ' + dir_name)
|
|
|
|
if ek.ek(os.path.basename, dir_name).startswith('_FAILED_'):
|
|
self._log_helper(u'The directory name indicates it failed to extract.')
|
|
failed = True
|
|
elif ek.ek(os.path.basename, dir_name).startswith('_UNDERSIZED_'):
|
|
self._log_helper(u'The directory name indicates that it was previously rejected for being undersized.')
|
|
failed = True
|
|
elif ek.ek(os.path.basename, dir_name).upper().startswith('_UNPACK'):
|
|
self._log_helper(u'The directory name indicates that this release is in the process of being unpacked.')
|
|
return False
|
|
|
|
if failed:
|
|
self._process_failed(ek.ek(os.path.join, path, dir_name), nzb_name_original, showObj=showObj)
|
|
return False
|
|
|
|
if helpers.is_hidden_folder(dir_name):
|
|
self._log_helper(u'Ignoring hidden folder: ' + dir_name)
|
|
return False
|
|
|
|
# make sure the directory isn't inside a show directory
|
|
my_db = db.DBConnection()
|
|
sql_results = my_db.select('SELECT * FROM tv_shows')
|
|
|
|
for sqlShow in sql_results:
|
|
if dir_name.lower().startswith(ek.ek(os.path.realpath, sqlShow['location']).lower() + os.sep)\
|
|
or dir_name.lower() == ek.ek(os.path.realpath, sqlShow['location']).lower():
|
|
self._log_helper(
|
|
u'Found an episode that has already been moved to its show dir, skipping',
|
|
logger.ERROR)
|
|
return False
|
|
|
|
# Get the videofile list for the next checks
|
|
all_files = []
|
|
all_dirs = []
|
|
process_path = None
|
|
for process_path, process_dir, fileList in ek.ek(os.walk, ek.ek(os.path.join, path, dir_name), topdown=False):
|
|
all_dirs += process_dir
|
|
all_files += fileList
|
|
|
|
video_files = filter(helpers.has_media_ext, all_files)
|
|
all_dirs.append(dir_name)
|
|
|
|
# check if the directory have at least one tv video file
|
|
for video in video_files:
|
|
try:
|
|
NameParser(showObj=showObj).parse(video, cache_result=False)
|
|
return True
|
|
except (InvalidNameException, InvalidShowException):
|
|
pass
|
|
|
|
for directory in all_dirs:
|
|
try:
|
|
NameParser(showObj=showObj).parse(directory, cache_result=False)
|
|
return True
|
|
except (InvalidNameException, InvalidShowException):
|
|
pass
|
|
|
|
if sickbeard.UNPACK and process_path and all_files:
|
|
# Search for packed release
|
|
packed_files = filter(helpers.is_first_rar_volume, all_files)
|
|
|
|
for packed in packed_files:
|
|
try:
|
|
NameParser(showObj=showObj).parse(packed, cache_result=False)
|
|
return True
|
|
except (InvalidNameException, InvalidShowException):
|
|
pass
|
|
|
|
return False
|
|
|
|
def _unrar(self, path, rar_files, force):
|
|
|
|
unpacked_files = []
|
|
|
|
if 'win32' == sys.platform:
|
|
rarfile.UNRAR_TOOL = ek.ek(os.path.join, sickbeard.PROG_DIR, 'lib', 'rarfile', 'UnRAR.exe')
|
|
|
|
if sickbeard.UNPACK and rar_files:
|
|
|
|
self._log_helper(u'Packed releases detected: ' + str(rar_files))
|
|
|
|
for archive in rar_files:
|
|
|
|
self._log_helper(u'Unpacking archive: ' + archive)
|
|
|
|
try:
|
|
rar_handle = rarfile.RarFile(ek.ek(os.path.join, path, archive))
|
|
except (StandardError, Exception):
|
|
self._log_helper(u'Failed to open archive: %s' % archive, logger.ERROR)
|
|
self._set_process_success(False)
|
|
continue
|
|
try:
|
|
# Skip extraction if any file in archive has previously been extracted
|
|
skip_file = False
|
|
for file_in_archive in [ek.ek(os.path.basename, x.filename)
|
|
for x in rar_handle.infolist() if not x.isdir()]:
|
|
if self._already_postprocessed(path, file_in_archive, force):
|
|
self._log_helper(
|
|
u'Archive file already processed, extraction skipped: ' + file_in_archive)
|
|
skip_file = True
|
|
break
|
|
|
|
if not skip_file:
|
|
# need to test for password since rar4 doesn't raise PasswordRequired
|
|
if rar_handle.needs_password():
|
|
raise rarfile.PasswordRequired
|
|
|
|
rar_handle.extractall(path=path)
|
|
rar_content = [ek.ek(os.path.normpath, x.filename)
|
|
for x in rar_handle.infolist() if not x.isdir()]
|
|
renamed = self.cleanup_names(path, rar_content)
|
|
cur_unpacked = rar_content if not renamed else \
|
|
(list(set(rar_content) - set(renamed.keys())) + renamed.values())
|
|
self._log_helper(u'Unpacked content: [u\'%s\']' % '\', u\''.join(map(unicode, cur_unpacked)))
|
|
unpacked_files += cur_unpacked
|
|
except (rarfile.PasswordRequired, rarfile.RarWrongPassword):
|
|
self._log_helper(u'Failed to unpack archive PasswordRequired: %s' % archive, logger.ERROR)
|
|
self._set_process_success(False)
|
|
self.fail_detected = True
|
|
except (StandardError, Exception):
|
|
self._log_helper(u'Failed to unpack archive: %s' % archive, logger.ERROR)
|
|
self._set_process_success(False)
|
|
finally:
|
|
rar_handle.close()
|
|
del rar_handle
|
|
|
|
elif rar_files:
|
|
# check for passworded rar's
|
|
for archive in rar_files:
|
|
try:
|
|
rar_handle = rarfile.RarFile(ek.ek(os.path.join, path, archive))
|
|
except (StandardError, Exception):
|
|
self._log_helper(u'Failed to open archive: %s' % archive, logger.ERROR)
|
|
continue
|
|
try:
|
|
if rar_handle.needs_password():
|
|
self._log_helper(u'Failed to unpack archive PasswordRequired: %s' % archive, logger.ERROR)
|
|
self._set_process_success(False)
|
|
self.failure_detected = True
|
|
rar_handle.close()
|
|
del rar_handle
|
|
except (StandardError, Exception):
|
|
pass
|
|
|
|
return unpacked_files
|
|
|
|
@staticmethod
|
|
def cleanup_names(directory, files=None):
|
|
|
|
is_renamed = {}
|
|
num_videos = 0
|
|
old_name = None
|
|
new_name = None
|
|
params = {
|
|
'base_name': ek.ek(os.path.basename, directory),
|
|
'reverse_pattern': re.compile('|'.join([
|
|
r'\.\d{2}e\d{2}s\.', r'\.p0(?:63|27|612)\.', r'\.[pi](?:084|675|0801)\.', r'\b[45]62[xh]\.',
|
|
r'\.yarulb\.', r'\.vtd[hp]\.', r'\.(?:ld[.-]?)?bew\.', r'\.pir.?(?:shv|dov|dvd|bew|db|rb)\.',
|
|
r'\brdvd\.', r'\.(?:vts|dcv)\.', r'\b(?:mac|pir)dh\b', r'\.(?:lanretni|reporp|kcaper|reneercs)\.',
|
|
r'\b(?:caa|3ca|3pm)\b', r'\.cstn\.', r'\.5r\.', r'\brcs\b'
|
|
]), flags=re.IGNORECASE),
|
|
'season_pattern': re.compile(r'(.*\.\d{2}e\d{2}s\.)(.*)', flags=re.IGNORECASE),
|
|
'word_pattern': re.compile(r'([^A-Z0-9]*[A-Z0-9]+)'),
|
|
'char_replace': [[r'(\w)1\.(\w)', r'\1i\2']],
|
|
'garbage_name': re.compile(r'^[a-zA-Z0-9]{3,}$'),
|
|
'media_pattern': re.compile('|'.join([
|
|
r'\.s\d{2}e\d{2}\.', r'\.(?:36|72|216)0p\.', r'\.(?:480|576|1080)[pi]\.', r'\.[xh]26[45]\b',
|
|
r'\.bluray\.', r'\.[hp]dtv\.', r'\.web(?:[.-]?dl)?\.', r'\.(?:vhs|vod|dvd|web|bd|br).?rip\.',
|
|
r'\.dvdr\b', r'\.(?:stv|vcd)\.', r'\bhd(?:cam|rip)\b', r'\.(?:internal|proper|repack|screener)\.',
|
|
r'\b(?:aac|ac3|mp3)\b', r'\.(?:ntsc|pal|secam)\.', r'\.r5\.', r'\bscr\b', r'\b(?:divx|xvid)\b'
|
|
]), flags=re.IGNORECASE)
|
|
}
|
|
|
|
def renamer(_dirpath, _filenames, _num_videos, _old_name, _new_name, base_name,
|
|
reverse_pattern, season_pattern, word_pattern, char_replace, garbage_name, media_pattern):
|
|
|
|
for cur_filename in _filenames:
|
|
|
|
file_name, file_extension = ek.ek(os.path.splitext, cur_filename)
|
|
file_path = ek.ek(os.path.join, _dirpath, cur_filename)
|
|
dir_name = ek.ek(os.path.dirname, file_path)
|
|
|
|
if None is not reverse_pattern.search(file_name):
|
|
na_parts = season_pattern.search(file_name)
|
|
if None is not na_parts:
|
|
word_p = word_pattern.findall(na_parts.group(2))
|
|
new_words = ''
|
|
for wp in word_p:
|
|
if '.' == wp[0]:
|
|
new_words += '.'
|
|
new_words += re.sub(r'\W', '', wp)
|
|
for cr in char_replace:
|
|
new_words = re.sub(cr[0], cr[1], new_words)
|
|
new_filename = new_words[::-1] + na_parts.group(1)[::-1]
|
|
else:
|
|
new_filename = file_name[::-1]
|
|
logger.log('Reversing base filename "%s" to "%s"' % (file_name, new_filename))
|
|
try:
|
|
ek.ek(os.rename, file_path, ek.ek(os.path.join, _dirpath, new_filename + file_extension))
|
|
is_renamed[ek.ek(os.path.relpath, file_path, directory)] = ek.ek(
|
|
os.path.relpath, new_filename + file_extension, directory)
|
|
except OSError as e:
|
|
logger.log('Error unable to rename file "%s" because %s' % (cur_filename, ex(e)), logger.ERROR)
|
|
elif helpers.has_media_ext(cur_filename) and \
|
|
None is not garbage_name.search(file_name) and None is not media_pattern.search(base_name):
|
|
_num_videos += 1
|
|
_old_name = file_path
|
|
_new_name = ek.ek(os.path.join, dir_name, '%s%s' % (base_name, file_extension))
|
|
return is_renamed, _num_videos, _old_name, _new_name
|
|
|
|
if files:
|
|
is_renamed, num_videos, old_name, new_name = renamer(
|
|
directory, files, num_videos, old_name, new_name, **params)
|
|
else:
|
|
for cur_dirpath, void, cur_filenames in ek.ek(os.walk, directory):
|
|
is_renamed, num_videos, old_name, new_name = renamer(
|
|
cur_dirpath, cur_filenames, num_videos, old_name, new_name, **params)
|
|
|
|
if all([not is_renamed, 1 == num_videos, old_name, new_name]):
|
|
try_name = ek.ek(os.path.basename, new_name)
|
|
logger.log('Renaming file "%s" using dirname as "%s"' % (ek.ek(os.path.basename, old_name), try_name))
|
|
try:
|
|
ek.ek(os.rename, old_name, new_name)
|
|
is_renamed[ek.ek(os.path.relpath, old_name, directory)] = ek.ek(os.path.relpath, new_name, directory)
|
|
except OSError as e:
|
|
logger.log('Error unable to rename file "%s" because %s' % (old_name, ex(e)), logger.ERROR)
|
|
|
|
return is_renamed
|
|
|
|
def join(self, directory):
|
|
|
|
result = False
|
|
chunks = {}
|
|
matcher = re.compile('\.[0-9]+$')
|
|
for dirpath, void, filenames in ek.ek(os.walk, directory):
|
|
for filename in filenames:
|
|
if None is not matcher.search(filename):
|
|
maybe_chunk = ek.ek(os.path.join, dirpath, filename)
|
|
base_filepath, ext = ek.ek(os.path.splitext, maybe_chunk)
|
|
if base_filepath not in chunks:
|
|
chunks[base_filepath] = []
|
|
chunks[base_filepath].append(maybe_chunk)
|
|
|
|
if not chunks:
|
|
return
|
|
|
|
for base_filepath in chunks:
|
|
chunks[base_filepath].sort()
|
|
chunk_set = chunks[base_filepath]
|
|
if ek.ek(os.path.isfile, base_filepath):
|
|
base_filesize = ek.ek(os.path.getsize, base_filepath)
|
|
chunk_sizes = [ek.ek(os.path.getsize, x) for x in chunk_set]
|
|
largest_chunk = max(chunk_sizes)
|
|
if largest_chunk >= base_filesize:
|
|
outfile = '%s.001' % base_filepath
|
|
if outfile not in chunk_set:
|
|
try:
|
|
ek.ek(os.rename, base_filepath, outfile)
|
|
except OSError:
|
|
logger.log('Error unable to rename file %s' % base_filepath, logger.ERROR)
|
|
return result
|
|
chunk_set.append(outfile)
|
|
chunk_set.sort()
|
|
else:
|
|
del_dir, del_file = ek.ek(os.path.split, base_filepath)
|
|
if not self._delete_files(del_dir, [del_file], force=True):
|
|
return result
|
|
else:
|
|
if base_filesize == sum(chunk_sizes):
|
|
logger.log('Join skipped. Total size of %s input files equal to output.. %s (%s bytes)' % (
|
|
len(chunk_set), base_filepath, base_filesize))
|
|
else:
|
|
logger.log('Join skipped. Found output file larger than input.. %s (%s bytes)' % (
|
|
base_filepath, base_filesize))
|
|
return result
|
|
|
|
with open(base_filepath, 'ab') as newfile:
|
|
for f in chunk_set:
|
|
logger.log('Joining file %s' % f)
|
|
try:
|
|
with open(f, 'rb') as part:
|
|
for wdata in iter(partial(part.read, 4096), b''):
|
|
try:
|
|
newfile.write(wdata)
|
|
except:
|
|
logger.log('Failed write to file %s' % f)
|
|
return result
|
|
except:
|
|
logger.log('Failed read from file %s' % f)
|
|
return result
|
|
result = base_filepath
|
|
|
|
return result
|
|
|
|
def _already_postprocessed(self, dir_name, videofile, force):
|
|
|
|
if force or not self.any_vid_processed:
|
|
return False
|
|
|
|
# Needed for accessing DB with a unicode dir_name
|
|
if not isinstance(dir_name, unicode):
|
|
dir_name = unicode(dir_name, 'utf_8')
|
|
|
|
parse_result = None
|
|
try:
|
|
parse_result = NameParser(try_scene_exceptions=True, convert=True).parse(videofile, cache_result=False)
|
|
except (InvalidNameException, InvalidShowException):
|
|
# Does not parse, move on to directory check
|
|
pass
|
|
if None is parse_result:
|
|
try:
|
|
parse_result = NameParser(try_scene_exceptions=True,convert=True).parse(dir_name, cache_result=False)
|
|
except (InvalidNameException, InvalidShowException):
|
|
# If the filename doesn't parse, then return false as last
|
|
# resort. We can assume that unparseable filenames are not
|
|
# processed in the past
|
|
return False
|
|
|
|
showlink = ('for "<a href="%s/home/displayShow?show=%s" target="_blank">%s</a>"' % (sickbeard.WEB_ROOT, parse_result.show.indexerid, parse_result.show.name),
|
|
parse_result.show.name)[self.any_vid_processed]
|
|
|
|
ep_detail_sql = ''
|
|
if parse_result.show.indexerid and 0 < len(parse_result.episode_numbers) and parse_result.season_number:
|
|
ep_detail_sql = " and tv_episodes.showid='%s' and tv_episodes.season='%s' and tv_episodes.episode='%s'"\
|
|
% (str(parse_result.show.indexerid),
|
|
str(parse_result.season_number),
|
|
str(parse_result.episode_numbers[0]))
|
|
|
|
# Avoid processing the same directory again if we use a process method <> move
|
|
my_db = db.DBConnection()
|
|
sql_result = my_db.select('SELECT * FROM tv_episodes WHERE release_name = ?', [dir_name])
|
|
if sql_result:
|
|
self._log_helper(u'Found a release directory %s that has already been processed,<br />.. skipping: %s'
|
|
% (showlink, dir_name))
|
|
if ep_detail_sql:
|
|
reset_status(parse_result.show.indexerid,
|
|
parse_result.season_number,
|
|
parse_result.episode_numbers[0])
|
|
return True
|
|
|
|
else:
|
|
# This is needed for video whose name differ from dir_name
|
|
if not isinstance(videofile, unicode):
|
|
videofile = unicode(videofile, 'utf_8')
|
|
|
|
sql_result = my_db.select('SELECT * FROM tv_episodes WHERE release_name = ?', [videofile.rpartition('.')[0]])
|
|
if sql_result:
|
|
self._log_helper(u'Found a video, but that release %s was already processed,<br />.. skipping: %s'
|
|
% (showlink, videofile))
|
|
if ep_detail_sql:
|
|
reset_status(parse_result.show.indexerid,
|
|
parse_result.season_number,
|
|
parse_result.episode_numbers[0])
|
|
return True
|
|
|
|
# Needed if we have downloaded the same episode @ different quality
|
|
search_sql = 'SELECT tv_episodes.indexerid, history.resource FROM tv_episodes INNER JOIN history'\
|
|
+ ' ON history.showid=tv_episodes.showid'\
|
|
+ ' WHERE history.season=tv_episodes.season and history.episode=tv_episodes.episode'\
|
|
+ ep_detail_sql\
|
|
+ ' and tv_episodes.status IN (%s)' % ','.join([str(x) for x in common.Quality.DOWNLOADED])\
|
|
+ ' and history.resource LIKE ?'
|
|
|
|
sql_result = my_db.select(search_sql, [u'%' + videofile])
|
|
if sql_result:
|
|
self._log_helper(u'Found a video, but the episode %s is already processed,<br />.. skipping: %s'
|
|
% (showlink, videofile))
|
|
if ep_detail_sql:
|
|
reset_status(parse_result.show.indexerid,
|
|
parse_result.season_number,
|
|
parse_result.episode_numbers[0])
|
|
return True
|
|
|
|
return False
|
|
|
|
def _process_media(self, process_path, video_files, nzb_name, process_method, force, force_replace, use_trash=False, showObj=None):
|
|
|
|
processor = None
|
|
for cur_video_file in video_files:
|
|
|
|
if self._already_postprocessed(process_path, cur_video_file, force):
|
|
self._set_process_success(False)
|
|
continue
|
|
|
|
cur_video_file_path = ek.ek(os.path.join, process_path, cur_video_file)
|
|
|
|
try:
|
|
processor = postProcessor.PostProcessor(cur_video_file_path, nzb_name, process_method, force_replace, use_trash=use_trash, webhandler=self.webhandler, showObj=showObj)
|
|
file_success = processor.process()
|
|
process_fail_message = ''
|
|
except exceptions.PostProcessingFailed:
|
|
file_success = False
|
|
process_fail_message = '<br />.. Post Processing Failed'
|
|
|
|
self._set_process_success(file_success)
|
|
|
|
if processor:
|
|
self._buffer(processor.log.strip('\n'))
|
|
|
|
if file_success:
|
|
self._log_helper(u'Successfully processed ' + cur_video_file, logger.MESSAGE)
|
|
elif self.any_vid_processed:
|
|
self._log_helper(u'Warning fail for %s%s' % (cur_video_file_path, process_fail_message),
|
|
logger.WARNING)
|
|
else:
|
|
self._log_helper(u'Did not use file %s%s' % (cur_video_file_path, process_fail_message),
|
|
logger.WARNING)
|
|
|
|
@staticmethod
|
|
def _get_path_dir_files(dir_name, nzb_name, pp_type):
|
|
path = ''
|
|
dirs = []
|
|
files = []
|
|
|
|
if dir_name == sickbeard.TV_DOWNLOAD_DIR and not nzb_name or 'manual' == pp_type: # Scheduled Post Processing Active
|
|
# Get at first all the subdir in the dir_name
|
|
for path, dirs, files in ek.ek(os.walk, dir_name):
|
|
break
|
|
else:
|
|
path, dirs = ek.ek(os.path.split, dir_name) # Script Post Processing
|
|
if None is not nzb_name and not nzb_name.endswith('.nzb') and \
|
|
ek.ek(os.path.isfile, ek.ek(os.path.join, dir_name, nzb_name)): # For single torrent file without directory
|
|
dirs = []
|
|
files = [ek.ek(os.path.join, dir_name, nzb_name)]
|
|
else:
|
|
dirs = [dirs]
|
|
files = []
|
|
|
|
return path, dirs, files
|
|
|
|
# noinspection PyArgumentList
|
|
def _process_failed(self, dir_name, nzb_name, showObj=None):
|
|
""" Process a download that did not complete correctly """
|
|
|
|
if sickbeard.USE_FAILED_DOWNLOADS:
|
|
processor = None
|
|
|
|
try:
|
|
processor = failedProcessor.FailedProcessor(dir_name, nzb_name, showObj)
|
|
self._set_process_success(processor.process())
|
|
process_fail_message = ''
|
|
except exceptions.FailedProcessingFailed as e:
|
|
self._set_process_success(False)
|
|
process_fail_message = ex(e)
|
|
|
|
if processor:
|
|
self._buffer(processor.log.strip('\n'))
|
|
|
|
if sickbeard.DELETE_FAILED and self.any_vid_processed:
|
|
self._delete_folder(dir_name, check_empty=False)
|
|
|
|
task = u'Failed download processing'
|
|
if self.any_vid_processed:
|
|
self._log_helper(u'Successful %s: (%s, %s)'
|
|
% (task.lower(), str(nzb_name), dir_name), logger.MESSAGE)
|
|
else:
|
|
self._log_helper(u'%s failed: (%s, %s): %s'
|
|
% (task, str(nzb_name), dir_name, process_fail_message), logger.WARNING)
|
|
|
|
|
|
# backward compatibility prevents the case of this function name from being updated to PEP8
|
|
def processDir(dir_name, nzb_name=None, process_method=None, force=False, force_replace=None, failed=False, type='auto', cleanup=False, webhandler=None, showObj=None):
|
|
# backward compatibility prevents the case of this function name from being updated to PEP8
|
|
return ProcessTVShow(webhandler).process_dir(dir_name, nzb_name, process_method, force, force_replace, failed, type, cleanup, showObj)
|