From 6b418ddea91bb4838b868f195e708b3222f8d52f Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Sun, 2 Oct 2016 03:04:02 +0200 Subject: [PATCH 1/3] Change load cached images (Trakt) on demand. Cleanup cache every day during show update (files older then 30 days). --- CHANGES.md | 3 +- gui/slick/images/trans.png | Bin 0 -> 75 bytes lib/scandir/LICENSE.txt | 27 ++ lib/scandir/__init__.py | 0 lib/scandir/scandir.py | 671 +++++++++++++++++++++++++++++++++++++ sickbeard/helpers.py | 36 ++ sickbeard/show_updater.py | 3 + sickbeard/webserve.py | 31 +- sickbeard/webserveInit.py | 1 + 9 files changed, 763 insertions(+), 9 deletions(-) create mode 100644 gui/slick/images/trans.png create mode 100644 lib/scandir/LICENSE.txt create mode 100644 lib/scandir/__init__.py create mode 100644 lib/scandir/scandir.py diff --git a/CHANGES.md b/CHANGES.md index a5d2a763..8ee490c1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -171,7 +171,7 @@ * Fix status reset of a snatched, downloaded, or archived episode when its date is set to never (no date) on the info source and there is no media file * Change only show unaired episodes on Manage/Backlog Overview and Manage/Episode Status Management where relevant -* Change locally cache "Add from Trakt" show posters, first run takes more time but is faster thereafter +* Change locally cache "Add from Trakt" show posters * Change allow pp to replace files with a repack or proper of same quality * Fix ensure downloaded eps are not shown on episode view * Fix allow propers to pp when show marked upgrade once @@ -196,6 +196,7 @@ * Change add support for freebsd /var/db/zoneinfo when getting local timezone information * Fix issue with post processing propers/repacks * Change use legacy tzlocal() if new gettz fails to create +* Change load cached images (Trakt) on demand ### 0.11.15 (2016-09-13 19:50:00 UTC) diff --git a/gui/slick/images/trans.png b/gui/slick/images/trans.png new file mode 100644 index 0000000000000000000000000000000000000000..4d7beb8065852f68443a92759630f668a48b8c6f GIT binary patch literal 75 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`LY^*;Ar_~T6C_v{Cy4YgaWF8j X2rw|V+OH}C$})Jm`njxgN@xNA_#zJ} literal 0 HcmV?d00001 diff --git a/lib/scandir/LICENSE.txt b/lib/scandir/LICENSE.txt new file mode 100644 index 00000000..0759f503 --- /dev/null +++ b/lib/scandir/LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2012, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of Ben Hoyt nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/scandir/__init__.py b/lib/scandir/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib/scandir/scandir.py b/lib/scandir/scandir.py new file mode 100644 index 00000000..eba65465 --- /dev/null +++ b/lib/scandir/scandir.py @@ -0,0 +1,671 @@ +"""scandir, a better directory iterator and faster os.walk(), now in the Python 3.5 stdlib + +scandir() is a generator version of os.listdir() that returns an +iterator over files in a directory, and also exposes the extra +information most OSes provide while iterating files in a directory +(such as type and stat information). + +This module also includes a version of os.walk() that uses scandir() +to speed it up significantly. + +See README.md or https://github.com/benhoyt/scandir for rationale and +docs, or read PEP 471 (https://www.python.org/dev/peps/pep-0471/) for +more details on its inclusion into Python 3.5 + +scandir is released under the new BSD 3-clause license. See +LICENSE.txt for the full license text. +""" + +from __future__ import division + +from errno import ENOENT +from os import listdir, lstat, stat, strerror +from os.path import join, islink +from stat import S_IFDIR, S_IFLNK, S_IFREG +import collections +import os +import sys + +try: + import _scandir +except ImportError: + _scandir = None + +try: + import ctypes +except ImportError: + ctypes = None + +if _scandir is None and ctypes is None: + import warnings + warnings.warn("scandir can't find the compiled _scandir C module " + "or ctypes, using slow generic fallback") + +__version__ = '1.3' +__all__ = ['scandir', 'walk'] + +# Windows FILE_ATTRIBUTE constants for interpreting the +# FIND_DATA.dwFileAttributes member +FILE_ATTRIBUTE_ARCHIVE = 32 +FILE_ATTRIBUTE_COMPRESSED = 2048 +FILE_ATTRIBUTE_DEVICE = 64 +FILE_ATTRIBUTE_DIRECTORY = 16 +FILE_ATTRIBUTE_ENCRYPTED = 16384 +FILE_ATTRIBUTE_HIDDEN = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 +FILE_ATTRIBUTE_NORMAL = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 +FILE_ATTRIBUTE_OFFLINE = 4096 +FILE_ATTRIBUTE_READONLY = 1 +FILE_ATTRIBUTE_REPARSE_POINT = 1024 +FILE_ATTRIBUTE_SPARSE_FILE = 512 +FILE_ATTRIBUTE_SYSTEM = 4 +FILE_ATTRIBUTE_TEMPORARY = 256 +FILE_ATTRIBUTE_VIRTUAL = 65536 + +IS_PY3 = sys.version_info >= (3, 0) + +if IS_PY3: + unicode = str # Because Python <= 3.2 doesn't have u'unicode' syntax + + +class GenericDirEntry(object): + __slots__ = ('name', '_stat', '_lstat', '_scandir_path', '_path') + + def __init__(self, scandir_path, name): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + self._stat = stat(self.path) + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + def is_dir(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFDIR + + def is_file(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFREG + + def is_symlink(self): + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFLNK + + def inode(self): + st = self.stat(follow_symlinks=False) + return st.st_ino + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + +def _scandir_generic(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + for name in listdir(path): + yield GenericDirEntry(path, name) + + +if IS_PY3 and sys.platform == 'win32': + def scandir_generic(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_generic(path) + scandir_generic.__doc__ = _scandir_generic.__doc__ +else: + scandir_generic = _scandir_generic + + +scandir_c = None +scandir_python = None + + +if sys.platform == 'win32': + if ctypes is not None: + from ctypes import wintypes + + # Various constants from windows.h + INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value + ERROR_FILE_NOT_FOUND = 2 + ERROR_NO_MORE_FILES = 18 + IO_REPARSE_TAG_SYMLINK = 0xA000000C + + # Numer of seconds between 1601-01-01 and 1970-01-01 + SECONDS_BETWEEN_EPOCHS = 11644473600 + + kernel32 = ctypes.windll.kernel32 + + # ctypes wrappers for (wide string versions of) FindFirstFile, + # FindNextFile, and FindClose + FindFirstFile = kernel32.FindFirstFileW + FindFirstFile.argtypes = [ + wintypes.LPCWSTR, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindFirstFile.restype = wintypes.HANDLE + + FindNextFile = kernel32.FindNextFileW + FindNextFile.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindNextFile.restype = wintypes.BOOL + + FindClose = kernel32.FindClose + FindClose.argtypes = [wintypes.HANDLE] + FindClose.restype = wintypes.BOOL + + Win32StatResult = collections.namedtuple('Win32StatResult', [ + 'st_mode', + 'st_ino', + 'st_dev', + 'st_nlink', + 'st_uid', + 'st_gid', + 'st_size', + 'st_atime', + 'st_mtime', + 'st_ctime', + 'st_atime_ns', + 'st_mtime_ns', + 'st_ctime_ns', + 'st_file_attributes', + ]) + + def filetime_to_time(filetime): + """Convert Win32 FILETIME to time since Unix epoch in seconds.""" + total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime + return total / 10000000 - SECONDS_BETWEEN_EPOCHS + + def find_data_to_stat(data): + """Convert Win32 FIND_DATA struct to stat_result.""" + # First convert Win32 dwFileAttributes to st_mode + attributes = data.dwFileAttributes + st_mode = 0 + if attributes & FILE_ATTRIBUTE_DIRECTORY: + st_mode |= S_IFDIR | 0o111 + else: + st_mode |= S_IFREG + if attributes & FILE_ATTRIBUTE_READONLY: + st_mode |= 0o444 + else: + st_mode |= 0o666 + if (attributes & FILE_ATTRIBUTE_REPARSE_POINT and + data.dwReserved0 == IO_REPARSE_TAG_SYMLINK): + st_mode ^= st_mode & 0o170000 + st_mode |= S_IFLNK + + st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow + st_atime = filetime_to_time(data.ftLastAccessTime) + st_mtime = filetime_to_time(data.ftLastWriteTime) + st_ctime = filetime_to_time(data.ftCreationTime) + + # Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, + # st_nlink, st_uid, st_gid + return Win32StatResult(st_mode, 0, 0, 0, 0, 0, st_size, + st_atime, st_mtime, st_ctime, + int(st_atime * 1000000000), + int(st_mtime * 1000000000), + int(st_ctime * 1000000000), + attributes) + + class Win32DirEntryPython(object): + __slots__ = ('name', '_stat', '_lstat', '_find_data', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, find_data): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._find_data = find_data + self._path = None + self._inode = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + # It's a symlink, call link-following stat() + self._stat = stat(self.path) + else: + # Not a symlink, stat is same as lstat value + if self._lstat is None: + self._lstat = find_data_to_stat(self._find_data) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + # Lazily convert to stat object, because it's slow + # in Python, and often we only need is_dir() etc + self._lstat = find_data_to_stat(self._find_data) + return self._lstat + + def is_dir(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFDIR + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY != 0) + + def is_file(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFREG + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY == 0) + + def is_symlink(self): + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT != 0 and + self._find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) + + def inode(self): + if self._inode is None: + self._inode = lstat(self.path).st_ino + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def win_error(error, filename): + exc = WindowsError(error, ctypes.FormatError(error)) + exc.filename = filename + return exc + + def _scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + # Call FindFirstFile and handle errors + if isinstance(path, bytes): + is_bytes = True + filename = join(path.decode('mbcs', 'strict'), '*.*') + else: + is_bytes = False + filename = join(path, '*.*') + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + handle = FindFirstFile(filename, data_p) + if handle == INVALID_HANDLE_VALUE: + error = ctypes.GetLastError() + if error == ERROR_FILE_NOT_FOUND: + # No files, don't yield anything + return + raise win_error(error, path) + + # Call FindNextFile in a loop, stopping when no more files + try: + while True: + # Skip '.' and '..' (current and parent directory), but + # otherwise yield (filename, stat_result) tuple + name = data.cFileName + if name not in ('.', '..'): + if is_bytes: + name = name.encode('mbcs', 'replace') + yield Win32DirEntryPython(path, name, data) + + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + success = FindNextFile(handle, data_p) + if not success: + error = ctypes.GetLastError() + if error == ERROR_NO_MORE_FILES: + break + raise win_error(error, path) + finally: + if not FindClose(handle): + raise win_error(ctypes.GetLastError(), path) + + if IS_PY3: + def scandir_python(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_python(path) + scandir_python.__doc__ = _scandir_python.__doc__ + else: + scandir_python = _scandir_python + + if _scandir is not None: + scandir_c = _scandir.scandir + + if _scandir is not None: + scandir = scandir_c + elif ctypes is not None: + scandir = scandir_python + else: + scandir = scandir_generic + + +# Linux, OS X, and BSD implementation +elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.platform: + have_dirent_d_type = (sys.platform != 'sunos5') + + if ctypes is not None and have_dirent_d_type: + import ctypes.util + + DIR_p = ctypes.c_void_p + + # Rather annoying how the dirent struct is slightly different on each + # platform. The only fields we care about are d_name and d_type. + class Dirent(ctypes.Structure): + if sys.platform.startswith('linux'): + _fields_ = ( + ('d_ino', ctypes.c_ulong), + ('d_off', ctypes.c_long), + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + else: + _fields_ = ( + ('d_ino', ctypes.c_uint32), # must be uint32, not ulong + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_namlen', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + + DT_UNKNOWN = 0 + DT_DIR = 4 + DT_REG = 8 + DT_LNK = 10 + + Dirent_p = ctypes.POINTER(Dirent) + Dirent_pp = ctypes.POINTER(Dirent_p) + + libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) + opendir = libc.opendir + opendir.argtypes = [ctypes.c_char_p] + opendir.restype = DIR_p + + readdir_r = libc.readdir_r + readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] + readdir_r.restype = ctypes.c_int + + closedir = libc.closedir + closedir.argtypes = [DIR_p] + closedir.restype = ctypes.c_int + + file_system_encoding = sys.getfilesystemencoding() + + class PosixDirEntry(object): + __slots__ = ('name', '_d_type', '_stat', '_lstat', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, d_type, inode): + self._scandir_path = scandir_path + self.name = name + self._d_type = d_type + self._inode = inode + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + self._stat = stat(self.path) + else: + if self._lstat is None: + self._lstat = lstat(self.path) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + def is_dir(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFDIR + else: + return self._d_type == DT_DIR + + def is_file(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFREG + else: + return self._d_type == DT_REG + + def is_symlink(self): + if self._d_type == DT_UNKNOWN: + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFLNK + else: + return self._d_type == DT_LNK + + def inode(self): + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def posix_error(filename): + errno = ctypes.get_errno() + exc = OSError(errno, strerror(errno)) + exc.filename = filename + return exc + + def scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + if isinstance(path, bytes): + opendir_path = path + is_bytes = True + else: + opendir_path = path.encode(file_system_encoding) + is_bytes = False + dir_p = opendir(opendir_path) + if not dir_p: + raise posix_error(path) + try: + result = Dirent_p() + while True: + entry = Dirent() + if readdir_r(dir_p, entry, result): + raise posix_error(path) + if not result: + break + name = entry.d_name + if name not in (b'.', b'..'): + if not is_bytes: + name = name.decode(file_system_encoding) + yield PosixDirEntry(path, name, entry.d_type, entry.d_ino) + finally: + if closedir(dir_p): + raise posix_error(path) + + if _scandir is not None: + scandir_c = _scandir.scandir + + if _scandir is not None: + scandir = scandir_c + elif ctypes is not None: + scandir = scandir_python + else: + scandir = scandir_generic + + +# Some other system -- no d_type or stat information +else: + scandir = scandir_generic + + +def _walk(top, topdown=True, onerror=None, followlinks=False): + """Like Python 3.5's implementation of os.walk() -- faster than + the pre-Python 3.5 version as it uses scandir() internally. + """ + dirs = [] + nondirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. os.walk + # always suppressed the exception then, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. That logic is copied here. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + return + + while True: + try: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + return + + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider that the entry is not + # a directory, same behaviour than os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: recurse into sub-directory, but exclude symlinks to + # directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider that the + # entry is not a symbolic link, same behaviour than + # os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + for entry in walk(entry.path, topdown, onerror, followlinks): + yield entry + + # Yield before recursion if going top down + if topdown: + yield top, dirs, nondirs + + # Recurse into sub-directories + for name in dirs: + new_path = join(top, name) + # Issue #23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + for entry in walk(new_path, topdown, onerror, followlinks): + yield entry + else: + # Yield after recursion if going bottom up + yield top, dirs, nondirs + + +if IS_PY3 or sys.platform != 'win32': + walk = _walk +else: + # Fix for broken unicode handling on Windows on Python 2.x, see: + # https://github.com/benhoyt/scandir/issues/54 + file_system_encoding = sys.getfilesystemencoding() + + def walk(top, topdown=True, onerror=None, followlinks=False): + if isinstance(top, bytes): + top = top.decode(file_system_encoding) + return _walk(top, topdown, onerror, followlinks) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index f1eb0fc4..bf6758c5 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -56,6 +56,7 @@ from sickbeard.common import USER_AGENT, mediaExtensions, subtitleExtensions, cp from sickbeard import encodingKludge as ek from lib.cachecontrol import CacheControl, caches +from lib.scandir.scandir import scandir from itertools import izip, cycle @@ -1455,3 +1456,38 @@ def cpu_sleep(): time.sleep(cpu_presets[sickbeard.CPU_PRESET]) +def scantree(path): + """Recursively yield DirEntry objects for given directory.""" + for entry in ek.ek(scandir, path): + if entry.is_dir(follow_symlinks=False): + for entry in scantree(entry.path): + yield entry + else: + yield entry + + +def cleanup_Cache(): + """Delete cached Images older then 30 days""" + basepath = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images') + cachepaths = [ek.ek(os.path.join, basepath, 'trakt')] + del_time = time.mktime((datetime.datetime.now() - datetime.timedelta(days=30)).timetuple()) + for c in cachepaths: + try: + for f in scantree(c): + if f.is_file(follow_symlinks=False) and f.stat(follow_symlinks=False).st_mtime < del_time: + try: + ek.ek(os.remove, f.path) + except: + pass + except: + pass + + +def set_file_timestamp(filename, min_age=3, new_time=None): + min_time = time.mktime((datetime.datetime.now() - datetime.timedelta(days=min_age)).timetuple()) + try: + if ek.ek(os.path.isfile, filename) and ek.ek(os.path.getmtime, filename) < min_time: + ek.ek(os.utime, filename, new_time) + except (StandardError, Exception): + pass + diff --git a/sickbeard/show_updater.py b/sickbeard/show_updater.py index 60375d89..dd854e74 100644 --- a/sickbeard/show_updater.py +++ b/sickbeard/show_updater.py @@ -59,6 +59,9 @@ class ShowUpdater: # clear the data of unused providers sickbeard.helpers.clear_unused_providers() + # cleanup image cache + sickbeard.helpers.cleanup_Cache() + # add missing mapped ids if not sickbeard.background_mapping_task.is_alive(): logger.log(u'Updating the Indexer mappings') diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index cdb66bc8..1127ef97 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -298,6 +298,7 @@ class WebHandler(BaseHandler): self.lock = threading.Lock() def page_not_found(self): + self.set_status(404) t = PageTemplate(headers=self.request.headers, file='404.tmpl') return t.respond() @@ -3067,14 +3068,7 @@ class NewHomeAddShows(Home): img_uri = item.get('show', {}).get('images', {}).get('poster', {}).get('thumb', {}) or '' if img_uri: - path = ek.ek(os.path.abspath, ek.ek( - os.path.join, sickbeard.CACHE_DIR, 'images', 'trakt', 'poster', 'thumb')) - helpers.make_dirs(path) - file_name = ek.ek(os.path.basename, img_uri) - cached_name = ek.ek(os.path.join, path, file_name) - if not ek.ek(os.path.isfile, cached_name): - helpers.download_file(img_uri, cached_name) - images = dict(poster=dict(thumb='cache/images/trakt/poster/thumb/%s' % file_name)) + images = dict(poster=dict(thumb='imagecache?path=trakt/poster/thumb&source=%s' % img_uri)) filtered.append(dict( premiered=dt_ordinal, @@ -5621,3 +5615,24 @@ class Cache(MainHandler): t.cacheResults = sql_results return t.respond() + + +class CachedImages(MainHandler): + def index(self, path='', source=None, *args, **kwargs): + + path = path.strip('/') + file_name = ek.ek(os.path.basename, source) + static_image_path = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images', path, file_name) + static_image_path = ek.ek(os.path.abspath, static_image_path.replace('\\', '/')) + if not ek.ek(os.path.isfile, static_image_path) and source is not None: + basepath = ek.ek(os.path.dirname, static_image_path) + helpers.make_dirs(basepath) + if not helpers.download_file(source, static_image_path) and source.find('trakt.us'): + helpers.download_file(source.replace('trakt.us', 'trakt.tv'), static_image_path) + + if not ek.ek(os.path.isfile, static_image_path): + self.redirect('images/trans.png') + else: + helpers.set_file_timestamp(static_image_path, min_age=3, new_time=None) + self.redirect('cache/images/%s/%s' % (path, file_name)) + diff --git a/sickbeard/webserveInit.py b/sickbeard/webserveInit.py index 19720e19..2a68f258 100644 --- a/sickbeard/webserveInit.py +++ b/sickbeard/webserveInit.py @@ -67,6 +67,7 @@ class WebServer(threading.Thread): self.app.add_handlers('.*$', [ (r'%s/api/builder(/?)(.*)' % self.options['web_root'], webserve.ApiBuilder), (r'%s/api(/?.*)' % self.options['web_root'], webapi.Api), + (r'%s/imagecache(/?.*)' % self.options['web_root'], webserve.CachedImages), (r'%s/cache(/?.*)' % self.options['web_root'], webserve.Cache), (r'%s/config/general(/?.*)' % self.options['web_root'], webserve.ConfigGeneral), (r'%s/config/search(/?.*)' % self.options['web_root'], webserve.ConfigSearch), From 116e0c64db9d2071261678486884c7a44471737f Mon Sep 17 00:00:00 2001 From: JackDandy Date: Mon, 3 Oct 2016 19:31:54 +0100 Subject: [PATCH 2/3] Change factor out cleanup_cache() into cleanup_cache() + delete_not_changed_in() for multiple use cases. Change make it simpler to add cache paths to cleanup_cache() using a list of tuples. Change PEP8 .. cleanup_cache > cleanup_Cache. --- sickbeard/helpers.py | 39 ++++++++++++++++++++++++++++----------- sickbeard/show_updater.py | 2 +- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index bf6758c5..a27d3e46 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1466,21 +1466,39 @@ def scantree(path): yield entry -def cleanup_Cache(): - """Delete cached Images older then 30 days""" - basepath = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images') - cachepaths = [ek.ek(os.path.join, basepath, 'trakt')] - del_time = time.mktime((datetime.datetime.now() - datetime.timedelta(days=30)).timetuple()) - for c in cachepaths: +def cleanup_cache(): + """ + Delete old cached files + """ + delete_not_changed_in([ek.ek(os.path.join, sickbeard.CACHE_DIR, *x) for x in [ + ('images', 'trakt')]]) + + +def delete_not_changed_in(paths, days=30, minutes=0): + """ + Delete files under paths not changed in n days and/or n minutes. + If a file was modified later than days/and or minutes, then don't delete it. + + :param paths: List of paths to scan for files to delete + :param days: Purge files not modified in this number of days (default: 30 days) + :param minutes: Purge files not modified in this number of minutes (default: 0 minutes) + :return: tuple; number of files that qualify for deletion, number of qualifying files that failed to be deleted + """ + del_time = time.mktime((datetime.datetime.now() - datetime.timedelta(days=days, minutes=minutes)).timetuple()) + errors = 0 + qualified = 0 + for c in paths: try: for f in scantree(c): - if f.is_file(follow_symlinks=False) and f.stat(follow_symlinks=False).st_mtime < del_time: + if f.is_file(follow_symlinks=False) and del_time > f.stat(follow_symlinks=False).st_mtime: try: ek.ek(os.remove, f.path) - except: + except (StandardError, Exception): + errors += 1 + qualified += 1 + except (StandardError, Exception): pass - except: - pass + return qualified, errors def set_file_timestamp(filename, min_age=3, new_time=None): @@ -1490,4 +1508,3 @@ def set_file_timestamp(filename, min_age=3, new_time=None): ek.ek(os.utime, filename, new_time) except (StandardError, Exception): pass - diff --git a/sickbeard/show_updater.py b/sickbeard/show_updater.py index dd854e74..cb913e7e 100644 --- a/sickbeard/show_updater.py +++ b/sickbeard/show_updater.py @@ -60,7 +60,7 @@ class ShowUpdater: sickbeard.helpers.clear_unused_providers() # cleanup image cache - sickbeard.helpers.cleanup_Cache() + sickbeard.helpers.cleanup_cache() # add missing mapped ids if not sickbeard.background_mapping_task.is_alive(): From 98a0d474c9e78ab7b7b4119234ebc7e8419e40e5 Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Mon, 3 Oct 2016 22:31:17 +0200 Subject: [PATCH 3/3] Change load cache images (IMDb and AniDB) on demand. --- CHANGES.md | 2 +- sickbeard/helpers.py | 2 +- sickbeard/webserve.py | 19 ++++--------------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8ee490c1..b898e335 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -196,7 +196,7 @@ * Change add support for freebsd /var/db/zoneinfo when getting local timezone information * Fix issue with post processing propers/repacks * Change use legacy tzlocal() if new gettz fails to create -* Change load cached images (Trakt) on demand +* Change load cached images on demand ### 0.11.15 (2016-09-13 19:50:00 UTC) diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index a27d3e46..5bd9e717 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1471,7 +1471,7 @@ def cleanup_cache(): Delete old cached files """ delete_not_changed_in([ek.ek(os.path.join, sickbeard.CACHE_DIR, *x) for x in [ - ('images', 'trakt')]]) + ('images', 'trakt'), ('images', 'imdb'), ('images', 'anidb')]]) def delete_not_changed_in(paths, days=30, minutes=0): diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 1127ef97..dcf52bc8 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -2656,13 +2656,7 @@ class NewHomeAddShows(Home): newest = dt_string img_uri = 'http://img7.anidb.net/pics/anime/%s' % image - path = ek.ek(os.path.abspath, ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images', 'anidb')) - helpers.make_dirs(path) - file_name = ek.ek(os.path.basename, img_uri) - cached_name = ek.ek(os.path.join, path, file_name) - if not ek.ek(os.path.isfile, cached_name): - helpers.download_file(img_uri, cached_name) - images = dict(poster=dict(thumb='cache/images/anidb/%s' % file_name)) + images = dict(poster=dict(thumb='imagecache?path=anidb&source=%s' % img_uri)) votes = rating = 0 counts = anime.find('./ratings/permanent') @@ -2814,6 +2808,7 @@ class NewHomeAddShows(Home): rating = None is not rating and rating.get('content') or '' voting = row.find('meta', attrs={'itemprop': 'ratingCount'}) voting = None is not voting and voting.get('content') or '' + img_uri = None if len(img): img_uri = img[0].get('loadlate') match = img_size.search(img_uri) @@ -2827,13 +2822,7 @@ class NewHomeAddShows(Home): match.group(12)] img_uri = img_uri.replace(match.group(), ''.join( [str(y) for x in map(None, parts, scaled) for y in x if y is not None])) - path = ek.ek(os.path.abspath, ek.ek(os.path.join, sickbeard.CACHE_DIR, 'images', 'imdb')) - helpers.make_dirs(path) - file_name = ek.ek(os.path.basename, img_uri) - cached_name = ek.ek(os.path.join, path, file_name) - if not ek.ek(os.path.isfile, cached_name): - helpers.download_file(img_uri, cached_name) - images = dict(poster=dict(thumb='cache/images/imdb/%s' % file_name)) + images = dict(poster=dict(thumb='imagecache?path=imdb&source=%s' % img_uri)) filtered.append(dict( premiered=dt_ordinal, @@ -2843,7 +2832,7 @@ class NewHomeAddShows(Home): genres=('No genre yet' if not len(genres) else genres[0].get_text().strip().lower().replace(' |', ',')), ids=ids, - images=images, + images='' if not img_uri else images, overview='No overview yet' if not overview else self.encode_html(overview[:250:]), rating=0 if not len(rating) else int(helpers.tryFloat(rating) * 10), title=title.get_text().strip(),