From 3a10574881b49d59aef52bda16721e8d919ad4b2 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Fri, 13 Jan 2023 02:13:40 +0000 Subject: [PATCH] =?UTF-8?q?Update=20diskcache=205.1.0=20(40ce0de)=20?= =?UTF-8?q?=E2=86=92=205.4.0=20(1cb1425).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 + lib/diskcache/__init__.py | 33 +- lib/diskcache/cli.py | 2 +- lib/diskcache/core.py | 565 +++++++++++++++++------------------ lib/diskcache/djangocache.py | 112 ++++--- lib/diskcache/fanout.py | 94 +++--- lib/diskcache/persistent.py | 100 ++----- lib/diskcache/recipes.py | 107 ++++--- 8 files changed, 503 insertions(+), 511 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 47f23e54..f6d787b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * Add Filelock 3.9.0 (ce3e891) * Remove Lockfile no longer used by Cachecontrol * Update Msgpack 1.0.0 (fa7d744) to 1.0.4 (b5acfd5) +* Update diskcache 5.1.0 (40ce0de) to 5.4.0 (1cb1425) * Update Rarfile 4.0 (55fe778) to 4.1a1 (8a72967) * Update UnRar x64 for Windows 6.11 to 6.20 * Update Send2Trash 1.5.0 (66afce7) to 1.8.1b0 (0ef9b32) diff --git a/lib/diskcache/__init__.py b/lib/diskcache/__init__.py index e4d747bf..2355128c 100644 --- a/lib/diskcache/__init__.py +++ b/lib/diskcache/__init__.py @@ -3,17 +3,31 @@ DiskCache API Reference ======================= The :doc:`tutorial` provides a helpful walkthrough of most methods. - """ from .core import ( - Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout + DEFAULT_SETTINGS, + ENOVAL, + EVICTION_POLICY, + UNKNOWN, + Cache, + Disk, + EmptyDirWarning, + JSONDisk, + Timeout, + UnknownFileWarning, ) -from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN from .fanout import FanoutCache from .persistent import Deque, Index -from .recipes import Averager, BoundedSemaphore, Lock, RLock -from .recipes import barrier, memoize_stampede, throttle +from .recipes import ( + Averager, + BoundedSemaphore, + Lock, + RLock, + barrier, + memoize_stampede, + throttle, +) __all__ = [ 'Averager', @@ -40,14 +54,15 @@ __all__ = [ try: from .djangocache import DjangoCache # noqa + __all__.append('DjangoCache') -except Exception: # pylint: disable=broad-except +except Exception: # pylint: disable=broad-except # pragma: no cover # Django not installed or not setup so ignore. pass __title__ = 'diskcache' -__version__ = '5.1.0' -__build__ = 0x050100 +__version__ = '5.4.0' +__build__ = 0x050400 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2020 Grant Jenks' +__copyright__ = 'Copyright 2016-2022 Grant Jenks' diff --git a/lib/diskcache/cli.py b/lib/diskcache/cli.py index 44bffebf..6a39f601 100644 --- a/lib/diskcache/cli.py +++ b/lib/diskcache/cli.py @@ -1 +1 @@ -"Command line interface to disk cache." +"""Command line interface to disk cache.""" diff --git a/lib/diskcache/core.py b/lib/diskcache/core.py index 1405445d..fadd8490 100644 --- a/lib/diskcache/core.py +++ b/lib/diskcache/core.py @@ -1,5 +1,4 @@ """Core disk and file backed cache API. - """ import codecs @@ -23,19 +22,13 @@ import zlib from sg_helpers import touch_file def full_name(func): - "Return full name of `func` by adding the module and function name." + """Return full name of `func` by adding the module and function name.""" return func.__module__ + '.' + func.__qualname__ -try: - WindowsError -except NameError: - class WindowsError(Exception): - "Windows error place-holder on platforms without support." - - class Constant(tuple): - "Pretty display of immutable constant." + """Pretty display of immutable constant.""" + def __new__(cls, name): return tuple.__new__(cls, (name,)) @@ -54,25 +47,25 @@ MODE_TEXT = 3 MODE_PICKLE = 4 DEFAULT_SETTINGS = { - u'statistics': 0, # False - u'tag_index': 0, # False - u'eviction_policy': u'least-recently-stored', - u'size_limit': 2 ** 30, # 1gb - u'cull_limit': 10, - u'sqlite_auto_vacuum': 1, # FULL - u'sqlite_cache_size': 2 ** 13, # 8,192 pages - u'sqlite_journal_mode': u'wal', - u'sqlite_mmap_size': 2 ** 26, # 64mb - u'sqlite_synchronous': 1, # NORMAL - u'disk_min_file_size': 2 ** 15, # 32kb - u'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, + 'statistics': 0, # False + 'tag_index': 0, # False + 'eviction_policy': 'least-recently-stored', + 'size_limit': 2 ** 30, # 1gb + 'cull_limit': 10, + 'sqlite_auto_vacuum': 1, # FULL + 'sqlite_cache_size': 2 ** 13, # 8,192 pages + 'sqlite_journal_mode': 'wal', + 'sqlite_mmap_size': 2 ** 26, # 64mb + 'sqlite_synchronous': 1, # NORMAL + 'disk_min_file_size': 2 ** 15, # 32kb + 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } METADATA = { - u'count': 0, - u'size': 0, - u'hits': 0, - u'misses': 0, + 'count': 0, + 'size': 0, + 'hits': 0, + 'misses': 0, } EVICTION_POLICY = { @@ -108,8 +101,9 @@ EVICTION_POLICY = { } -class Disk(object): - "Cache key and value serialization for SQLite database and files." +class Disk: + """Cache key and value serialization for SQLite database and files.""" + def __init__(self, directory, min_file_size=0, pickle_protocol=0): """Initialize disk instance. @@ -122,7 +116,6 @@ class Disk(object): self.min_file_size = min_file_size self.pickle_protocol = pickle_protocol - def hash(self, key): """Compute portable hash for `key`. @@ -144,7 +137,6 @@ class Disk(object): assert type_disk_key is float return zlib.adler32(struct.pack('!d', disk_key)) & mask - def put(self, key): """Convert `key` to fields key and raw for Cache table. @@ -157,17 +149,20 @@ class Disk(object): if type_key is bytes: return sqlite3.Binary(key), True - elif ((type_key is str) - or (type_key is int - and -9223372036854775808 <= key <= 9223372036854775807) - or (type_key is float)): + elif ( + (type_key is str) + or ( + type_key is int + and -9223372036854775808 <= key <= 9223372036854775807 + ) + or (type_key is float) + ): return key, True else: data = pickle.dumps(key, protocol=self.pickle_protocol) result = pickletools.optimize(data) return sqlite3.Binary(result), False - def get(self, key, raw): """Convert fields `key` and `raw` from Cache table to key. @@ -182,7 +177,6 @@ class Disk(object): else: return pickle.load(io.BytesIO(key)) - def store(self, value, read, key=UNKNOWN): """Convert `value` to fields size, mode, filename, and value for Cache table. @@ -197,39 +191,32 @@ class Disk(object): type_value = type(value) min_file_size = self.min_file_size - if ((type_value is str and len(value) < min_file_size) - or (type_value is int - and -9223372036854775808 <= value <= 9223372036854775807) - or (type_value is float)): + if ( + (type_value is str and len(value) < min_file_size) + or ( + type_value is int + and -9223372036854775808 <= value <= 9223372036854775807 + ) + or (type_value is float) + ): return 0, MODE_RAW, None, value elif type_value is bytes: if len(value) < min_file_size: return 0, MODE_RAW, None, sqlite3.Binary(value) else: filename, full_path = self.filename(key, value) - - with open(full_path, 'wb') as writer: - writer.write(value) - + self._write(full_path, io.BytesIO(value), 'xb') return len(value), MODE_BINARY, filename, None elif type_value is str: filename, full_path = self.filename(key, value) - - with open(full_path, 'w', encoding='UTF-8') as writer: - writer.write(value) - + self._write(full_path, io.StringIO(value), 'x', 'UTF-8') size = op.getsize(full_path) return size, MODE_TEXT, filename, None elif read: - size = 0 reader = ft.partial(value.read, 2 ** 22) filename, full_path = self.filename(key, value) - - with open(full_path, 'wb') as writer: - for chunk in iter(reader, b''): - size += len(chunk) - writer.write(chunk) - + iterator = iter(reader, b'') + size = self._write(full_path, iterator, 'xb') return size, MODE_BINARY, filename, None else: result = pickle.dumps(value, protocol=self.pickle_protocol) @@ -238,12 +225,33 @@ class Disk(object): return 0, MODE_PICKLE, None, sqlite3.Binary(result) else: filename, full_path = self.filename(key, value) - - with open(full_path, 'wb') as writer: - writer.write(result) - + self._write(full_path, io.BytesIO(result), 'xb') return len(result), MODE_PICKLE, filename, None + def _write(self, full_path, iterator, mode, encoding=None): + # pylint: disable=no-self-use + full_dir, _ = op.split(full_path) + + for count in range(1, 11): + with cl.suppress(OSError): + os.makedirs(full_dir) + + try: + # Another cache may have deleted the directory before + # the file could be opened. + writer = open(full_path, mode, encoding=encoding) + except OSError: + if count == 10: + # Give up after 10 tries to open the file. + raise + continue + + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + writer.write(chunk) + return size def fetch(self, mode, filename, value, read): """Convert fields `mode`, `filename`, and `value` from Cache table to @@ -254,9 +262,10 @@ class Disk(object): :param value: database value :param bool read: when True, return an open file handle :return: corresponding Python value + :raises: IOError if the value cannot be read """ - # pylint: disable=no-self-use,unidiomatic-typecheck + # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with if mode == MODE_RAW: return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: @@ -276,7 +285,6 @@ class Disk(object): else: return pickle.load(io.BytesIO(value)) - def filename(self, key=UNKNOWN, value=UNKNOWN): """Return filename and full-path tuple for file storage. @@ -299,43 +307,35 @@ class Disk(object): hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8') sub_dir = op.join(hex_name[:2], hex_name[2:4]) name = hex_name[4:] + '.val' - directory = op.join(self._directory, sub_dir) - - try: - os.makedirs(directory) - except OSError as error: - if error.errno != errno.EEXIST: - raise - filename = op.join(sub_dir, name) full_path = op.join(self._directory, filename) return filename, full_path + def remove(self, file_path): + """Remove a file given by `file_path`. - def remove(self, filename): - """Remove a file given by `filename`. + This method is cross-thread and cross-process safe. If an OSError + occurs, it is suppressed. - This method is cross-thread and cross-process safe. If an "error no - entry" occurs, it is suppressed. - - :param str filename: relative path to file + :param str file_path: relative path to file """ - full_path = op.join(self._directory, filename) + full_path = op.join(self._directory, file_path) + full_dir, _ = op.split(full_path) - try: + # Suppress OSError that may occur if two caches attempt to delete the + # same file or directory at the same time. + + with cl.suppress(OSError): os.remove(full_path) - except WindowsError: - pass - except OSError as error: - if error.errno != errno.ENOENT: - # ENOENT may occur if two caches attempt to delete the same - # file at the same time. - raise + + with cl.suppress(OSError): + os.removedirs(full_dir) class JSONDisk(Disk): - "Cache key and value using JSON serialization with zlib compression." + """Cache key and value using JSON serialization with zlib compression.""" + def __init__(self, directory, compress_level=1, **kwargs): """Initialize JSON disk instance. @@ -352,25 +352,21 @@ class JSONDisk(Disk): self.compress_level = compress_level super().__init__(directory, **kwargs) - def put(self, key): json_bytes = json.dumps(key).encode('utf-8') data = zlib.compress(json_bytes, self.compress_level) return super().put(data) - def get(self, key, raw): data = super().get(key, raw) return json.loads(zlib.decompress(data).decode('utf-8')) - def store(self, value, read, key=UNKNOWN): if not read: json_bytes = json.dumps(value).encode('utf-8') value = zlib.compress(json_bytes, self.compress_level) return super().store(value, read, key=key) - def fetch(self, mode, filename, value, read): data = super().fetch(mode, filename, value, read) if not read: @@ -379,31 +375,33 @@ class JSONDisk(Disk): class Timeout(Exception): - "Database timeout expired." + """Database timeout expired.""" class UnknownFileWarning(UserWarning): - "Warning used by Cache.check for unknown files." + """Warning used by Cache.check for unknown files.""" class EmptyDirWarning(UserWarning): - "Warning used by Cache.check for empty directories." + """Warning used by Cache.check for empty directories.""" -def args_to_key(base, args, kwargs, typed): +def args_to_key(base, args, kwargs, typed, ignore): """Create cache key out of function arguments. :param tuple base: base of key :param tuple args: function arguments :param dict kwargs: function keyword arguments :param bool typed: include types in cache key + :param set ignore: positional or keyword args to ignore :return: cache key tuple """ - key = base + args + args = tuple(arg for index, arg in enumerate(args) if index not in ignore) + key = base + args + (None,) if kwargs: - key += (ENOVAL,) + kwargs = {key: val for key, val in kwargs.items() if key not in ignore} sorted_items = sorted(kwargs.items()) for item in sorted_items: @@ -418,8 +416,9 @@ def args_to_key(base, args, kwargs, typed): return key -class Cache(object): - "Disk and file backed cache." +class Cache: + """Disk and file backed cache.""" + def __init__(self, directory=None, timeout=60, disk=Disk, **settings): """Initialize cache instance. @@ -452,7 +451,7 @@ class Cache(object): raise EnvironmentError( error.errno, 'Cache directory "%s" does not exist' - ' and could not be created' % self._directory + ' and could not be created' % self._directory, ) from None sql = self._sql_retry @@ -460,9 +459,9 @@ class Cache(object): # Setup Settings table. try: - current_settings = dict(sql( - 'SELECT key, value FROM Settings' - ).fetchall()) + current_settings = dict( + sql('SELECT key, value FROM Settings').fetchall() + ) except sqlite3.OperationalError: current_settings = {} @@ -479,7 +478,8 @@ class Cache(object): if key.startswith('sqlite_'): self.reset(key, value, update=False) - sql('CREATE TABLE IF NOT EXISTS Settings (' + sql( + 'CREATE TABLE IF NOT EXISTS Settings (' ' key TEXT NOT NULL UNIQUE,' ' value)' ) @@ -487,7 +487,8 @@ class Cache(object): # Setup Disk object (must happen after settings initialized). kwargs = { - key[5:]: value for key, value in sets.items() + key[5:]: value + for key, value in sets.items() if key.startswith('disk_') } self._disk = disk(directory, **kwargs) @@ -504,11 +505,12 @@ class Cache(object): sql(query, (key, value)) self.reset(key) - (self._page_size,), = sql('PRAGMA page_size').fetchall() + ((self._page_size,),) = sql('PRAGMA page_size').fetchall() # Setup Cache table. - sql('CREATE TABLE IF NOT EXISTS Cache (' + sql( + 'CREATE TABLE IF NOT EXISTS Cache (' ' rowid INTEGER PRIMARY KEY,' ' key BLOB,' ' raw INTEGER,' @@ -523,11 +525,13 @@ class Cache(object): ' value BLOB)' ) - sql('CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON' + sql( + 'CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON' ' Cache(key, raw)' ) - sql('CREATE INDEX IF NOT EXISTS Cache_expire_time ON' + sql( + 'CREATE INDEX IF NOT EXISTS Cache_expire_time ON' ' Cache (expire_time)' ) @@ -538,32 +542,37 @@ class Cache(object): # Use triggers to keep Metadata updated. - sql('CREATE TRIGGER IF NOT EXISTS Settings_count_insert' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_insert' ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value + 1' ' WHERE key = "count"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_count_delete' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_delete' ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value - 1' ' WHERE key = "count"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_insert' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_insert' ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value + NEW.size' ' WHERE key = "size"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_update' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_update' ' AFTER UPDATE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings' ' SET value = value + NEW.size - OLD.size' ' WHERE key = "size"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_delete' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_delete' ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value - OLD.size' ' WHERE key = "size"; END' @@ -582,25 +591,21 @@ class Cache(object): self._timeout = timeout self._sql # pylint: disable=pointless-statement - @property def directory(self): """Cache directory.""" return self._directory - @property def timeout(self): """SQLite connection timeout value in seconds.""" return self._timeout - @property def disk(self): """Disk used for serialization.""" return self._disk - @property def _con(self): # Check process ID to support process forking. If the process @@ -619,7 +624,8 @@ class Cache(object): touch_file(DBNAME, dir_name=self._directory) con = self._local.con = sqlite3.connect( op.join(self._directory, DBNAME), - timeout=self._timeout + timeout=self._timeout, + isolation_level=None, ) # Some SQLite pragmas work on a per-connection basis so @@ -639,20 +645,9 @@ class Cache(object): return con - - def _execute(self, *args, **kwargs): - result = self._con.execute(*args, **kwargs) - try: - self._con.commit() - except (BaseException, Exception): - pass - return result - - @property def _sql(self): - return self._execute - + return self._con.execute @property def _sql_retry(self): @@ -677,11 +672,10 @@ class Cache(object): diff = time.time() - start if diff > 60: raise - time.sleep(1) + time.sleep(0.001) return _execute_with_retry - @cl.contextmanager def transact(self, retry=False): """Context manager to perform a transaction by locking the cache. @@ -713,7 +707,6 @@ class Cache(object): with self._transact(retry=retry): yield - @cl.contextmanager def _transact(self, retry=False, filename=None): sql = self._sql @@ -727,7 +720,7 @@ class Cache(object): else: while True: try: - # sql('BEGIN IMMEDIATE', no_commit_call=True) + sql('BEGIN IMMEDIATE') begin = True self._txn_id = tid break @@ -744,20 +737,17 @@ class Cache(object): if begin: assert self._txn_id == tid self._txn_id = None - # sql('ROLLBACK') - self._con.rollback() + sql('ROLLBACK') raise else: if begin: assert self._txn_id == tid self._txn_id = None - # sql('COMMIT') - self._con.commit() + sql('COMMIT') for name in filenames: if name is not None: _disk_remove(name) - def set(self, key, value, expire=None, read=False, tag=None, retry=False): """Set `key` and `value` item in cache. @@ -813,7 +803,7 @@ class Cache(object): ).fetchall() if rows: - (rowid, old_filename), = rows + ((rowid, old_filename),) = rows cleanup(old_filename) self._row_update(rowid, now, columns) else: @@ -823,7 +813,6 @@ class Cache(object): return True - def __setitem__(self, key, value): """Set corresponding `value` for `key` in cache. @@ -835,11 +824,11 @@ class Cache(object): """ self.set(key, value, retry=True) - def _row_update(self, rowid, now, columns): sql = self._sql expire_time, tag, size, mode, filename, value = columns - sql('UPDATE Cache SET' + sql( + 'UPDATE Cache SET' ' store_time = ?,' ' expire_time = ?,' ' access_time = ?,' @@ -849,11 +838,12 @@ class Cache(object): ' mode = ?,' ' filename = ?,' ' value = ?' - ' WHERE rowid = ?', ( - now, # store_time + ' WHERE rowid = ?', + ( + now, # store_time expire_time, - now, # access_time - 0, # access_count + now, # access_time + 0, # access_count tag, size, mode, @@ -863,20 +853,21 @@ class Cache(object): ), ) - def _row_insert(self, key, raw, now, columns): sql = self._sql expire_time, tag, size, mode, filename, value = columns - sql('INSERT INTO Cache(' + sql( + 'INSERT INTO Cache(' ' key, raw, store_time, expire_time, access_time,' ' access_count, tag, size, mode, filename, value' - ') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', ( + ') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + ( key, raw, - now, # store_time + now, # store_time expire_time, - now, # access_time - 0, # access_count + now, # access_time + 0, # access_count tag, size, mode, @@ -885,7 +876,6 @@ class Cache(object): ), ) - def _cull(self, now, sql, cleanup, limit=None): cull_limit = self.cull_limit if limit is None else limit @@ -904,13 +894,12 @@ class Cache(object): rows = sql(select_expired, (now, cull_limit)).fetchall() if rows: - delete_expired = ( - 'DELETE FROM Cache WHERE rowid IN (%s)' - % (select_expired_template % 'rowid') + delete_expired = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_expired_template % 'rowid' ) sql(delete_expired, (now, cull_limit)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) cull_limit -= len(rows) @@ -929,16 +918,14 @@ class Cache(object): rows = sql(select_filename, (cull_limit,)).fetchall() if rows: - delete = ( - 'DELETE FROM Cache WHERE rowid IN (%s)' - % (select_policy.format(fields='rowid', now=now)) + delete = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_policy.format(fields='rowid', now=now) ) sql(delete, (cull_limit,)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) - def touch(self, key, expire=None, retry=False): """Touch `key` in cache and update `expire` time. @@ -965,17 +952,17 @@ class Cache(object): ).fetchall() if rows: - (rowid, old_expire_time), = rows + ((rowid, old_expire_time),) = rows if old_expire_time is None or old_expire_time > now: - sql('UPDATE Cache SET expire_time = ? WHERE rowid = ?', + sql( + 'UPDATE Cache SET expire_time = ? WHERE rowid = ?', (expire_time, rowid), ) return True return False - def add(self, key, value, expire=None, read=False, tag=None, retry=False): """Add `key` and `value` item to cache. @@ -1015,7 +1002,7 @@ class Cache(object): ).fetchall() if rows: - (rowid, old_filename, old_expire_time), = rows + ((rowid, old_filename, old_expire_time),) = rows if old_expire_time is None or old_expire_time > now: cleanup(filename) @@ -1030,7 +1017,6 @@ class Cache(object): return True - def incr(self, key, delta=1, default=0, retry=False): """Increment value by delta for item with key. @@ -1071,22 +1057,22 @@ class Cache(object): raise KeyError(key) value = default + delta - columns = ( - (None, None) + self._disk.store(value, False, key=key) + columns = (None, None) + self._disk.store( + value, False, key=key ) self._row_insert(db_key, raw, now, columns) self._cull(now, sql, cleanup) return value - (rowid, expire_time, filename, value), = rows + ((rowid, expire_time, filename, value),) = rows if expire_time is not None and expire_time < now: if default is None: raise KeyError(key) value = default + delta - columns = ( - (None, None) + self._disk.store(value, False, key=key) + columns = (None, None) + self._disk.store( + value, False, key=key ) self._row_update(rowid, now, columns) self._cull(now, sql, cleanup) @@ -1106,7 +1092,6 @@ class Cache(object): return value - def decr(self, key, delta=1, default=0, retry=False): """Decrement value by delta for item with key. @@ -1137,9 +1122,15 @@ class Cache(object): """ return self.incr(key, -delta, default, retry) - - def get(self, key, default=None, read=False, expire_time=False, tag=False, - retry=False): + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Retrieve value from cache. If `key` is missing, return `default`. Raises :exc:`Timeout` error when database timeout occurs and `retry` is @@ -1178,7 +1169,7 @@ class Cache(object): if not rows: return default - (rowid, db_expire_time, db_tag, mode, filename, db_value), = rows + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1202,19 +1193,17 @@ class Cache(object): sql(cache_miss) return default - (rowid, db_expire_time, db_tag, - mode, filename, db_value), = rows # noqa: E127 + ( + (rowid, db_expire_time, db_tag, mode, filename, db_value), + ) = rows # noqa: E127 try: value = self._disk.fetch(mode, filename, db_value, read) - except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - if self.statistics: - sql(cache_miss) - return default - else: - raise + except IOError: + # Key was deleted before we could retrieve result. + if self.statistics: + sql(cache_miss) + return default if self.statistics: sql(cache_hit) @@ -1234,7 +1223,6 @@ class Cache(object): else: return value - def __getitem__(self, key): """Return corresponding value for `key` from cache. @@ -1248,7 +1236,6 @@ class Cache(object): raise KeyError(key) return value - def read(self, key, retry=False): """Return file handle value corresponding to `key` from cache. @@ -1267,7 +1254,6 @@ class Cache(object): raise KeyError(key) return handle - def __contains__(self, key): """Return `True` if `key` matching item is found in cache. @@ -1287,8 +1273,9 @@ class Cache(object): return bool(rows) - - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -1326,18 +1313,15 @@ class Cache(object): if not rows: return default - (rowid, db_expire_time, db_tag, mode, filename, db_value), = rows + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) try: value = self._disk.fetch(mode, filename, db_value, False) - except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - return default - else: - raise + except IOError: + # Key was deleted before we could retrieve result. + return default finally: if filename is not None: self._disk.remove(filename) @@ -1351,7 +1335,6 @@ class Cache(object): else: return value - def __delitem__(self, key, retry=True): """Delete corresponding item for `key` from cache. @@ -1377,13 +1360,12 @@ class Cache(object): if not rows: raise KeyError(key) - (rowid, filename), = rows + ((rowid, filename),) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) cleanup(filename) return True - def delete(self, key, retry=False): """Delete corresponding item for `key` from cache. @@ -1403,9 +1385,16 @@ class Cache(object): except KeyError: return False - - def push(self, value, prefix=None, side='back', expire=None, read=False, - tag=None, retry=False): + def push( + self, + value, + prefix=None, + side='back', + expire=None, + read=False, + tag=None, + retry=False, + ): """Push `value` onto `side` of queue identified by `prefix` in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1471,10 +1460,10 @@ class Cache(object): rows = sql(select, (min_key, max_key, raw)).fetchall() if rows: - (key,), = rows + ((key,),) = rows if prefix is not None: - num = int(key[(key.rfind('-') + 1):]) + num = int(key[(key.rfind('-') + 1) :]) else: num = key @@ -1496,9 +1485,15 @@ class Cache(object): return db_key - - def pull(self, prefix=None, default=(None, None), side='front', - expire_time=False, tag=False, retry=False): + def pull( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): """Pull key and value item pair from `side` of queue in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1579,8 +1574,9 @@ class Cache(object): if not rows: return default - (rowid, key, db_expire, db_tag, mode, name, - db_value), = rows + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1591,11 +1587,9 @@ class Cache(object): try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + except IOError: + # Key was deleted before we could retrieve result. + continue finally: if name is not None: self._disk.remove(name) @@ -1610,9 +1604,15 @@ class Cache(object): else: return key, value - - def peek(self, prefix=None, default=(None, None), side='front', - expire_time=False, tag=False, retry=False): + def peek( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): """Peek at key and value item pair from `side` of queue in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1689,8 +1689,9 @@ class Cache(object): if not rows: return default - (rowid, key, db_expire, db_tag, mode, name, - db_value), = rows + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows if db_expire is not None and db_expire < time.time(): sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1700,11 +1701,9 @@ class Cache(object): try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + except IOError: + # Key was deleted before we could retrieve result. + continue finally: if name is not None: self._disk.remove(name) @@ -1719,7 +1718,6 @@ class Cache(object): else: return key, value - def peekitem(self, last=True, expire_time=False, tag=False, retry=False): """Peek at key and value item pair in cache based on iteration order. @@ -1761,8 +1759,18 @@ class Cache(object): if not rows: raise KeyError('dictionary is empty') - (rowid, db_key, raw, db_expire, db_tag, mode, name, - db_value), = rows + ( + ( + rowid, + db_key, + raw, + db_expire, + db_tag, + mode, + name, + db_value, + ), + ) = rows if db_expire is not None and db_expire < time.time(): sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1774,11 +1782,9 @@ class Cache(object): try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + except IOError: + # Key was deleted before we could retrieve result. + continue break if expire_time and tag: @@ -1790,8 +1796,9 @@ class Cache(object): else: return key, value - - def memoize(self, name=None, typed=False, expire=None, tag=None): + def memoize( + self, name=None, typed=False, expire=None, tag=None, ignore=() + ): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -1850,6 +1857,7 @@ class Cache(object): :param float expire: seconds until arguments expire (default None, no expiry) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ @@ -1858,12 +1866,12 @@ class Cache(object): raise TypeError('name cannot be callable') def decorator(func): - "Decorator created by memoize() for callable `func`." + """Decorator created by memoize() for callable `func`.""" base = (full_name(func),) if name is None else (name,) @ft.wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) result = self.get(key, default=ENOVAL, retry=True) @@ -1875,15 +1883,14 @@ class Cache(object): return result def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." - return args_to_key(base, args, kwargs, typed) + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper return decorator - def check(self, fix=False, retry=False): """Check database and file system consistency. @@ -1912,8 +1919,8 @@ class Cache(object): rows = sql('PRAGMA integrity_check').fetchall() - if len(rows) != 1 or rows[0][0] != u'ok': - for message, in rows: + if len(rows) != 1 or rows[0][0] != 'ok': + for (message,) in rows: warnings.warn(message) if fix: @@ -1944,7 +1951,8 @@ class Cache(object): warnings.warn(message % args) if fix: - sql('UPDATE Cache SET size = ?' + sql( + 'UPDATE Cache SET size = ?' ' WHERE rowid = ?', (real_size, rowid), ) @@ -1985,14 +1993,15 @@ class Cache(object): # Check Settings.count against count of Cache rows. self.reset('count') - (count,), = sql('SELECT COUNT(key) FROM Cache').fetchall() + ((count,),) = sql('SELECT COUNT(key) FROM Cache').fetchall() if self.count != count: message = 'Settings.count != COUNT(Cache.key); %d != %d' warnings.warn(message % (self.count, count)) if fix: - sql('UPDATE Settings SET value = ? WHERE key = ?', + sql( + 'UPDATE Settings SET value = ? WHERE key = ?', (count, 'count'), ) @@ -2000,20 +2009,20 @@ class Cache(object): self.reset('size') select_size = 'SELECT COALESCE(SUM(size), 0) FROM Cache' - (size,), = sql(select_size).fetchall() + ((size,),) = sql(select_size).fetchall() if self.size != size: message = 'Settings.size != SUM(Cache.size); %d != %d' warnings.warn(message % (self.size, size)) if fix: - sql('UPDATE Settings SET value = ? WHERE key =?', + sql( + 'UPDATE Settings SET value = ? WHERE key =?', (size, 'size'), ) return warns - def create_tag_index(self): """Create tag index on cache database. @@ -2026,7 +2035,6 @@ class Cache(object): sql('CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid)') self.reset('tag_index', 1) - def drop_tag_index(self): """Drop tag index on cache database. @@ -2037,7 +2045,6 @@ class Cache(object): sql('DROP INDEX IF EXISTS Cache_tag_rowid') self.reset('tag_index', 0) - def evict(self, tag, retry=False): """Remove items with matching `tag` from cache. @@ -2065,7 +2072,6 @@ class Cache(object): args = [tag, 0, 100] return self._select_delete(select, args, arg_index=1, retry=retry) - def expire(self, now=None, retry=False): """Remove expired items from cache. @@ -2093,7 +2099,6 @@ class Cache(object): args = [0, now or time.time(), 100] return self._select_delete(select, args, row_index=1, retry=retry) - def cull(self, retry=False): """Cull items from cache until volume is less than size limit. @@ -2123,7 +2128,7 @@ class Cache(object): select_policy = EVICTION_POLICY[self.eviction_policy]['cull'] if select_policy is None: - return + return 0 select_filename = select_policy.format(fields='filename', now=now) @@ -2142,14 +2147,13 @@ class Cache(object): ) sql(delete, (10,)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) except Timeout: raise Timeout(count) from None return count - def clear(self, retry=False): """Remove all items from cache. @@ -2176,9 +2180,9 @@ class Cache(object): args = [0, 100] return self._select_delete(select, args, retry=retry) - - def _select_delete(self, select, args, row_index=0, arg_index=0, - retry=False): + def _select_delete( + self, select, args, row_index=0, arg_index=0, retry=False + ): count = 0 delete = 'DELETE FROM Cache WHERE rowid IN (%s)' @@ -2202,7 +2206,6 @@ class Cache(object): return count - def iterkeys(self, reverse=False): """Iterate Cache keys in database sort order. @@ -2246,7 +2249,7 @@ class Cache(object): row = sql(select).fetchall() if row: - (key, raw), = row + ((key, raw),) = row else: return @@ -2261,11 +2264,10 @@ class Cache(object): for key, raw in rows: yield _disk_get(key, raw) - def _iter(self, ascending=True): sql = self._sql rows = sql('SELECT MAX(rowid) FROM Cache').fetchall() - (max_rowid,), = rows + ((max_rowid,),) = rows yield # Signal ready. if max_rowid is None: @@ -2295,21 +2297,18 @@ class Cache(object): for rowid, key, raw in rows: yield _disk_get(key, raw) - def __iter__(self): - "Iterate keys in cache including expired items." + """Iterate keys in cache including expired items.""" iterator = self._iter() next(iterator) return iterator - def __reversed__(self): - "Reverse iterate keys in cache including expired items." + """Reverse iterate keys in cache including expired items.""" iterator = self._iter(ascending=False) next(iterator) return iterator - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -2329,31 +2328,23 @@ class Cache(object): return result - def volume(self): """Return estimated total size of cache on disk. :return: size in bytes """ - (page_count,), = self._sql('PRAGMA page_count').fetchall() + ((page_count,),) = self._sql('PRAGMA page_count').fetchall() total_size = self._page_size * page_count + self.reset('size') return total_size - def close(self): - """Close database connection. - - """ + """Close database connection.""" con = getattr(self._local, 'con', None) if con is None: return - try: - con.commit() - except (BaseException, Exception): - pass con.close() try: @@ -2361,31 +2352,25 @@ class Cache(object): except AttributeError: pass - def __enter__(self): # Create connection in thread. # pylint: disable=unused-variable connection = self._con # noqa return self - def __exit__(self, *exception): self.close() - def __len__(self): - "Count of items in cache including expired items." + """Count of items in cache including expired items.""" return self.reset('count') - def __getstate__(self): return (self.directory, self.timeout, type(self.disk)) - def __setstate__(self, state): self.__init__(*state) - def reset(self, key, value=ENOVAL, update=True): """Reset `key` and `value` item from Settings table. @@ -2419,7 +2404,7 @@ class Cache(object): if value is ENOVAL: select = 'SELECT value FROM Settings WHERE key = ?' - (value,), = sql_retry(select, (key,)).fetchall() + ((value,),) = sql_retry(select, (key,)).fetchall() setattr(self, key, value) return value @@ -2447,7 +2432,9 @@ class Cache(object): while True: try: try: - (old_value,), = sql('PRAGMA %s' % (pragma)).fetchall() + ((old_value,),) = sql( + 'PRAGMA %s' % (pragma) + ).fetchall() update = old_value != value except ValueError: update = True @@ -2460,7 +2447,7 @@ class Cache(object): diff = time.time() - start if diff > 60: raise - time.sleep(1) + time.sleep(0.001) elif key.startswith('disk_'): attr = key[5:] setattr(self._disk, attr, value) diff --git a/lib/diskcache/djangocache.py b/lib/diskcache/djangocache.py index 329b9662..8bf85ce6 100644 --- a/lib/diskcache/djangocache.py +++ b/lib/diskcache/djangocache.py @@ -1,11 +1,12 @@ -"Django-compatible disk and file backed cache." +"""Django-compatible disk and file backed cache.""" from functools import wraps + from django.core.cache.backends.base import BaseCache try: from django.core.cache.backends.base import DEFAULT_TIMEOUT -except ImportError: +except ImportError: # pragma: no cover # For older versions of Django simply use 300 seconds. DEFAULT_TIMEOUT = 300 @@ -14,7 +15,8 @@ from .fanout import FanoutCache class DjangoCache(BaseCache): - "Django-compatible disk and file backed cache." + """Django-compatible disk and file backed cache.""" + def __init__(self, directory, params): """Initialize DjangoCache instance. @@ -28,13 +30,11 @@ class DjangoCache(BaseCache): options = params.get('OPTIONS', {}) self._cache = FanoutCache(directory, shards, timeout, **options) - @property def directory(self): """Cache directory.""" return self._cache.directory - def cache(self, name): """Return Cache with given `name` in subdirectory. @@ -44,7 +44,6 @@ class DjangoCache(BaseCache): """ return self._cache.cache(name) - def deque(self, name): """Return Deque with given `name` in subdirectory. @@ -54,7 +53,6 @@ class DjangoCache(BaseCache): """ return self._cache.deque(name) - def index(self, name): """Return Index with given `name` in subdirectory. @@ -64,9 +62,16 @@ class DjangoCache(BaseCache): """ return self._cache.index(name) - - def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, - read=False, tag=None, retry=True): + def add( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): """Set a value in the cache if the key does not already exist. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -89,9 +94,16 @@ class DjangoCache(BaseCache): timeout = self.get_backend_timeout(timeout=timeout) return self._cache.add(key, value, timeout, read, tag, retry) - - def get(self, key, default=None, version=None, read=False, - expire_time=False, tag=False, retry=False): + def get( + self, + key, + default=None, + version=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Fetch a given key from the cache. If the key does not exist, return default, which itself defaults to None. @@ -111,7 +123,6 @@ class DjangoCache(BaseCache): key = self.make_key(key, version=version) return self._cache.get(key, default, read, expire_time, tag, retry) - def read(self, key, version=None): """Return file handle corresponding to `key` from Cache. @@ -124,9 +135,16 @@ class DjangoCache(BaseCache): key = self.make_key(key, version=version) return self._cache.read(key) - - def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, - read=False, tag=None, retry=True): + def set( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): """Set a value in the cache. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -146,7 +164,6 @@ class DjangoCache(BaseCache): timeout = self.get_backend_timeout(timeout=timeout) return self._cache.set(key, value, timeout, read, tag, retry) - def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True): """Touch a key in the cache. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -164,9 +181,15 @@ class DjangoCache(BaseCache): timeout = self.get_backend_timeout(timeout=timeout) return self._cache.touch(key, timeout, retry) - - def pop(self, key, default=None, version=None, expire_time=False, - tag=False, retry=True): + def pop( + self, + key, + default=None, + version=None, + expire_time=False, + tag=False, + retry=True, + ): """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -186,7 +209,6 @@ class DjangoCache(BaseCache): key = self.make_key(key, version=version) return self._cache.pop(key, default, expire_time, tag, retry) - def delete(self, key, version=None, retry=True): """Delete a key from the cache, failing silently. @@ -198,8 +220,7 @@ class DjangoCache(BaseCache): """ # pylint: disable=arguments-differ key = self.make_key(key, version=version) - self._cache.delete(key, retry) - + return self._cache.delete(key, retry) def incr(self, key, delta=1, version=None, default=None, retry=True): """Increment value by delta for item with key. @@ -230,7 +251,6 @@ class DjangoCache(BaseCache): except KeyError: raise ValueError("Key '%s' not found" % key) from None - def decr(self, key, delta=1, version=None, default=None, retry=True): """Decrement value by delta for item with key. @@ -259,7 +279,6 @@ class DjangoCache(BaseCache): # pylint: disable=arguments-differ return self.incr(key, -delta, version, default, retry) - def has_key(self, key, version=None): """Returns True if the key is in the cache and has not expired. @@ -271,7 +290,6 @@ class DjangoCache(BaseCache): key = self.make_key(key, version=version) return key in self._cache - def expire(self): """Remove expired items from cache. @@ -280,7 +298,6 @@ class DjangoCache(BaseCache): """ return self._cache.expire() - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -291,7 +308,6 @@ class DjangoCache(BaseCache): """ return self._cache.stats(enable=enable, reset=reset) - def create_tag_index(self): """Create tag index on cache database. @@ -302,7 +318,6 @@ class DjangoCache(BaseCache): """ self._cache.create_tag_index() - def drop_tag_index(self): """Drop tag index on cache database. @@ -311,7 +326,6 @@ class DjangoCache(BaseCache): """ self._cache.drop_tag_index() - def evict(self, tag): """Remove items with matching `tag` from cache. @@ -321,7 +335,6 @@ class DjangoCache(BaseCache): """ return self._cache.evict(tag) - def cull(self): """Cull items from cache until volume is less than size limit. @@ -330,18 +343,15 @@ class DjangoCache(BaseCache): """ return self._cache.cull() - def clear(self): - "Remove *all* values from the cache at once." + """Remove *all* values from the cache at once.""" return self._cache.clear() - def close(self, **kwargs): - "Close the cache connection." + """Close the cache connection.""" # pylint: disable=unused-argument self._cache.close() - def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): """Return seconds to expiration. @@ -356,9 +366,15 @@ class DjangoCache(BaseCache): timeout = -1 return None if timeout is None else timeout - - def memoize(self, name=None, timeout=DEFAULT_TIMEOUT, version=None, - typed=False, tag=None): + def memoize( + self, + name=None, + timeout=DEFAULT_TIMEOUT, + version=None, + typed=False, + tag=None, + ignore=(), + ): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -392,6 +408,7 @@ class DjangoCache(BaseCache): :param int version: key version number (default None, cache parameter) :param bool typed: cache different types separately (default False) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ @@ -400,12 +417,12 @@ class DjangoCache(BaseCache): raise TypeError('name cannot be callable') def decorator(func): - "Decorator created by memoize() for callable `func`." + """Decorator created by memoize() for callable `func`.""" base = (full_name(func),) if name is None else (name,) @wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) result = self.get(key, ENOVAL, version, retry=True) @@ -418,14 +435,19 @@ class DjangoCache(BaseCache): ) if valid_timeout: self.set( - key, result, timeout, version, tag=tag, retry=True, + key, + result, + timeout, + version, + tag=tag, + retry=True, ) return result def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." - return args_to_key(base, args, kwargs, typed) + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper diff --git a/lib/diskcache/fanout.py b/lib/diskcache/fanout.py index 7e227c8d..dc5240c5 100644 --- a/lib/diskcache/fanout.py +++ b/lib/diskcache/fanout.py @@ -1,4 +1,4 @@ -"Fanout cache automatically shards keys and values." +"""Fanout cache automatically shards keys and values.""" import contextlib as cl import functools @@ -9,14 +9,16 @@ import sqlite3 import tempfile import time -from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout +from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout from .persistent import Deque, Index -class FanoutCache(object): - "Cache that shards keys and values." - def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk, - **settings): +class FanoutCache: + """Cache that shards keys and values.""" + + def __init__( + self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings + ): """Initialize cache instance. :param str directory: cache directory @@ -36,6 +38,7 @@ class FanoutCache(object): self._count = shards self._directory = directory + self._disk = disk self._shards = tuple( Cache( directory=op.join(directory, '%03d' % num), @@ -51,20 +54,17 @@ class FanoutCache(object): self._deques = {} self._indexes = {} - @property def directory(self): """Cache directory.""" return self._directory - def __getattr__(self, name): safe_names = {'timeout', 'disk'} valid_name = name in DEFAULT_SETTINGS or name in safe_names assert valid_name, 'cannot access {} in cache shard'.format(name) return getattr(self._shards[0], name) - @cl.contextmanager def transact(self, retry=True): """Context manager to perform a transaction by locking the cache. @@ -98,7 +98,6 @@ class FanoutCache(object): stack.enter_context(shard_transaction) yield - def set(self, key, value, expire=None, read=False, tag=None, retry=False): """Set `key` and `value` item in cache. @@ -125,7 +124,6 @@ class FanoutCache(object): except Timeout: return False - def __setitem__(self, key, value): """Set `key` and `value` item in cache. @@ -139,7 +137,6 @@ class FanoutCache(object): shard = self._shards[index] shard[key] = value - def touch(self, key, expire=None, retry=False): """Touch `key` in cache and update `expire` time. @@ -160,7 +157,6 @@ class FanoutCache(object): except Timeout: return False - def add(self, key, value, expire=None, read=False, tag=None, retry=False): """Add `key` and `value` item to cache. @@ -192,7 +188,6 @@ class FanoutCache(object): except Timeout: return False - def incr(self, key, delta=1, default=0, retry=False): """Increment value by delta for item with key. @@ -224,7 +219,6 @@ class FanoutCache(object): except Timeout: return None - def decr(self, key, delta=1, default=0, retry=False): """Decrement value by delta for item with key. @@ -259,9 +253,15 @@ class FanoutCache(object): except Timeout: return None - - def get(self, key, default=None, read=False, expire_time=False, tag=False, - retry=False): + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Retrieve value from cache. If `key` is missing, return `default`. If database timeout occurs then returns `default` unless `retry` is set @@ -285,7 +285,6 @@ class FanoutCache(object): except (Timeout, sqlite3.OperationalError): return default - def __getitem__(self, key): """Return corresponding value for `key` from cache. @@ -300,7 +299,6 @@ class FanoutCache(object): shard = self._shards[index] return shard[key] - def read(self, key): """Return file handle corresponding to `key` from cache. @@ -314,7 +312,6 @@ class FanoutCache(object): raise KeyError(key) return handle - def __contains__(self, key): """Return `True` if `key` matching item is found in cache. @@ -326,8 +323,9 @@ class FanoutCache(object): shard = self._shards[index] return key in shard - - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -353,7 +351,6 @@ class FanoutCache(object): except Timeout: return default - def delete(self, key, retry=False): """Delete corresponding item for `key` from cache. @@ -374,7 +371,6 @@ class FanoutCache(object): except Timeout: return False - def __delitem__(self, key): """Delete corresponding item for `key` from cache. @@ -388,7 +384,6 @@ class FanoutCache(object): shard = self._shards[index] del shard[key] - def check(self, fix=False, retry=False): """Check database and file system consistency. @@ -412,7 +407,6 @@ class FanoutCache(object): warnings = (shard.check(fix, retry) for shard in self._shards) return functools.reduce(operator.iadd, warnings, []) - def expire(self, retry=False): """Remove expired items from cache. @@ -425,7 +419,6 @@ class FanoutCache(object): """ return self._remove('expire', args=(time.time(),), retry=retry) - def create_tag_index(self): """Create tag index on cache database. @@ -437,7 +430,6 @@ class FanoutCache(object): for shard in self._shards: shard.create_tag_index() - def drop_tag_index(self): """Drop tag index on cache database. @@ -447,7 +439,6 @@ class FanoutCache(object): for shard in self._shards: shard.drop_tag_index() - def evict(self, tag, retry=False): """Remove items with matching `tag` from cache. @@ -461,7 +452,6 @@ class FanoutCache(object): """ return self._remove('evict', args=(tag,), retry=retry) - def cull(self, retry=False): """Cull items from cache until volume is less than size limit. @@ -474,7 +464,6 @@ class FanoutCache(object): """ return self._remove('cull', retry=retry) - def clear(self, retry=False): """Remove all items from cache. @@ -487,7 +476,6 @@ class FanoutCache(object): """ return self._remove('clear', retry=retry) - def _remove(self, name, args=(), retry=False): total = 0 for shard in self._shards: @@ -502,7 +490,6 @@ class FanoutCache(object): break return total - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -516,7 +503,6 @@ class FanoutCache(object): total_misses = sum(misses for _, misses in results) return total_hits, total_misses - def volume(self): """Return estimated total size of cache on disk. @@ -525,49 +511,40 @@ class FanoutCache(object): """ return sum(shard.volume() for shard in self._shards) - def close(self): - "Close database connection." + """Close database connection.""" for shard in self._shards: shard.close() self._caches.clear() self._deques.clear() self._indexes.clear() - def __enter__(self): return self - def __exit__(self, *exception): self.close() - def __getstate__(self): return (self._directory, self._count, self.timeout, type(self.disk)) - def __setstate__(self, state): self.__init__(*state) - def __iter__(self): - "Iterate keys in cache including expired items." + """Iterate keys in cache including expired items.""" iterators = (iter(shard) for shard in self._shards) return it.chain.from_iterable(iterators) - def __reversed__(self): - "Reverse iterate keys in cache including expired items." + """Reverse iterate keys in cache including expired items.""" iterators = (reversed(shard) for shard in reversed(self._shards)) return it.chain.from_iterable(iterators) - def __len__(self): - "Count of items in cache including expired items." + """Count of items in cache including expired items.""" return sum(len(shard) for shard in self._shards) - def reset(self, key, value=ENOVAL): """Reset `key` and `value` item from Settings table. @@ -596,7 +573,6 @@ class FanoutCache(object): break return result - def cache(self, name): """Return Cache with given `name` in subdirectory. @@ -622,11 +598,10 @@ class FanoutCache(object): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) - temp = Cache(directory=directory) + temp = Cache(directory=directory, disk=self._disk) _caches[name] = temp return temp - def deque(self, name): """Return Deque with given `name` in subdirectory. @@ -651,10 +626,10 @@ class FanoutCache(object): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) - temp = Deque(directory=directory) - _deques[name] = temp - return temp - + cache = Cache(directory=directory, disk=self._disk) + deque = Deque.fromcache(cache) + _deques[name] = deque + return deque def index(self, name): """Return Index with given `name` in subdirectory. @@ -683,9 +658,10 @@ class FanoutCache(object): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'index', *parts) - temp = Index(directory) - _indexes[name] = temp - return temp + cache = Cache(directory=directory, disk=self._disk) + index = Index.fromcache(cache) + _indexes[name] = index + return index -FanoutCache.memoize = Cache.memoize +FanoutCache.memoize = Cache.memoize # type: ignore diff --git a/lib/diskcache/persistent.py b/lib/diskcache/persistent.py index d0a452e6..ed23398d 100644 --- a/lib/diskcache/persistent.py +++ b/lib/diskcache/persistent.py @@ -1,22 +1,27 @@ """Persistent Data Types - """ import operator as op - +import sys from collections import OrderedDict -from collections.abc import MutableMapping, Sequence -from collections.abc import KeysView, ValuesView, ItemsView +from collections.abc import ( + ItemsView, + KeysView, + MutableMapping, + Sequence, + ValuesView, +) from contextlib import contextmanager from shutil import rmtree -from .core import Cache, ENOVAL +from .core import ENOVAL, Cache def _make_compare(seq_op, doc): - "Make compare method with Sequence semantics." + """Make compare method with Sequence semantics.""" + def compare(self, that): - "Compare method for deque and sequence." + """Compare method for deque and sequence.""" if not isinstance(that, Sequence): return NotImplemented @@ -70,6 +75,7 @@ class Deque(Sequence): [3, 2, 1, 0, 0, -1, -2, -3] """ + def __init__(self, iterable=(), directory=None): """Initialize deque instance. @@ -81,9 +87,7 @@ class Deque(Sequence): """ self._cache = Cache(directory, eviction_policy='none') - with self.transact(): - self.extend(iterable) - + self.extend(iterable) @classmethod def fromcache(cls, cache, iterable=()): @@ -111,19 +115,16 @@ class Deque(Sequence): self.extend(iterable) return self - @property def cache(self): - "Cache used by deque." + """Cache used by deque.""" return self._cache - @property def directory(self): - "Directory path where deque is stored." + """Directory path where deque is stored.""" return self._cache.directory - def _index(self, index, func): len_self = len(self) @@ -154,7 +155,6 @@ class Deque(Sequence): raise IndexError('deque index out of range') - def __getitem__(self, index): """deque.__getitem__(index) <==> deque[index] @@ -177,7 +177,6 @@ class Deque(Sequence): """ return self._index(index, self._cache.__getitem__) - def __setitem__(self, index, value): """deque.__setitem__(index, value) <==> deque[index] = value @@ -196,10 +195,11 @@ class Deque(Sequence): :raises IndexError: if index out of range """ + def _set_value(key): return self._cache.__setitem__(key, value) - self._index(index, _set_value) + self._index(index, _set_value) def __delitem__(self, index): """deque.__delitem__(index) <==> del deque[index] @@ -220,7 +220,6 @@ class Deque(Sequence): """ self._index(index, self._cache.__delitem__) - def __repr__(self): """deque.__repr__() <==> repr(deque) @@ -230,7 +229,6 @@ class Deque(Sequence): name = type(self).__name__ return '{0}(directory={1!r})'.format(name, self.directory) - __eq__ = _make_compare(op.eq, 'equal to') __ne__ = _make_compare(op.ne, 'not equal to') __lt__ = _make_compare(op.lt, 'less than') @@ -238,7 +236,6 @@ class Deque(Sequence): __le__ = _make_compare(op.le, 'less than or equal to') __ge__ = _make_compare(op.ge, 'greater than or equal to') - def __iadd__(self, iterable): """deque.__iadd__(iterable) <==> deque += iterable @@ -251,7 +248,6 @@ class Deque(Sequence): self.extend(iterable) return self - def __iter__(self): """deque.__iter__() <==> iter(deque) @@ -266,7 +262,6 @@ class Deque(Sequence): except KeyError: pass - def __len__(self): """deque.__len__() <==> len(deque) @@ -275,7 +270,6 @@ class Deque(Sequence): """ return len(self._cache) - def __reversed__(self): """deque.__reversed__() <==> reversed(deque) @@ -298,15 +292,12 @@ class Deque(Sequence): except KeyError: pass - def __getstate__(self): return self.directory - def __setstate__(self, state): self.__init__(directory=state) - def append(self, value): """Add `value` to back of deque. @@ -322,7 +313,6 @@ class Deque(Sequence): """ self._cache.push(value, retry=True) - def appendleft(self, value): """Add `value` to front of deque. @@ -338,7 +328,6 @@ class Deque(Sequence): """ self._cache.push(value, side='front', retry=True) - def clear(self): """Remove all elements from deque. @@ -352,7 +341,6 @@ class Deque(Sequence): """ self._cache.clear(retry=True) - def count(self, value): """Return number of occurrences of `value` in deque. @@ -371,7 +359,6 @@ class Deque(Sequence): """ return sum(1 for item in self if value == item) - def extend(self, iterable): """Extend back side of deque with values from `iterable`. @@ -381,7 +368,6 @@ class Deque(Sequence): for value in iterable: self.append(value) - def extendleft(self, iterable): """Extend front side of deque with value from `iterable`. @@ -396,7 +382,6 @@ class Deque(Sequence): for value in iterable: self.appendleft(value) - def peek(self): """Peek at value at back of deque. @@ -423,7 +408,6 @@ class Deque(Sequence): raise IndexError('peek from an empty deque') return value - def peekleft(self): """Peek at value at front of deque. @@ -450,7 +434,6 @@ class Deque(Sequence): raise IndexError('peek from an empty deque') return value - def pop(self): """Remove and return value at back of deque. @@ -477,7 +460,6 @@ class Deque(Sequence): raise IndexError('pop from an empty deque') return value - def popleft(self): """Remove and return value at front of deque. @@ -502,7 +484,6 @@ class Deque(Sequence): raise IndexError('pop from an empty deque') return value - def remove(self, value): """Remove first occurrence of `value` in deque. @@ -540,7 +521,6 @@ class Deque(Sequence): raise ValueError('deque.remove(value): value not in deque') - def reverse(self): """Reverse deque in place. @@ -563,7 +543,6 @@ class Deque(Sequence): del temp rmtree(directory) - def rotate(self, steps=1): """Rotate deque right by `steps`. @@ -612,9 +591,7 @@ class Deque(Sequence): else: self.append(value) - - __hash__ = None - + __hash__ = None # type: ignore @contextmanager def transact(self): @@ -665,6 +642,7 @@ class Index(MutableMapping): ('c', 3) """ + def __init__(self, *args, **kwargs): """Initialize index in directory and update items. @@ -692,7 +670,6 @@ class Index(MutableMapping): self._cache = Cache(directory, eviction_policy='none') self.update(*args, **kwargs) - @classmethod def fromcache(cls, cache, *args, **kwargs): """Initialize index using `cache` and update items. @@ -720,19 +697,16 @@ class Index(MutableMapping): self.update(*args, **kwargs) return self - @property def cache(self): - "Cache used by index." + """Cache used by index.""" return self._cache - @property def directory(self): - "Directory path where items are stored." + """Directory path where items are stored.""" return self._cache.directory - def __getitem__(self, key): """index.__getitem__(key) <==> index[key] @@ -756,7 +730,6 @@ class Index(MutableMapping): """ return self._cache[key] - def __setitem__(self, key, value): """index.__setitem__(key, value) <==> index[key] = value @@ -774,7 +747,6 @@ class Index(MutableMapping): """ self._cache[key] = value - def __delitem__(self, key): """index.__delitem__(key) <==> del index[key] @@ -797,7 +769,6 @@ class Index(MutableMapping): """ del self._cache[key] - def setdefault(self, key, default=None): """Set and get value for `key` in index using `default`. @@ -822,7 +793,6 @@ class Index(MutableMapping): except KeyError: _cache.add(key, default, retry=True) - def peekitem(self, last=True): """Peek at key and value item pair in index based on iteration order. @@ -841,7 +811,6 @@ class Index(MutableMapping): """ return self._cache.peekitem(last, retry=True) - def pop(self, key, default=ENOVAL): """Remove corresponding item for `key` from index and return value. @@ -872,7 +841,6 @@ class Index(MutableMapping): raise KeyError(key) return value - def popitem(self, last=True): """Remove and return item pair. @@ -907,7 +875,6 @@ class Index(MutableMapping): return key, value - def push(self, value, prefix=None, side='back'): """Push `value` onto `side` of queue in index identified by `prefix`. @@ -939,7 +906,6 @@ class Index(MutableMapping): """ return self._cache.push(value, prefix, side, retry=True) - def pull(self, prefix=None, default=(None, None), side='front'): """Pull key and value item pair from `side` of queue in index. @@ -980,7 +946,6 @@ class Index(MutableMapping): """ return self._cache.pull(prefix, default, side, retry=True) - def clear(self): """Remove all items from index. @@ -994,7 +959,6 @@ class Index(MutableMapping): """ self._cache.clear(retry=True) - def __iter__(self): """index.__iter__() <==> iter(index) @@ -1003,7 +967,6 @@ class Index(MutableMapping): """ return iter(self._cache) - def __reversed__(self): """index.__reversed__() <==> reversed(index) @@ -1020,7 +983,6 @@ class Index(MutableMapping): """ return reversed(self._cache) - def __len__(self): """index.__len__() <==> len(index) @@ -1029,7 +991,6 @@ class Index(MutableMapping): """ return len(self._cache) - def keys(self): """Set-like object providing a view of index keys. @@ -1044,7 +1005,6 @@ class Index(MutableMapping): """ return KeysView(self) - def values(self): """Set-like object providing a view of index values. @@ -1059,7 +1019,6 @@ class Index(MutableMapping): """ return ValuesView(self) - def items(self): """Set-like object providing a view of index items. @@ -1074,18 +1033,14 @@ class Index(MutableMapping): """ return ItemsView(self) - - __hash__ = None - + __hash__ = None # type: ignore def __getstate__(self): return self.directory - def __setstate__(self, state): self.__init__(state) - def __eq__(self, other): """index.__eq__(other) <==> index == other @@ -1119,7 +1074,6 @@ class Index(MutableMapping): else: return all(self[key] == other.get(key, ENOVAL) for key in self) - def __ne__(self, other): """index.__ne__(other) <==> index != other @@ -1143,8 +1097,7 @@ class Index(MutableMapping): """ return not self == other - - def memoize(self, name=None, typed=False): + def memoize(self, name=None, typed=False, ignore=()): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -1195,11 +1148,11 @@ class Index(MutableMapping): :param str name: name given for callable (default None, automatic) :param bool typed: cache different types separately (default False) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ - return self._cache.memoize(name, typed) - + return self._cache.memoize(name, typed, ignore=ignore) @contextmanager def transact(self): @@ -1228,7 +1181,6 @@ class Index(MutableMapping): with self._cache.transact(retry=True): yield - def __repr__(self): """index.__repr__() <==> repr(index) diff --git a/lib/diskcache/recipes.py b/lib/diskcache/recipes.py index 85509c73..b5af6dd7 100644 --- a/lib/diskcache/recipes.py +++ b/lib/diskcache/recipes.py @@ -1,5 +1,4 @@ """Disk Cache Recipes - """ import functools @@ -12,7 +11,7 @@ import time from .core import ENOVAL, args_to_key, full_name -class Averager(object): +class Averager: """Recipe for calculating a running average. Sometimes known as "online statistics," the running average maintains the @@ -32,6 +31,7 @@ class Averager(object): None """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -39,27 +39,30 @@ class Averager(object): self._tag = tag def add(self, value): - "Add `value` to average." + """Add `value` to average.""" with self._cache.transact(retry=True): total, count = self._cache.get(self._key, default=(0.0, 0)) total += value count += 1 self._cache.set( - self._key, (total, count), expire=self._expire, tag=self._tag, + self._key, + (total, count), + expire=self._expire, + tag=self._tag, ) def get(self): - "Get current average or return `None` if count equals zero." + """Get current average or return `None` if count equals zero.""" total, count = self._cache.get(self._key, default=(0.0, 0), retry=True) return None if count == 0 else total / count def pop(self): - "Return current average and delete key." + """Return current average and delete key.""" total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True) return None if count == 0 else total / count -class Lock(object): +class Lock: """Recipe for cross-process and cross-thread lock. >>> import diskcache @@ -71,6 +74,7 @@ class Lock(object): ... pass """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -78,7 +82,7 @@ class Lock(object): self._tag = tag def acquire(self): - "Acquire lock using spin-lock algorithm." + """Acquire lock using spin-lock algorithm.""" while True: added = self._cache.add( self._key, @@ -92,11 +96,11 @@ class Lock(object): time.sleep(0.001) def release(self): - "Release lock by deleting key." + """Release lock by deleting key.""" self._cache.delete(self._key, retry=True) def locked(self): - "Return true if the lock is acquired." + """Return true if the lock is acquired.""" return self._key in self._cache def __enter__(self): @@ -106,7 +110,7 @@ class Lock(object): self.release() -class RLock(object): +class RLock: """Recipe for cross-process and cross-thread re-entrant lock. >>> import diskcache @@ -124,6 +128,7 @@ class RLock(object): AssertionError: cannot release un-acquired lock """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -131,7 +136,7 @@ class RLock(object): self._tag = tag def acquire(self): - "Acquire lock by incrementing count using spin-lock algorithm." + """Acquire lock by incrementing count using spin-lock algorithm.""" pid = os.getpid() tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) @@ -141,14 +146,16 @@ class RLock(object): value, count = self._cache.get(self._key, default=(None, 0)) if pid_tid == value or count == 0: self._cache.set( - self._key, (pid_tid, count + 1), - expire=self._expire, tag=self._tag, + self._key, + (pid_tid, count + 1), + expire=self._expire, + tag=self._tag, ) return time.sleep(0.001) def release(self): - "Release lock by decrementing count." + """Release lock by decrementing count.""" pid = os.getpid() tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) @@ -158,8 +165,10 @@ class RLock(object): is_owned = pid_tid == value and count > 0 assert is_owned, 'cannot release un-acquired lock' self._cache.set( - self._key, (value, count - 1), - expire=self._expire, tag=self._tag, + self._key, + (value, count - 1), + expire=self._expire, + tag=self._tag, ) def __enter__(self): @@ -169,7 +178,7 @@ class RLock(object): self.release() -class BoundedSemaphore(object): +class BoundedSemaphore: """Recipe for cross-process and cross-thread bounded semaphore. >>> import diskcache @@ -187,6 +196,7 @@ class BoundedSemaphore(object): AssertionError: cannot release un-acquired semaphore """ + def __init__(self, cache, key, value=1, expire=None, tag=None): self._cache = cache self._key = key @@ -195,26 +205,31 @@ class BoundedSemaphore(object): self._tag = tag def acquire(self): - "Acquire semaphore by decrementing value using spin-lock algorithm." + """Acquire semaphore by decrementing value using spin-lock algorithm.""" while True: with self._cache.transact(retry=True): value = self._cache.get(self._key, default=self._value) if value > 0: self._cache.set( - self._key, value - 1, - expire=self._expire, tag=self._tag, + self._key, + value - 1, + expire=self._expire, + tag=self._tag, ) return time.sleep(0.001) def release(self): - "Release semaphore by incrementing value." + """Release semaphore by incrementing value.""" with self._cache.transact(retry=True): value = self._cache.get(self._key, default=self._value) assert self._value > value, 'cannot release un-acquired semaphore' value += 1 self._cache.set( - self._key, value, expire=self._expire, tag=self._tag, + self._key, + value, + expire=self._expire, + tag=self._tag, ) def __enter__(self): @@ -224,8 +239,16 @@ class BoundedSemaphore(object): self.release() -def throttle(cache, count, seconds, name=None, expire=None, tag=None, - time_func=time.time, sleep_func=time.sleep): +def throttle( + cache, + count, + seconds, + name=None, + expire=None, + tag=None, + time_func=time.time, + sleep_func=time.sleep, +): """Decorator to throttle calls to function. >>> import diskcache, time @@ -242,6 +265,7 @@ def throttle(cache, count, seconds, name=None, expire=None, tag=None, True """ + def decorator(func): rate = count / float(seconds) key = full_name(func) if name is None else name @@ -298,6 +322,7 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): >>> pool.terminate() """ + def decorator(func): key = full_name(func) if name is None else name lock = lock_factory(cache, key, expire=expire, tag=tag) @@ -312,7 +337,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): return decorator -def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): +def memoize_stampede( + cache, expire, name=None, typed=False, tag=None, beta=1, ignore=() +): """Memoizing cache decorator with cache stampede protection. Cache stampedes are a type of system overload that can occur when parallel @@ -365,16 +392,17 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): :param str name: name given for callable (default None, automatic) :param bool typed: cache different types separately (default False) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ # Caution: Nearly identical code exists in Cache.memoize def decorator(func): - "Decorator created by memoize call for callable." + """Decorator created by memoize call for callable.""" base = (full_name(func),) if name is None else (name,) def timer(*args, **kwargs): - "Time execution of `func` and return result and time delta." + """Time execution of `func` and return result and time delta.""" start = time.time() result = func(*args, **kwargs) delta = time.time() - start @@ -382,10 +410,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): @functools.wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) pair, expire_time = cache.get( - key, default=ENOVAL, expire_time=True, retry=True, + key, + default=ENOVAL, + expire_time=True, + retry=True, ) if pair is not ENOVAL: @@ -400,7 +431,10 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): thread_key = key + (ENOVAL,) thread_added = cache.add( - thread_key, None, expire=delta, retry=True, + thread_key, + None, + expire=delta, + retry=True, ) if thread_added: @@ -409,8 +443,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): with cache: pair = timer(*args, **kwargs) cache.set( - key, pair, expire=expire, tag=tag, retry=True, + key, + pair, + expire=expire, + tag=tag, + retry=True, ) + thread = threading.Thread(target=recompute) thread.daemon = True thread.start() @@ -422,8 +461,8 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): return pair[0] def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." - return args_to_key(base, args, kwargs, typed) + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper