Update diskcache 5.1.0 (40ce0de) → 5.4.0 (1cb1425).

This commit is contained in:
JackDandy 2023-01-13 02:13:40 +00:00
parent 767ae2962f
commit 3a10574881
8 changed files with 503 additions and 511 deletions

View file

@ -4,6 +4,7 @@
* Add Filelock 3.9.0 (ce3e891)
* Remove Lockfile no longer used by Cachecontrol
* Update Msgpack 1.0.0 (fa7d744) to 1.0.4 (b5acfd5)
* Update diskcache 5.1.0 (40ce0de) to 5.4.0 (1cb1425)
* Update Rarfile 4.0 (55fe778) to 4.1a1 (8a72967)
* Update UnRar x64 for Windows 6.11 to 6.20
* Update Send2Trash 1.5.0 (66afce7) to 1.8.1b0 (0ef9b32)

View file

@ -3,17 +3,31 @@ DiskCache API Reference
=======================
The :doc:`tutorial` provides a helpful walkthrough of most methods.
"""
from .core import (
Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout
DEFAULT_SETTINGS,
ENOVAL,
EVICTION_POLICY,
UNKNOWN,
Cache,
Disk,
EmptyDirWarning,
JSONDisk,
Timeout,
UnknownFileWarning,
)
from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN
from .fanout import FanoutCache
from .persistent import Deque, Index
from .recipes import Averager, BoundedSemaphore, Lock, RLock
from .recipes import barrier, memoize_stampede, throttle
from .recipes import (
Averager,
BoundedSemaphore,
Lock,
RLock,
barrier,
memoize_stampede,
throttle,
)
__all__ = [
'Averager',
@ -40,14 +54,15 @@ __all__ = [
try:
from .djangocache import DjangoCache # noqa
__all__.append('DjangoCache')
except Exception: # pylint: disable=broad-except
except Exception: # pylint: disable=broad-except # pragma: no cover
# Django not installed or not setup so ignore.
pass
__title__ = 'diskcache'
__version__ = '5.1.0'
__build__ = 0x050100
__version__ = '5.4.0'
__build__ = 0x050400
__author__ = 'Grant Jenks'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016-2020 Grant Jenks'
__copyright__ = 'Copyright 2016-2022 Grant Jenks'

View file

@ -1 +1 @@
"Command line interface to disk cache."
"""Command line interface to disk cache."""

File diff suppressed because it is too large Load diff

View file

@ -1,11 +1,12 @@
"Django-compatible disk and file backed cache."
"""Django-compatible disk and file backed cache."""
from functools import wraps
from django.core.cache.backends.base import BaseCache
try:
from django.core.cache.backends.base import DEFAULT_TIMEOUT
except ImportError:
except ImportError: # pragma: no cover
# For older versions of Django simply use 300 seconds.
DEFAULT_TIMEOUT = 300
@ -14,7 +15,8 @@ from .fanout import FanoutCache
class DjangoCache(BaseCache):
"Django-compatible disk and file backed cache."
"""Django-compatible disk and file backed cache."""
def __init__(self, directory, params):
"""Initialize DjangoCache instance.
@ -28,13 +30,11 @@ class DjangoCache(BaseCache):
options = params.get('OPTIONS', {})
self._cache = FanoutCache(directory, shards, timeout, **options)
@property
def directory(self):
"""Cache directory."""
return self._cache.directory
def cache(self, name):
"""Return Cache with given `name` in subdirectory.
@ -44,7 +44,6 @@ class DjangoCache(BaseCache):
"""
return self._cache.cache(name)
def deque(self, name):
"""Return Deque with given `name` in subdirectory.
@ -54,7 +53,6 @@ class DjangoCache(BaseCache):
"""
return self._cache.deque(name)
def index(self, name):
"""Return Index with given `name` in subdirectory.
@ -64,9 +62,16 @@ class DjangoCache(BaseCache):
"""
return self._cache.index(name)
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None,
read=False, tag=None, retry=True):
def add(
self,
key,
value,
timeout=DEFAULT_TIMEOUT,
version=None,
read=False,
tag=None,
retry=True,
):
"""Set a value in the cache if the key does not already exist. If
timeout is given, that timeout will be used for the key; otherwise the
default cache timeout will be used.
@ -89,9 +94,16 @@ class DjangoCache(BaseCache):
timeout = self.get_backend_timeout(timeout=timeout)
return self._cache.add(key, value, timeout, read, tag, retry)
def get(self, key, default=None, version=None, read=False,
expire_time=False, tag=False, retry=False):
def get(
self,
key,
default=None,
version=None,
read=False,
expire_time=False,
tag=False,
retry=False,
):
"""Fetch a given key from the cache. If the key does not exist, return
default, which itself defaults to None.
@ -111,7 +123,6 @@ class DjangoCache(BaseCache):
key = self.make_key(key, version=version)
return self._cache.get(key, default, read, expire_time, tag, retry)
def read(self, key, version=None):
"""Return file handle corresponding to `key` from Cache.
@ -124,9 +135,16 @@ class DjangoCache(BaseCache):
key = self.make_key(key, version=version)
return self._cache.read(key)
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None,
read=False, tag=None, retry=True):
def set(
self,
key,
value,
timeout=DEFAULT_TIMEOUT,
version=None,
read=False,
tag=None,
retry=True,
):
"""Set a value in the cache. If timeout is given, that timeout will be
used for the key; otherwise the default cache timeout will be used.
@ -146,7 +164,6 @@ class DjangoCache(BaseCache):
timeout = self.get_backend_timeout(timeout=timeout)
return self._cache.set(key, value, timeout, read, tag, retry)
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True):
"""Touch a key in the cache. If timeout is given, that timeout will be
used for the key; otherwise the default cache timeout will be used.
@ -164,9 +181,15 @@ class DjangoCache(BaseCache):
timeout = self.get_backend_timeout(timeout=timeout)
return self._cache.touch(key, timeout, retry)
def pop(self, key, default=None, version=None, expire_time=False,
tag=False, retry=True):
def pop(
self,
key,
default=None,
version=None,
expire_time=False,
tag=False,
retry=True,
):
"""Remove corresponding item for `key` from cache and return value.
If `key` is missing, return `default`.
@ -186,7 +209,6 @@ class DjangoCache(BaseCache):
key = self.make_key(key, version=version)
return self._cache.pop(key, default, expire_time, tag, retry)
def delete(self, key, version=None, retry=True):
"""Delete a key from the cache, failing silently.
@ -198,8 +220,7 @@ class DjangoCache(BaseCache):
"""
# pylint: disable=arguments-differ
key = self.make_key(key, version=version)
self._cache.delete(key, retry)
return self._cache.delete(key, retry)
def incr(self, key, delta=1, version=None, default=None, retry=True):
"""Increment value by delta for item with key.
@ -230,7 +251,6 @@ class DjangoCache(BaseCache):
except KeyError:
raise ValueError("Key '%s' not found" % key) from None
def decr(self, key, delta=1, version=None, default=None, retry=True):
"""Decrement value by delta for item with key.
@ -259,7 +279,6 @@ class DjangoCache(BaseCache):
# pylint: disable=arguments-differ
return self.incr(key, -delta, version, default, retry)
def has_key(self, key, version=None):
"""Returns True if the key is in the cache and has not expired.
@ -271,7 +290,6 @@ class DjangoCache(BaseCache):
key = self.make_key(key, version=version)
return key in self._cache
def expire(self):
"""Remove expired items from cache.
@ -280,7 +298,6 @@ class DjangoCache(BaseCache):
"""
return self._cache.expire()
def stats(self, enable=True, reset=False):
"""Return cache statistics hits and misses.
@ -291,7 +308,6 @@ class DjangoCache(BaseCache):
"""
return self._cache.stats(enable=enable, reset=reset)
def create_tag_index(self):
"""Create tag index on cache database.
@ -302,7 +318,6 @@ class DjangoCache(BaseCache):
"""
self._cache.create_tag_index()
def drop_tag_index(self):
"""Drop tag index on cache database.
@ -311,7 +326,6 @@ class DjangoCache(BaseCache):
"""
self._cache.drop_tag_index()
def evict(self, tag):
"""Remove items with matching `tag` from cache.
@ -321,7 +335,6 @@ class DjangoCache(BaseCache):
"""
return self._cache.evict(tag)
def cull(self):
"""Cull items from cache until volume is less than size limit.
@ -330,18 +343,15 @@ class DjangoCache(BaseCache):
"""
return self._cache.cull()
def clear(self):
"Remove *all* values from the cache at once."
"""Remove *all* values from the cache at once."""
return self._cache.clear()
def close(self, **kwargs):
"Close the cache connection."
"""Close the cache connection."""
# pylint: disable=unused-argument
self._cache.close()
def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT):
"""Return seconds to expiration.
@ -356,9 +366,15 @@ class DjangoCache(BaseCache):
timeout = -1
return None if timeout is None else timeout
def memoize(self, name=None, timeout=DEFAULT_TIMEOUT, version=None,
typed=False, tag=None):
def memoize(
self,
name=None,
timeout=DEFAULT_TIMEOUT,
version=None,
typed=False,
tag=None,
ignore=(),
):
"""Memoizing cache decorator.
Decorator to wrap callable with memoizing function using cache.
@ -392,6 +408,7 @@ class DjangoCache(BaseCache):
:param int version: key version number (default None, cache parameter)
:param bool typed: cache different types separately (default False)
:param str tag: text to associate with arguments (default None)
:param set ignore: positional or keyword args to ignore (default ())
:return: callable decorator
"""
@ -400,12 +417,12 @@ class DjangoCache(BaseCache):
raise TypeError('name cannot be callable')
def decorator(func):
"Decorator created by memoize() for callable `func`."
"""Decorator created by memoize() for callable `func`."""
base = (full_name(func),) if name is None else (name,)
@wraps(func)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
"""Wrapper for callable to cache arguments and return values."""
key = wrapper.__cache_key__(*args, **kwargs)
result = self.get(key, ENOVAL, version, retry=True)
@ -418,14 +435,19 @@ class DjangoCache(BaseCache):
)
if valid_timeout:
self.set(
key, result, timeout, version, tag=tag, retry=True,
key,
result,
timeout,
version,
tag=tag,
retry=True,
)
return result
def __cache_key__(*args, **kwargs):
"Make key for cache given function arguments."
return args_to_key(base, args, kwargs, typed)
"""Make key for cache given function arguments."""
return args_to_key(base, args, kwargs, typed, ignore)
wrapper.__cache_key__ = __cache_key__
return wrapper

View file

@ -1,4 +1,4 @@
"Fanout cache automatically shards keys and values."
"""Fanout cache automatically shards keys and values."""
import contextlib as cl
import functools
@ -9,14 +9,16 @@ import sqlite3
import tempfile
import time
from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout
from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout
from .persistent import Deque, Index
class FanoutCache(object):
"Cache that shards keys and values."
def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk,
**settings):
class FanoutCache:
"""Cache that shards keys and values."""
def __init__(
self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings
):
"""Initialize cache instance.
:param str directory: cache directory
@ -36,6 +38,7 @@ class FanoutCache(object):
self._count = shards
self._directory = directory
self._disk = disk
self._shards = tuple(
Cache(
directory=op.join(directory, '%03d' % num),
@ -51,20 +54,17 @@ class FanoutCache(object):
self._deques = {}
self._indexes = {}
@property
def directory(self):
"""Cache directory."""
return self._directory
def __getattr__(self, name):
safe_names = {'timeout', 'disk'}
valid_name = name in DEFAULT_SETTINGS or name in safe_names
assert valid_name, 'cannot access {} in cache shard'.format(name)
return getattr(self._shards[0], name)
@cl.contextmanager
def transact(self, retry=True):
"""Context manager to perform a transaction by locking the cache.
@ -98,7 +98,6 @@ class FanoutCache(object):
stack.enter_context(shard_transaction)
yield
def set(self, key, value, expire=None, read=False, tag=None, retry=False):
"""Set `key` and `value` item in cache.
@ -125,7 +124,6 @@ class FanoutCache(object):
except Timeout:
return False
def __setitem__(self, key, value):
"""Set `key` and `value` item in cache.
@ -139,7 +137,6 @@ class FanoutCache(object):
shard = self._shards[index]
shard[key] = value
def touch(self, key, expire=None, retry=False):
"""Touch `key` in cache and update `expire` time.
@ -160,7 +157,6 @@ class FanoutCache(object):
except Timeout:
return False
def add(self, key, value, expire=None, read=False, tag=None, retry=False):
"""Add `key` and `value` item to cache.
@ -192,7 +188,6 @@ class FanoutCache(object):
except Timeout:
return False
def incr(self, key, delta=1, default=0, retry=False):
"""Increment value by delta for item with key.
@ -224,7 +219,6 @@ class FanoutCache(object):
except Timeout:
return None
def decr(self, key, delta=1, default=0, retry=False):
"""Decrement value by delta for item with key.
@ -259,9 +253,15 @@ class FanoutCache(object):
except Timeout:
return None
def get(self, key, default=None, read=False, expire_time=False, tag=False,
retry=False):
def get(
self,
key,
default=None,
read=False,
expire_time=False,
tag=False,
retry=False,
):
"""Retrieve value from cache. If `key` is missing, return `default`.
If database timeout occurs then returns `default` unless `retry` is set
@ -285,7 +285,6 @@ class FanoutCache(object):
except (Timeout, sqlite3.OperationalError):
return default
def __getitem__(self, key):
"""Return corresponding value for `key` from cache.
@ -300,7 +299,6 @@ class FanoutCache(object):
shard = self._shards[index]
return shard[key]
def read(self, key):
"""Return file handle corresponding to `key` from cache.
@ -314,7 +312,6 @@ class FanoutCache(object):
raise KeyError(key)
return handle
def __contains__(self, key):
"""Return `True` if `key` matching item is found in cache.
@ -326,8 +323,9 @@ class FanoutCache(object):
shard = self._shards[index]
return key in shard
def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501
def pop(
self, key, default=None, expire_time=False, tag=False, retry=False
): # noqa: E501
"""Remove corresponding item for `key` from cache and return value.
If `key` is missing, return `default`.
@ -353,7 +351,6 @@ class FanoutCache(object):
except Timeout:
return default
def delete(self, key, retry=False):
"""Delete corresponding item for `key` from cache.
@ -374,7 +371,6 @@ class FanoutCache(object):
except Timeout:
return False
def __delitem__(self, key):
"""Delete corresponding item for `key` from cache.
@ -388,7 +384,6 @@ class FanoutCache(object):
shard = self._shards[index]
del shard[key]
def check(self, fix=False, retry=False):
"""Check database and file system consistency.
@ -412,7 +407,6 @@ class FanoutCache(object):
warnings = (shard.check(fix, retry) for shard in self._shards)
return functools.reduce(operator.iadd, warnings, [])
def expire(self, retry=False):
"""Remove expired items from cache.
@ -425,7 +419,6 @@ class FanoutCache(object):
"""
return self._remove('expire', args=(time.time(),), retry=retry)
def create_tag_index(self):
"""Create tag index on cache database.
@ -437,7 +430,6 @@ class FanoutCache(object):
for shard in self._shards:
shard.create_tag_index()
def drop_tag_index(self):
"""Drop tag index on cache database.
@ -447,7 +439,6 @@ class FanoutCache(object):
for shard in self._shards:
shard.drop_tag_index()
def evict(self, tag, retry=False):
"""Remove items with matching `tag` from cache.
@ -461,7 +452,6 @@ class FanoutCache(object):
"""
return self._remove('evict', args=(tag,), retry=retry)
def cull(self, retry=False):
"""Cull items from cache until volume is less than size limit.
@ -474,7 +464,6 @@ class FanoutCache(object):
"""
return self._remove('cull', retry=retry)
def clear(self, retry=False):
"""Remove all items from cache.
@ -487,7 +476,6 @@ class FanoutCache(object):
"""
return self._remove('clear', retry=retry)
def _remove(self, name, args=(), retry=False):
total = 0
for shard in self._shards:
@ -502,7 +490,6 @@ class FanoutCache(object):
break
return total
def stats(self, enable=True, reset=False):
"""Return cache statistics hits and misses.
@ -516,7 +503,6 @@ class FanoutCache(object):
total_misses = sum(misses for _, misses in results)
return total_hits, total_misses
def volume(self):
"""Return estimated total size of cache on disk.
@ -525,49 +511,40 @@ class FanoutCache(object):
"""
return sum(shard.volume() for shard in self._shards)
def close(self):
"Close database connection."
"""Close database connection."""
for shard in self._shards:
shard.close()
self._caches.clear()
self._deques.clear()
self._indexes.clear()
def __enter__(self):
return self
def __exit__(self, *exception):
self.close()
def __getstate__(self):
return (self._directory, self._count, self.timeout, type(self.disk))
def __setstate__(self, state):
self.__init__(*state)
def __iter__(self):
"Iterate keys in cache including expired items."
"""Iterate keys in cache including expired items."""
iterators = (iter(shard) for shard in self._shards)
return it.chain.from_iterable(iterators)
def __reversed__(self):
"Reverse iterate keys in cache including expired items."
"""Reverse iterate keys in cache including expired items."""
iterators = (reversed(shard) for shard in reversed(self._shards))
return it.chain.from_iterable(iterators)
def __len__(self):
"Count of items in cache including expired items."
"""Count of items in cache including expired items."""
return sum(len(shard) for shard in self._shards)
def reset(self, key, value=ENOVAL):
"""Reset `key` and `value` item from Settings table.
@ -596,7 +573,6 @@ class FanoutCache(object):
break
return result
def cache(self, name):
"""Return Cache with given `name` in subdirectory.
@ -622,11 +598,10 @@ class FanoutCache(object):
except KeyError:
parts = name.split('/')
directory = op.join(self._directory, 'cache', *parts)
temp = Cache(directory=directory)
temp = Cache(directory=directory, disk=self._disk)
_caches[name] = temp
return temp
def deque(self, name):
"""Return Deque with given `name` in subdirectory.
@ -651,10 +626,10 @@ class FanoutCache(object):
except KeyError:
parts = name.split('/')
directory = op.join(self._directory, 'deque', *parts)
temp = Deque(directory=directory)
_deques[name] = temp
return temp
cache = Cache(directory=directory, disk=self._disk)
deque = Deque.fromcache(cache)
_deques[name] = deque
return deque
def index(self, name):
"""Return Index with given `name` in subdirectory.
@ -683,9 +658,10 @@ class FanoutCache(object):
except KeyError:
parts = name.split('/')
directory = op.join(self._directory, 'index', *parts)
temp = Index(directory)
_indexes[name] = temp
return temp
cache = Cache(directory=directory, disk=self._disk)
index = Index.fromcache(cache)
_indexes[name] = index
return index
FanoutCache.memoize = Cache.memoize
FanoutCache.memoize = Cache.memoize # type: ignore

View file

@ -1,22 +1,27 @@
"""Persistent Data Types
"""
import operator as op
import sys
from collections import OrderedDict
from collections.abc import MutableMapping, Sequence
from collections.abc import KeysView, ValuesView, ItemsView
from collections.abc import (
ItemsView,
KeysView,
MutableMapping,
Sequence,
ValuesView,
)
from contextlib import contextmanager
from shutil import rmtree
from .core import Cache, ENOVAL
from .core import ENOVAL, Cache
def _make_compare(seq_op, doc):
"Make compare method with Sequence semantics."
"""Make compare method with Sequence semantics."""
def compare(self, that):
"Compare method for deque and sequence."
"""Compare method for deque and sequence."""
if not isinstance(that, Sequence):
return NotImplemented
@ -70,6 +75,7 @@ class Deque(Sequence):
[3, 2, 1, 0, 0, -1, -2, -3]
"""
def __init__(self, iterable=(), directory=None):
"""Initialize deque instance.
@ -81,9 +87,7 @@ class Deque(Sequence):
"""
self._cache = Cache(directory, eviction_policy='none')
with self.transact():
self.extend(iterable)
self.extend(iterable)
@classmethod
def fromcache(cls, cache, iterable=()):
@ -111,19 +115,16 @@ class Deque(Sequence):
self.extend(iterable)
return self
@property
def cache(self):
"Cache used by deque."
"""Cache used by deque."""
return self._cache
@property
def directory(self):
"Directory path where deque is stored."
"""Directory path where deque is stored."""
return self._cache.directory
def _index(self, index, func):
len_self = len(self)
@ -154,7 +155,6 @@ class Deque(Sequence):
raise IndexError('deque index out of range')
def __getitem__(self, index):
"""deque.__getitem__(index) <==> deque[index]
@ -177,7 +177,6 @@ class Deque(Sequence):
"""
return self._index(index, self._cache.__getitem__)
def __setitem__(self, index, value):
"""deque.__setitem__(index, value) <==> deque[index] = value
@ -196,10 +195,11 @@ class Deque(Sequence):
:raises IndexError: if index out of range
"""
def _set_value(key):
return self._cache.__setitem__(key, value)
self._index(index, _set_value)
self._index(index, _set_value)
def __delitem__(self, index):
"""deque.__delitem__(index) <==> del deque[index]
@ -220,7 +220,6 @@ class Deque(Sequence):
"""
self._index(index, self._cache.__delitem__)
def __repr__(self):
"""deque.__repr__() <==> repr(deque)
@ -230,7 +229,6 @@ class Deque(Sequence):
name = type(self).__name__
return '{0}(directory={1!r})'.format(name, self.directory)
__eq__ = _make_compare(op.eq, 'equal to')
__ne__ = _make_compare(op.ne, 'not equal to')
__lt__ = _make_compare(op.lt, 'less than')
@ -238,7 +236,6 @@ class Deque(Sequence):
__le__ = _make_compare(op.le, 'less than or equal to')
__ge__ = _make_compare(op.ge, 'greater than or equal to')
def __iadd__(self, iterable):
"""deque.__iadd__(iterable) <==> deque += iterable
@ -251,7 +248,6 @@ class Deque(Sequence):
self.extend(iterable)
return self
def __iter__(self):
"""deque.__iter__() <==> iter(deque)
@ -266,7 +262,6 @@ class Deque(Sequence):
except KeyError:
pass
def __len__(self):
"""deque.__len__() <==> len(deque)
@ -275,7 +270,6 @@ class Deque(Sequence):
"""
return len(self._cache)
def __reversed__(self):
"""deque.__reversed__() <==> reversed(deque)
@ -298,15 +292,12 @@ class Deque(Sequence):
except KeyError:
pass
def __getstate__(self):
return self.directory
def __setstate__(self, state):
self.__init__(directory=state)
def append(self, value):
"""Add `value` to back of deque.
@ -322,7 +313,6 @@ class Deque(Sequence):
"""
self._cache.push(value, retry=True)
def appendleft(self, value):
"""Add `value` to front of deque.
@ -338,7 +328,6 @@ class Deque(Sequence):
"""
self._cache.push(value, side='front', retry=True)
def clear(self):
"""Remove all elements from deque.
@ -352,7 +341,6 @@ class Deque(Sequence):
"""
self._cache.clear(retry=True)
def count(self, value):
"""Return number of occurrences of `value` in deque.
@ -371,7 +359,6 @@ class Deque(Sequence):
"""
return sum(1 for item in self if value == item)
def extend(self, iterable):
"""Extend back side of deque with values from `iterable`.
@ -381,7 +368,6 @@ class Deque(Sequence):
for value in iterable:
self.append(value)
def extendleft(self, iterable):
"""Extend front side of deque with value from `iterable`.
@ -396,7 +382,6 @@ class Deque(Sequence):
for value in iterable:
self.appendleft(value)
def peek(self):
"""Peek at value at back of deque.
@ -423,7 +408,6 @@ class Deque(Sequence):
raise IndexError('peek from an empty deque')
return value
def peekleft(self):
"""Peek at value at front of deque.
@ -450,7 +434,6 @@ class Deque(Sequence):
raise IndexError('peek from an empty deque')
return value
def pop(self):
"""Remove and return value at back of deque.
@ -477,7 +460,6 @@ class Deque(Sequence):
raise IndexError('pop from an empty deque')
return value
def popleft(self):
"""Remove and return value at front of deque.
@ -502,7 +484,6 @@ class Deque(Sequence):
raise IndexError('pop from an empty deque')
return value
def remove(self, value):
"""Remove first occurrence of `value` in deque.
@ -540,7 +521,6 @@ class Deque(Sequence):
raise ValueError('deque.remove(value): value not in deque')
def reverse(self):
"""Reverse deque in place.
@ -563,7 +543,6 @@ class Deque(Sequence):
del temp
rmtree(directory)
def rotate(self, steps=1):
"""Rotate deque right by `steps`.
@ -612,9 +591,7 @@ class Deque(Sequence):
else:
self.append(value)
__hash__ = None
__hash__ = None # type: ignore
@contextmanager
def transact(self):
@ -665,6 +642,7 @@ class Index(MutableMapping):
('c', 3)
"""
def __init__(self, *args, **kwargs):
"""Initialize index in directory and update items.
@ -692,7 +670,6 @@ class Index(MutableMapping):
self._cache = Cache(directory, eviction_policy='none')
self.update(*args, **kwargs)
@classmethod
def fromcache(cls, cache, *args, **kwargs):
"""Initialize index using `cache` and update items.
@ -720,19 +697,16 @@ class Index(MutableMapping):
self.update(*args, **kwargs)
return self
@property
def cache(self):
"Cache used by index."
"""Cache used by index."""
return self._cache
@property
def directory(self):
"Directory path where items are stored."
"""Directory path where items are stored."""
return self._cache.directory
def __getitem__(self, key):
"""index.__getitem__(key) <==> index[key]
@ -756,7 +730,6 @@ class Index(MutableMapping):
"""
return self._cache[key]
def __setitem__(self, key, value):
"""index.__setitem__(key, value) <==> index[key] = value
@ -774,7 +747,6 @@ class Index(MutableMapping):
"""
self._cache[key] = value
def __delitem__(self, key):
"""index.__delitem__(key) <==> del index[key]
@ -797,7 +769,6 @@ class Index(MutableMapping):
"""
del self._cache[key]
def setdefault(self, key, default=None):
"""Set and get value for `key` in index using `default`.
@ -822,7 +793,6 @@ class Index(MutableMapping):
except KeyError:
_cache.add(key, default, retry=True)
def peekitem(self, last=True):
"""Peek at key and value item pair in index based on iteration order.
@ -841,7 +811,6 @@ class Index(MutableMapping):
"""
return self._cache.peekitem(last, retry=True)
def pop(self, key, default=ENOVAL):
"""Remove corresponding item for `key` from index and return value.
@ -872,7 +841,6 @@ class Index(MutableMapping):
raise KeyError(key)
return value
def popitem(self, last=True):
"""Remove and return item pair.
@ -907,7 +875,6 @@ class Index(MutableMapping):
return key, value
def push(self, value, prefix=None, side='back'):
"""Push `value` onto `side` of queue in index identified by `prefix`.
@ -939,7 +906,6 @@ class Index(MutableMapping):
"""
return self._cache.push(value, prefix, side, retry=True)
def pull(self, prefix=None, default=(None, None), side='front'):
"""Pull key and value item pair from `side` of queue in index.
@ -980,7 +946,6 @@ class Index(MutableMapping):
"""
return self._cache.pull(prefix, default, side, retry=True)
def clear(self):
"""Remove all items from index.
@ -994,7 +959,6 @@ class Index(MutableMapping):
"""
self._cache.clear(retry=True)
def __iter__(self):
"""index.__iter__() <==> iter(index)
@ -1003,7 +967,6 @@ class Index(MutableMapping):
"""
return iter(self._cache)
def __reversed__(self):
"""index.__reversed__() <==> reversed(index)
@ -1020,7 +983,6 @@ class Index(MutableMapping):
"""
return reversed(self._cache)
def __len__(self):
"""index.__len__() <==> len(index)
@ -1029,7 +991,6 @@ class Index(MutableMapping):
"""
return len(self._cache)
def keys(self):
"""Set-like object providing a view of index keys.
@ -1044,7 +1005,6 @@ class Index(MutableMapping):
"""
return KeysView(self)
def values(self):
"""Set-like object providing a view of index values.
@ -1059,7 +1019,6 @@ class Index(MutableMapping):
"""
return ValuesView(self)
def items(self):
"""Set-like object providing a view of index items.
@ -1074,18 +1033,14 @@ class Index(MutableMapping):
"""
return ItemsView(self)
__hash__ = None
__hash__ = None # type: ignore
def __getstate__(self):
return self.directory
def __setstate__(self, state):
self.__init__(state)
def __eq__(self, other):
"""index.__eq__(other) <==> index == other
@ -1119,7 +1074,6 @@ class Index(MutableMapping):
else:
return all(self[key] == other.get(key, ENOVAL) for key in self)
def __ne__(self, other):
"""index.__ne__(other) <==> index != other
@ -1143,8 +1097,7 @@ class Index(MutableMapping):
"""
return not self == other
def memoize(self, name=None, typed=False):
def memoize(self, name=None, typed=False, ignore=()):
"""Memoizing cache decorator.
Decorator to wrap callable with memoizing function using cache.
@ -1195,11 +1148,11 @@ class Index(MutableMapping):
:param str name: name given for callable (default None, automatic)
:param bool typed: cache different types separately (default False)
:param set ignore: positional or keyword args to ignore (default ())
:return: callable decorator
"""
return self._cache.memoize(name, typed)
return self._cache.memoize(name, typed, ignore=ignore)
@contextmanager
def transact(self):
@ -1228,7 +1181,6 @@ class Index(MutableMapping):
with self._cache.transact(retry=True):
yield
def __repr__(self):
"""index.__repr__() <==> repr(index)

View file

@ -1,5 +1,4 @@
"""Disk Cache Recipes
"""
import functools
@ -12,7 +11,7 @@ import time
from .core import ENOVAL, args_to_key, full_name
class Averager(object):
class Averager:
"""Recipe for calculating a running average.
Sometimes known as "online statistics," the running average maintains the
@ -32,6 +31,7 @@ class Averager(object):
None
"""
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
@ -39,27 +39,30 @@ class Averager(object):
self._tag = tag
def add(self, value):
"Add `value` to average."
"""Add `value` to average."""
with self._cache.transact(retry=True):
total, count = self._cache.get(self._key, default=(0.0, 0))
total += value
count += 1
self._cache.set(
self._key, (total, count), expire=self._expire, tag=self._tag,
self._key,
(total, count),
expire=self._expire,
tag=self._tag,
)
def get(self):
"Get current average or return `None` if count equals zero."
"""Get current average or return `None` if count equals zero."""
total, count = self._cache.get(self._key, default=(0.0, 0), retry=True)
return None if count == 0 else total / count
def pop(self):
"Return current average and delete key."
"""Return current average and delete key."""
total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True)
return None if count == 0 else total / count
class Lock(object):
class Lock:
"""Recipe for cross-process and cross-thread lock.
>>> import diskcache
@ -71,6 +74,7 @@ class Lock(object):
... pass
"""
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
@ -78,7 +82,7 @@ class Lock(object):
self._tag = tag
def acquire(self):
"Acquire lock using spin-lock algorithm."
"""Acquire lock using spin-lock algorithm."""
while True:
added = self._cache.add(
self._key,
@ -92,11 +96,11 @@ class Lock(object):
time.sleep(0.001)
def release(self):
"Release lock by deleting key."
"""Release lock by deleting key."""
self._cache.delete(self._key, retry=True)
def locked(self):
"Return true if the lock is acquired."
"""Return true if the lock is acquired."""
return self._key in self._cache
def __enter__(self):
@ -106,7 +110,7 @@ class Lock(object):
self.release()
class RLock(object):
class RLock:
"""Recipe for cross-process and cross-thread re-entrant lock.
>>> import diskcache
@ -124,6 +128,7 @@ class RLock(object):
AssertionError: cannot release un-acquired lock
"""
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
@ -131,7 +136,7 @@ class RLock(object):
self._tag = tag
def acquire(self):
"Acquire lock by incrementing count using spin-lock algorithm."
"""Acquire lock by incrementing count using spin-lock algorithm."""
pid = os.getpid()
tid = threading.get_ident()
pid_tid = '{}-{}'.format(pid, tid)
@ -141,14 +146,16 @@ class RLock(object):
value, count = self._cache.get(self._key, default=(None, 0))
if pid_tid == value or count == 0:
self._cache.set(
self._key, (pid_tid, count + 1),
expire=self._expire, tag=self._tag,
self._key,
(pid_tid, count + 1),
expire=self._expire,
tag=self._tag,
)
return
time.sleep(0.001)
def release(self):
"Release lock by decrementing count."
"""Release lock by decrementing count."""
pid = os.getpid()
tid = threading.get_ident()
pid_tid = '{}-{}'.format(pid, tid)
@ -158,8 +165,10 @@ class RLock(object):
is_owned = pid_tid == value and count > 0
assert is_owned, 'cannot release un-acquired lock'
self._cache.set(
self._key, (value, count - 1),
expire=self._expire, tag=self._tag,
self._key,
(value, count - 1),
expire=self._expire,
tag=self._tag,
)
def __enter__(self):
@ -169,7 +178,7 @@ class RLock(object):
self.release()
class BoundedSemaphore(object):
class BoundedSemaphore:
"""Recipe for cross-process and cross-thread bounded semaphore.
>>> import diskcache
@ -187,6 +196,7 @@ class BoundedSemaphore(object):
AssertionError: cannot release un-acquired semaphore
"""
def __init__(self, cache, key, value=1, expire=None, tag=None):
self._cache = cache
self._key = key
@ -195,26 +205,31 @@ class BoundedSemaphore(object):
self._tag = tag
def acquire(self):
"Acquire semaphore by decrementing value using spin-lock algorithm."
"""Acquire semaphore by decrementing value using spin-lock algorithm."""
while True:
with self._cache.transact(retry=True):
value = self._cache.get(self._key, default=self._value)
if value > 0:
self._cache.set(
self._key, value - 1,
expire=self._expire, tag=self._tag,
self._key,
value - 1,
expire=self._expire,
tag=self._tag,
)
return
time.sleep(0.001)
def release(self):
"Release semaphore by incrementing value."
"""Release semaphore by incrementing value."""
with self._cache.transact(retry=True):
value = self._cache.get(self._key, default=self._value)
assert self._value > value, 'cannot release un-acquired semaphore'
value += 1
self._cache.set(
self._key, value, expire=self._expire, tag=self._tag,
self._key,
value,
expire=self._expire,
tag=self._tag,
)
def __enter__(self):
@ -224,8 +239,16 @@ class BoundedSemaphore(object):
self.release()
def throttle(cache, count, seconds, name=None, expire=None, tag=None,
time_func=time.time, sleep_func=time.sleep):
def throttle(
cache,
count,
seconds,
name=None,
expire=None,
tag=None,
time_func=time.time,
sleep_func=time.sleep,
):
"""Decorator to throttle calls to function.
>>> import diskcache, time
@ -242,6 +265,7 @@ def throttle(cache, count, seconds, name=None, expire=None, tag=None,
True
"""
def decorator(func):
rate = count / float(seconds)
key = full_name(func) if name is None else name
@ -298,6 +322,7 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None):
>>> pool.terminate()
"""
def decorator(func):
key = full_name(func) if name is None else name
lock = lock_factory(cache, key, expire=expire, tag=tag)
@ -312,7 +337,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None):
return decorator
def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
def memoize_stampede(
cache, expire, name=None, typed=False, tag=None, beta=1, ignore=()
):
"""Memoizing cache decorator with cache stampede protection.
Cache stampedes are a type of system overload that can occur when parallel
@ -365,16 +392,17 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
:param str name: name given for callable (default None, automatic)
:param bool typed: cache different types separately (default False)
:param str tag: text to associate with arguments (default None)
:param set ignore: positional or keyword args to ignore (default ())
:return: callable decorator
"""
# Caution: Nearly identical code exists in Cache.memoize
def decorator(func):
"Decorator created by memoize call for callable."
"""Decorator created by memoize call for callable."""
base = (full_name(func),) if name is None else (name,)
def timer(*args, **kwargs):
"Time execution of `func` and return result and time delta."
"""Time execution of `func` and return result and time delta."""
start = time.time()
result = func(*args, **kwargs)
delta = time.time() - start
@ -382,10 +410,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
@functools.wraps(func)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
"""Wrapper for callable to cache arguments and return values."""
key = wrapper.__cache_key__(*args, **kwargs)
pair, expire_time = cache.get(
key, default=ENOVAL, expire_time=True, retry=True,
key,
default=ENOVAL,
expire_time=True,
retry=True,
)
if pair is not ENOVAL:
@ -400,7 +431,10 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
thread_key = key + (ENOVAL,)
thread_added = cache.add(
thread_key, None, expire=delta, retry=True,
thread_key,
None,
expire=delta,
retry=True,
)
if thread_added:
@ -409,8 +443,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
with cache:
pair = timer(*args, **kwargs)
cache.set(
key, pair, expire=expire, tag=tag, retry=True,
key,
pair,
expire=expire,
tag=tag,
retry=True,
)
thread = threading.Thread(target=recompute)
thread.daemon = True
thread.start()
@ -422,8 +461,8 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
return pair[0]
def __cache_key__(*args, **kwargs):
"Make key for cache given function arguments."
return args_to_key(base, args, kwargs, typed)
"""Make key for cache given function arguments."""
return args_to_key(base, args, kwargs, typed, ignore)
wrapper.__cache_key__ = __cache_key__
return wrapper