mirror of
https://github.com/SickGear/SickGear.git
synced 2024-12-18 08:43:37 +00:00
Merge branch 'feature/UpdateDiskcache' into dev
This commit is contained in:
JackDandy
commit
4a4f0837c2
8 changed files with 503 additions and 511 deletions
|
|
@ -4,6 +4,7 @@
|
|||
* Add Filelock 3.9.0 (ce3e891)
|
||||
* Remove Lockfile no longer used by Cachecontrol
|
||||
* Update Msgpack 1.0.0 (fa7d744) to 1.0.4 (b5acfd5)
|
||||
* Update diskcache 5.1.0 (40ce0de) to 5.4.0 (1cb1425)
|
||||
* Update Rarfile 4.0 (55fe778) to 4.1a1 (8a72967)
|
||||
* Update UnRar x64 for Windows 6.11 to 6.20
|
||||
* Update Send2Trash 1.5.0 (66afce7) to 1.8.1b0 (0ef9b32)
|
||||
|
|
|
|||
|
|
@ -3,17 +3,31 @@ DiskCache API Reference
|
|||
=======================
|
||||
|
||||
The :doc:`tutorial` provides a helpful walkthrough of most methods.
|
||||
|
||||
"""
|
||||
|
||||
from .core import (
|
||||
Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout
|
||||
DEFAULT_SETTINGS,
|
||||
ENOVAL,
|
||||
EVICTION_POLICY,
|
||||
UNKNOWN,
|
||||
Cache,
|
||||
Disk,
|
||||
EmptyDirWarning,
|
||||
JSONDisk,
|
||||
Timeout,
|
||||
UnknownFileWarning,
|
||||
)
|
||||
from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN
|
||||
from .fanout import FanoutCache
|
||||
from .persistent import Deque, Index
|
||||
from .recipes import Averager, BoundedSemaphore, Lock, RLock
|
||||
from .recipes import barrier, memoize_stampede, throttle
|
||||
from .recipes import (
|
||||
Averager,
|
||||
BoundedSemaphore,
|
||||
Lock,
|
||||
RLock,
|
||||
barrier,
|
||||
memoize_stampede,
|
||||
throttle,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'Averager',
|
||||
|
|
@ -40,14 +54,15 @@ __all__ = [
|
|||
|
||||
try:
|
||||
from .djangocache import DjangoCache # noqa
|
||||
|
||||
__all__.append('DjangoCache')
|
||||
except Exception: # pylint: disable=broad-except
|
||||
except Exception: # pylint: disable=broad-except # pragma: no cover
|
||||
# Django not installed or not setup so ignore.
|
||||
pass
|
||||
|
||||
__title__ = 'diskcache'
|
||||
__version__ = '5.1.0'
|
||||
__build__ = 0x050100
|
||||
__version__ = '5.4.0'
|
||||
__build__ = 0x050400
|
||||
__author__ = 'Grant Jenks'
|
||||
__license__ = 'Apache 2.0'
|
||||
__copyright__ = 'Copyright 2016-2020 Grant Jenks'
|
||||
__copyright__ = 'Copyright 2016-2022 Grant Jenks'
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
"Command line interface to disk cache."
|
||||
"""Command line interface to disk cache."""
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,11 +1,12 @@
|
|||
"Django-compatible disk and file backed cache."
|
||||
"""Django-compatible disk and file backed cache."""
|
||||
|
||||
from functools import wraps
|
||||
|
||||
from django.core.cache.backends.base import BaseCache
|
||||
|
||||
try:
|
||||
from django.core.cache.backends.base import DEFAULT_TIMEOUT
|
||||
except ImportError:
|
||||
except ImportError: # pragma: no cover
|
||||
# For older versions of Django simply use 300 seconds.
|
||||
DEFAULT_TIMEOUT = 300
|
||||
|
||||
|
|
@ -14,7 +15,8 @@ from .fanout import FanoutCache
|
|||
|
||||
|
||||
class DjangoCache(BaseCache):
|
||||
"Django-compatible disk and file backed cache."
|
||||
"""Django-compatible disk and file backed cache."""
|
||||
|
||||
def __init__(self, directory, params):
|
||||
"""Initialize DjangoCache instance.
|
||||
|
||||
|
|
@ -28,13 +30,11 @@ class DjangoCache(BaseCache):
|
|||
options = params.get('OPTIONS', {})
|
||||
self._cache = FanoutCache(directory, shards, timeout, **options)
|
||||
|
||||
|
||||
@property
|
||||
def directory(self):
|
||||
"""Cache directory."""
|
||||
return self._cache.directory
|
||||
|
||||
|
||||
def cache(self, name):
|
||||
"""Return Cache with given `name` in subdirectory.
|
||||
|
||||
|
|
@ -44,7 +44,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.cache(name)
|
||||
|
||||
|
||||
def deque(self, name):
|
||||
"""Return Deque with given `name` in subdirectory.
|
||||
|
||||
|
|
@ -54,7 +53,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.deque(name)
|
||||
|
||||
|
||||
def index(self, name):
|
||||
"""Return Index with given `name` in subdirectory.
|
||||
|
||||
|
|
@ -64,9 +62,16 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.index(name)
|
||||
|
||||
|
||||
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None,
|
||||
read=False, tag=None, retry=True):
|
||||
def add(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
timeout=DEFAULT_TIMEOUT,
|
||||
version=None,
|
||||
read=False,
|
||||
tag=None,
|
||||
retry=True,
|
||||
):
|
||||
"""Set a value in the cache if the key does not already exist. If
|
||||
timeout is given, that timeout will be used for the key; otherwise the
|
||||
default cache timeout will be used.
|
||||
|
|
@ -89,9 +94,16 @@ class DjangoCache(BaseCache):
|
|||
timeout = self.get_backend_timeout(timeout=timeout)
|
||||
return self._cache.add(key, value, timeout, read, tag, retry)
|
||||
|
||||
|
||||
def get(self, key, default=None, version=None, read=False,
|
||||
expire_time=False, tag=False, retry=False):
|
||||
def get(
|
||||
self,
|
||||
key,
|
||||
default=None,
|
||||
version=None,
|
||||
read=False,
|
||||
expire_time=False,
|
||||
tag=False,
|
||||
retry=False,
|
||||
):
|
||||
"""Fetch a given key from the cache. If the key does not exist, return
|
||||
default, which itself defaults to None.
|
||||
|
||||
|
|
@ -111,7 +123,6 @@ class DjangoCache(BaseCache):
|
|||
key = self.make_key(key, version=version)
|
||||
return self._cache.get(key, default, read, expire_time, tag, retry)
|
||||
|
||||
|
||||
def read(self, key, version=None):
|
||||
"""Return file handle corresponding to `key` from Cache.
|
||||
|
||||
|
|
@ -124,9 +135,16 @@ class DjangoCache(BaseCache):
|
|||
key = self.make_key(key, version=version)
|
||||
return self._cache.read(key)
|
||||
|
||||
|
||||
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None,
|
||||
read=False, tag=None, retry=True):
|
||||
def set(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
timeout=DEFAULT_TIMEOUT,
|
||||
version=None,
|
||||
read=False,
|
||||
tag=None,
|
||||
retry=True,
|
||||
):
|
||||
"""Set a value in the cache. If timeout is given, that timeout will be
|
||||
used for the key; otherwise the default cache timeout will be used.
|
||||
|
||||
|
|
@ -146,7 +164,6 @@ class DjangoCache(BaseCache):
|
|||
timeout = self.get_backend_timeout(timeout=timeout)
|
||||
return self._cache.set(key, value, timeout, read, tag, retry)
|
||||
|
||||
|
||||
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True):
|
||||
"""Touch a key in the cache. If timeout is given, that timeout will be
|
||||
used for the key; otherwise the default cache timeout will be used.
|
||||
|
|
@ -164,9 +181,15 @@ class DjangoCache(BaseCache):
|
|||
timeout = self.get_backend_timeout(timeout=timeout)
|
||||
return self._cache.touch(key, timeout, retry)
|
||||
|
||||
|
||||
def pop(self, key, default=None, version=None, expire_time=False,
|
||||
tag=False, retry=True):
|
||||
def pop(
|
||||
self,
|
||||
key,
|
||||
default=None,
|
||||
version=None,
|
||||
expire_time=False,
|
||||
tag=False,
|
||||
retry=True,
|
||||
):
|
||||
"""Remove corresponding item for `key` from cache and return value.
|
||||
|
||||
If `key` is missing, return `default`.
|
||||
|
|
@ -186,7 +209,6 @@ class DjangoCache(BaseCache):
|
|||
key = self.make_key(key, version=version)
|
||||
return self._cache.pop(key, default, expire_time, tag, retry)
|
||||
|
||||
|
||||
def delete(self, key, version=None, retry=True):
|
||||
"""Delete a key from the cache, failing silently.
|
||||
|
||||
|
|
@ -198,8 +220,7 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
# pylint: disable=arguments-differ
|
||||
key = self.make_key(key, version=version)
|
||||
self._cache.delete(key, retry)
|
||||
|
||||
return self._cache.delete(key, retry)
|
||||
|
||||
def incr(self, key, delta=1, version=None, default=None, retry=True):
|
||||
"""Increment value by delta for item with key.
|
||||
|
|
@ -230,7 +251,6 @@ class DjangoCache(BaseCache):
|
|||
except KeyError:
|
||||
raise ValueError("Key '%s' not found" % key) from None
|
||||
|
||||
|
||||
def decr(self, key, delta=1, version=None, default=None, retry=True):
|
||||
"""Decrement value by delta for item with key.
|
||||
|
||||
|
|
@ -259,7 +279,6 @@ class DjangoCache(BaseCache):
|
|||
# pylint: disable=arguments-differ
|
||||
return self.incr(key, -delta, version, default, retry)
|
||||
|
||||
|
||||
def has_key(self, key, version=None):
|
||||
"""Returns True if the key is in the cache and has not expired.
|
||||
|
||||
|
|
@ -271,7 +290,6 @@ class DjangoCache(BaseCache):
|
|||
key = self.make_key(key, version=version)
|
||||
return key in self._cache
|
||||
|
||||
|
||||
def expire(self):
|
||||
"""Remove expired items from cache.
|
||||
|
||||
|
|
@ -280,7 +298,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.expire()
|
||||
|
||||
|
||||
def stats(self, enable=True, reset=False):
|
||||
"""Return cache statistics hits and misses.
|
||||
|
||||
|
|
@ -291,7 +308,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.stats(enable=enable, reset=reset)
|
||||
|
||||
|
||||
def create_tag_index(self):
|
||||
"""Create tag index on cache database.
|
||||
|
||||
|
|
@ -302,7 +318,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
self._cache.create_tag_index()
|
||||
|
||||
|
||||
def drop_tag_index(self):
|
||||
"""Drop tag index on cache database.
|
||||
|
||||
|
|
@ -311,7 +326,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
self._cache.drop_tag_index()
|
||||
|
||||
|
||||
def evict(self, tag):
|
||||
"""Remove items with matching `tag` from cache.
|
||||
|
||||
|
|
@ -321,7 +335,6 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.evict(tag)
|
||||
|
||||
|
||||
def cull(self):
|
||||
"""Cull items from cache until volume is less than size limit.
|
||||
|
||||
|
|
@ -330,18 +343,15 @@ class DjangoCache(BaseCache):
|
|||
"""
|
||||
return self._cache.cull()
|
||||
|
||||
|
||||
def clear(self):
|
||||
"Remove *all* values from the cache at once."
|
||||
"""Remove *all* values from the cache at once."""
|
||||
return self._cache.clear()
|
||||
|
||||
|
||||
def close(self, **kwargs):
|
||||
"Close the cache connection."
|
||||
"""Close the cache connection."""
|
||||
# pylint: disable=unused-argument
|
||||
self._cache.close()
|
||||
|
||||
|
||||
def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT):
|
||||
"""Return seconds to expiration.
|
||||
|
||||
|
|
@ -356,9 +366,15 @@ class DjangoCache(BaseCache):
|
|||
timeout = -1
|
||||
return None if timeout is None else timeout
|
||||
|
||||
|
||||
def memoize(self, name=None, timeout=DEFAULT_TIMEOUT, version=None,
|
||||
typed=False, tag=None):
|
||||
def memoize(
|
||||
self,
|
||||
name=None,
|
||||
timeout=DEFAULT_TIMEOUT,
|
||||
version=None,
|
||||
typed=False,
|
||||
tag=None,
|
||||
ignore=(),
|
||||
):
|
||||
"""Memoizing cache decorator.
|
||||
|
||||
Decorator to wrap callable with memoizing function using cache.
|
||||
|
|
@ -392,6 +408,7 @@ class DjangoCache(BaseCache):
|
|||
:param int version: key version number (default None, cache parameter)
|
||||
:param bool typed: cache different types separately (default False)
|
||||
:param str tag: text to associate with arguments (default None)
|
||||
:param set ignore: positional or keyword args to ignore (default ())
|
||||
:return: callable decorator
|
||||
|
||||
"""
|
||||
|
|
@ -400,12 +417,12 @@ class DjangoCache(BaseCache):
|
|||
raise TypeError('name cannot be callable')
|
||||
|
||||
def decorator(func):
|
||||
"Decorator created by memoize() for callable `func`."
|
||||
"""Decorator created by memoize() for callable `func`."""
|
||||
base = (full_name(func),) if name is None else (name,)
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
"Wrapper for callable to cache arguments and return values."
|
||||
"""Wrapper for callable to cache arguments and return values."""
|
||||
key = wrapper.__cache_key__(*args, **kwargs)
|
||||
result = self.get(key, ENOVAL, version, retry=True)
|
||||
|
||||
|
|
@ -418,14 +435,19 @@ class DjangoCache(BaseCache):
|
|||
)
|
||||
if valid_timeout:
|
||||
self.set(
|
||||
key, result, timeout, version, tag=tag, retry=True,
|
||||
key,
|
||||
result,
|
||||
timeout,
|
||||
version,
|
||||
tag=tag,
|
||||
retry=True,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def __cache_key__(*args, **kwargs):
|
||||
"Make key for cache given function arguments."
|
||||
return args_to_key(base, args, kwargs, typed)
|
||||
"""Make key for cache given function arguments."""
|
||||
return args_to_key(base, args, kwargs, typed, ignore)
|
||||
|
||||
wrapper.__cache_key__ = __cache_key__
|
||||
return wrapper
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"Fanout cache automatically shards keys and values."
|
||||
"""Fanout cache automatically shards keys and values."""
|
||||
|
||||
import contextlib as cl
|
||||
import functools
|
||||
|
|
@ -9,14 +9,16 @@ import sqlite3
|
|||
import tempfile
|
||||
import time
|
||||
|
||||
from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout
|
||||
from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout
|
||||
from .persistent import Deque, Index
|
||||
|
||||
|
||||
class FanoutCache(object):
|
||||
"Cache that shards keys and values."
|
||||
def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk,
|
||||
**settings):
|
||||
class FanoutCache:
|
||||
"""Cache that shards keys and values."""
|
||||
|
||||
def __init__(
|
||||
self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings
|
||||
):
|
||||
"""Initialize cache instance.
|
||||
|
||||
:param str directory: cache directory
|
||||
|
|
@ -36,6 +38,7 @@ class FanoutCache(object):
|
|||
|
||||
self._count = shards
|
||||
self._directory = directory
|
||||
self._disk = disk
|
||||
self._shards = tuple(
|
||||
Cache(
|
||||
directory=op.join(directory, '%03d' % num),
|
||||
|
|
@ -51,20 +54,17 @@ class FanoutCache(object):
|
|||
self._deques = {}
|
||||
self._indexes = {}
|
||||
|
||||
|
||||
@property
|
||||
def directory(self):
|
||||
"""Cache directory."""
|
||||
return self._directory
|
||||
|
||||
|
||||
def __getattr__(self, name):
|
||||
safe_names = {'timeout', 'disk'}
|
||||
valid_name = name in DEFAULT_SETTINGS or name in safe_names
|
||||
assert valid_name, 'cannot access {} in cache shard'.format(name)
|
||||
return getattr(self._shards[0], name)
|
||||
|
||||
|
||||
@cl.contextmanager
|
||||
def transact(self, retry=True):
|
||||
"""Context manager to perform a transaction by locking the cache.
|
||||
|
|
@ -98,7 +98,6 @@ class FanoutCache(object):
|
|||
stack.enter_context(shard_transaction)
|
||||
yield
|
||||
|
||||
|
||||
def set(self, key, value, expire=None, read=False, tag=None, retry=False):
|
||||
"""Set `key` and `value` item in cache.
|
||||
|
||||
|
|
@ -125,7 +124,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return False
|
||||
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""Set `key` and `value` item in cache.
|
||||
|
||||
|
|
@ -139,7 +137,6 @@ class FanoutCache(object):
|
|||
shard = self._shards[index]
|
||||
shard[key] = value
|
||||
|
||||
|
||||
def touch(self, key, expire=None, retry=False):
|
||||
"""Touch `key` in cache and update `expire` time.
|
||||
|
||||
|
|
@ -160,7 +157,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return False
|
||||
|
||||
|
||||
def add(self, key, value, expire=None, read=False, tag=None, retry=False):
|
||||
"""Add `key` and `value` item to cache.
|
||||
|
||||
|
|
@ -192,7 +188,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return False
|
||||
|
||||
|
||||
def incr(self, key, delta=1, default=0, retry=False):
|
||||
"""Increment value by delta for item with key.
|
||||
|
||||
|
|
@ -224,7 +219,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return None
|
||||
|
||||
|
||||
def decr(self, key, delta=1, default=0, retry=False):
|
||||
"""Decrement value by delta for item with key.
|
||||
|
||||
|
|
@ -259,9 +253,15 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return None
|
||||
|
||||
|
||||
def get(self, key, default=None, read=False, expire_time=False, tag=False,
|
||||
retry=False):
|
||||
def get(
|
||||
self,
|
||||
key,
|
||||
default=None,
|
||||
read=False,
|
||||
expire_time=False,
|
||||
tag=False,
|
||||
retry=False,
|
||||
):
|
||||
"""Retrieve value from cache. If `key` is missing, return `default`.
|
||||
|
||||
If database timeout occurs then returns `default` unless `retry` is set
|
||||
|
|
@ -285,7 +285,6 @@ class FanoutCache(object):
|
|||
except (Timeout, sqlite3.OperationalError):
|
||||
return default
|
||||
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""Return corresponding value for `key` from cache.
|
||||
|
||||
|
|
@ -300,7 +299,6 @@ class FanoutCache(object):
|
|||
shard = self._shards[index]
|
||||
return shard[key]
|
||||
|
||||
|
||||
def read(self, key):
|
||||
"""Return file handle corresponding to `key` from cache.
|
||||
|
||||
|
|
@ -314,7 +312,6 @@ class FanoutCache(object):
|
|||
raise KeyError(key)
|
||||
return handle
|
||||
|
||||
|
||||
def __contains__(self, key):
|
||||
"""Return `True` if `key` matching item is found in cache.
|
||||
|
||||
|
|
@ -326,8 +323,9 @@ class FanoutCache(object):
|
|||
shard = self._shards[index]
|
||||
return key in shard
|
||||
|
||||
|
||||
def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501
|
||||
def pop(
|
||||
self, key, default=None, expire_time=False, tag=False, retry=False
|
||||
): # noqa: E501
|
||||
"""Remove corresponding item for `key` from cache and return value.
|
||||
|
||||
If `key` is missing, return `default`.
|
||||
|
|
@ -353,7 +351,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return default
|
||||
|
||||
|
||||
def delete(self, key, retry=False):
|
||||
"""Delete corresponding item for `key` from cache.
|
||||
|
||||
|
|
@ -374,7 +371,6 @@ class FanoutCache(object):
|
|||
except Timeout:
|
||||
return False
|
||||
|
||||
|
||||
def __delitem__(self, key):
|
||||
"""Delete corresponding item for `key` from cache.
|
||||
|
||||
|
|
@ -388,7 +384,6 @@ class FanoutCache(object):
|
|||
shard = self._shards[index]
|
||||
del shard[key]
|
||||
|
||||
|
||||
def check(self, fix=False, retry=False):
|
||||
"""Check database and file system consistency.
|
||||
|
||||
|
|
@ -412,7 +407,6 @@ class FanoutCache(object):
|
|||
warnings = (shard.check(fix, retry) for shard in self._shards)
|
||||
return functools.reduce(operator.iadd, warnings, [])
|
||||
|
||||
|
||||
def expire(self, retry=False):
|
||||
"""Remove expired items from cache.
|
||||
|
||||
|
|
@ -425,7 +419,6 @@ class FanoutCache(object):
|
|||
"""
|
||||
return self._remove('expire', args=(time.time(),), retry=retry)
|
||||
|
||||
|
||||
def create_tag_index(self):
|
||||
"""Create tag index on cache database.
|
||||
|
||||
|
|
@ -437,7 +430,6 @@ class FanoutCache(object):
|
|||
for shard in self._shards:
|
||||
shard.create_tag_index()
|
||||
|
||||
|
||||
def drop_tag_index(self):
|
||||
"""Drop tag index on cache database.
|
||||
|
||||
|
|
@ -447,7 +439,6 @@ class FanoutCache(object):
|
|||
for shard in self._shards:
|
||||
shard.drop_tag_index()
|
||||
|
||||
|
||||
def evict(self, tag, retry=False):
|
||||
"""Remove items with matching `tag` from cache.
|
||||
|
||||
|
|
@ -461,7 +452,6 @@ class FanoutCache(object):
|
|||
"""
|
||||
return self._remove('evict', args=(tag,), retry=retry)
|
||||
|
||||
|
||||
def cull(self, retry=False):
|
||||
"""Cull items from cache until volume is less than size limit.
|
||||
|
||||
|
|
@ -474,7 +464,6 @@ class FanoutCache(object):
|
|||
"""
|
||||
return self._remove('cull', retry=retry)
|
||||
|
||||
|
||||
def clear(self, retry=False):
|
||||
"""Remove all items from cache.
|
||||
|
||||
|
|
@ -487,7 +476,6 @@ class FanoutCache(object):
|
|||
"""
|
||||
return self._remove('clear', retry=retry)
|
||||
|
||||
|
||||
def _remove(self, name, args=(), retry=False):
|
||||
total = 0
|
||||
for shard in self._shards:
|
||||
|
|
@ -502,7 +490,6 @@ class FanoutCache(object):
|
|||
break
|
||||
return total
|
||||
|
||||
|
||||
def stats(self, enable=True, reset=False):
|
||||
"""Return cache statistics hits and misses.
|
||||
|
||||
|
|
@ -516,7 +503,6 @@ class FanoutCache(object):
|
|||
total_misses = sum(misses for _, misses in results)
|
||||
return total_hits, total_misses
|
||||
|
||||
|
||||
def volume(self):
|
||||
"""Return estimated total size of cache on disk.
|
||||
|
||||
|
|
@ -525,49 +511,40 @@ class FanoutCache(object):
|
|||
"""
|
||||
return sum(shard.volume() for shard in self._shards)
|
||||
|
||||
|
||||
def close(self):
|
||||
"Close database connection."
|
||||
"""Close database connection."""
|
||||
for shard in self._shards:
|
||||
shard.close()
|
||||
self._caches.clear()
|
||||
self._deques.clear()
|
||||
self._indexes.clear()
|
||||
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
|
||||
def __exit__(self, *exception):
|
||||
self.close()
|
||||
|
||||
|
||||
def __getstate__(self):
|
||||
return (self._directory, self._count, self.timeout, type(self.disk))
|
||||
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__init__(*state)
|
||||
|
||||
|
||||
def __iter__(self):
|
||||
"Iterate keys in cache including expired items."
|
||||
"""Iterate keys in cache including expired items."""
|
||||
iterators = (iter(shard) for shard in self._shards)
|
||||
return it.chain.from_iterable(iterators)
|
||||
|
||||
|
||||
def __reversed__(self):
|
||||
"Reverse iterate keys in cache including expired items."
|
||||
"""Reverse iterate keys in cache including expired items."""
|
||||
iterators = (reversed(shard) for shard in reversed(self._shards))
|
||||
return it.chain.from_iterable(iterators)
|
||||
|
||||
|
||||
def __len__(self):
|
||||
"Count of items in cache including expired items."
|
||||
"""Count of items in cache including expired items."""
|
||||
return sum(len(shard) for shard in self._shards)
|
||||
|
||||
|
||||
def reset(self, key, value=ENOVAL):
|
||||
"""Reset `key` and `value` item from Settings table.
|
||||
|
||||
|
|
@ -596,7 +573,6 @@ class FanoutCache(object):
|
|||
break
|
||||
return result
|
||||
|
||||
|
||||
def cache(self, name):
|
||||
"""Return Cache with given `name` in subdirectory.
|
||||
|
||||
|
|
@ -622,11 +598,10 @@ class FanoutCache(object):
|
|||
except KeyError:
|
||||
parts = name.split('/')
|
||||
directory = op.join(self._directory, 'cache', *parts)
|
||||
temp = Cache(directory=directory)
|
||||
temp = Cache(directory=directory, disk=self._disk)
|
||||
_caches[name] = temp
|
||||
return temp
|
||||
|
||||
|
||||
def deque(self, name):
|
||||
"""Return Deque with given `name` in subdirectory.
|
||||
|
||||
|
|
@ -651,10 +626,10 @@ class FanoutCache(object):
|
|||
except KeyError:
|
||||
parts = name.split('/')
|
||||
directory = op.join(self._directory, 'deque', *parts)
|
||||
temp = Deque(directory=directory)
|
||||
_deques[name] = temp
|
||||
return temp
|
||||
|
||||
cache = Cache(directory=directory, disk=self._disk)
|
||||
deque = Deque.fromcache(cache)
|
||||
_deques[name] = deque
|
||||
return deque
|
||||
|
||||
def index(self, name):
|
||||
"""Return Index with given `name` in subdirectory.
|
||||
|
|
@ -683,9 +658,10 @@ class FanoutCache(object):
|
|||
except KeyError:
|
||||
parts = name.split('/')
|
||||
directory = op.join(self._directory, 'index', *parts)
|
||||
temp = Index(directory)
|
||||
_indexes[name] = temp
|
||||
return temp
|
||||
cache = Cache(directory=directory, disk=self._disk)
|
||||
index = Index.fromcache(cache)
|
||||
_indexes[name] = index
|
||||
return index
|
||||
|
||||
|
||||
FanoutCache.memoize = Cache.memoize
|
||||
FanoutCache.memoize = Cache.memoize # type: ignore
|
||||
|
|
|
|||
|
|
@ -1,22 +1,27 @@
|
|||
"""Persistent Data Types
|
||||
|
||||
"""
|
||||
|
||||
import operator as op
|
||||
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from collections.abc import MutableMapping, Sequence
|
||||
from collections.abc import KeysView, ValuesView, ItemsView
|
||||
from collections.abc import (
|
||||
ItemsView,
|
||||
KeysView,
|
||||
MutableMapping,
|
||||
Sequence,
|
||||
ValuesView,
|
||||
)
|
||||
from contextlib import contextmanager
|
||||
from shutil import rmtree
|
||||
|
||||
from .core import Cache, ENOVAL
|
||||
from .core import ENOVAL, Cache
|
||||
|
||||
|
||||
def _make_compare(seq_op, doc):
|
||||
"Make compare method with Sequence semantics."
|
||||
"""Make compare method with Sequence semantics."""
|
||||
|
||||
def compare(self, that):
|
||||
"Compare method for deque and sequence."
|
||||
"""Compare method for deque and sequence."""
|
||||
if not isinstance(that, Sequence):
|
||||
return NotImplemented
|
||||
|
||||
|
|
@ -70,6 +75,7 @@ class Deque(Sequence):
|
|||
[3, 2, 1, 0, 0, -1, -2, -3]
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, iterable=(), directory=None):
|
||||
"""Initialize deque instance.
|
||||
|
||||
|
|
@ -81,9 +87,7 @@ class Deque(Sequence):
|
|||
|
||||
"""
|
||||
self._cache = Cache(directory, eviction_policy='none')
|
||||
with self.transact():
|
||||
self.extend(iterable)
|
||||
|
||||
self.extend(iterable)
|
||||
|
||||
@classmethod
|
||||
def fromcache(cls, cache, iterable=()):
|
||||
|
|
@ -111,19 +115,16 @@ class Deque(Sequence):
|
|||
self.extend(iterable)
|
||||
return self
|
||||
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
"Cache used by deque."
|
||||
"""Cache used by deque."""
|
||||
return self._cache
|
||||
|
||||
|
||||
@property
|
||||
def directory(self):
|
||||
"Directory path where deque is stored."
|
||||
"""Directory path where deque is stored."""
|
||||
return self._cache.directory
|
||||
|
||||
|
||||
def _index(self, index, func):
|
||||
len_self = len(self)
|
||||
|
||||
|
|
@ -154,7 +155,6 @@ class Deque(Sequence):
|
|||
|
||||
raise IndexError('deque index out of range')
|
||||
|
||||
|
||||
def __getitem__(self, index):
|
||||
"""deque.__getitem__(index) <==> deque[index]
|
||||
|
||||
|
|
@ -177,7 +177,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
return self._index(index, self._cache.__getitem__)
|
||||
|
||||
|
||||
def __setitem__(self, index, value):
|
||||
"""deque.__setitem__(index, value) <==> deque[index] = value
|
||||
|
||||
|
|
@ -196,10 +195,11 @@ class Deque(Sequence):
|
|||
:raises IndexError: if index out of range
|
||||
|
||||
"""
|
||||
|
||||
def _set_value(key):
|
||||
return self._cache.__setitem__(key, value)
|
||||
self._index(index, _set_value)
|
||||
|
||||
self._index(index, _set_value)
|
||||
|
||||
def __delitem__(self, index):
|
||||
"""deque.__delitem__(index) <==> del deque[index]
|
||||
|
|
@ -220,7 +220,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
self._index(index, self._cache.__delitem__)
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
"""deque.__repr__() <==> repr(deque)
|
||||
|
||||
|
|
@ -230,7 +229,6 @@ class Deque(Sequence):
|
|||
name = type(self).__name__
|
||||
return '{0}(directory={1!r})'.format(name, self.directory)
|
||||
|
||||
|
||||
__eq__ = _make_compare(op.eq, 'equal to')
|
||||
__ne__ = _make_compare(op.ne, 'not equal to')
|
||||
__lt__ = _make_compare(op.lt, 'less than')
|
||||
|
|
@ -238,7 +236,6 @@ class Deque(Sequence):
|
|||
__le__ = _make_compare(op.le, 'less than or equal to')
|
||||
__ge__ = _make_compare(op.ge, 'greater than or equal to')
|
||||
|
||||
|
||||
def __iadd__(self, iterable):
|
||||
"""deque.__iadd__(iterable) <==> deque += iterable
|
||||
|
||||
|
|
@ -251,7 +248,6 @@ class Deque(Sequence):
|
|||
self.extend(iterable)
|
||||
return self
|
||||
|
||||
|
||||
def __iter__(self):
|
||||
"""deque.__iter__() <==> iter(deque)
|
||||
|
||||
|
|
@ -266,7 +262,6 @@ class Deque(Sequence):
|
|||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def __len__(self):
|
||||
"""deque.__len__() <==> len(deque)
|
||||
|
||||
|
|
@ -275,7 +270,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
return len(self._cache)
|
||||
|
||||
|
||||
def __reversed__(self):
|
||||
"""deque.__reversed__() <==> reversed(deque)
|
||||
|
||||
|
|
@ -298,15 +292,12 @@ class Deque(Sequence):
|
|||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def __getstate__(self):
|
||||
return self.directory
|
||||
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__init__(directory=state)
|
||||
|
||||
|
||||
def append(self, value):
|
||||
"""Add `value` to back of deque.
|
||||
|
||||
|
|
@ -322,7 +313,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
self._cache.push(value, retry=True)
|
||||
|
||||
|
||||
def appendleft(self, value):
|
||||
"""Add `value` to front of deque.
|
||||
|
||||
|
|
@ -338,7 +328,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
self._cache.push(value, side='front', retry=True)
|
||||
|
||||
|
||||
def clear(self):
|
||||
"""Remove all elements from deque.
|
||||
|
||||
|
|
@ -352,7 +341,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
self._cache.clear(retry=True)
|
||||
|
||||
|
||||
def count(self, value):
|
||||
"""Return number of occurrences of `value` in deque.
|
||||
|
||||
|
|
@ -371,7 +359,6 @@ class Deque(Sequence):
|
|||
"""
|
||||
return sum(1 for item in self if value == item)
|
||||
|
||||
|
||||
def extend(self, iterable):
|
||||
"""Extend back side of deque with values from `iterable`.
|
||||
|
||||
|
|
@ -381,7 +368,6 @@ class Deque(Sequence):
|
|||
for value in iterable:
|
||||
self.append(value)
|
||||
|
||||
|
||||
def extendleft(self, iterable):
|
||||
"""Extend front side of deque with value from `iterable`.
|
||||
|
||||
|
|
@ -396,7 +382,6 @@ class Deque(Sequence):
|
|||
for value in iterable:
|
||||
self.appendleft(value)
|
||||
|
||||
|
||||
def peek(self):
|
||||
"""Peek at value at back of deque.
|
||||
|
||||
|
|
@ -423,7 +408,6 @@ class Deque(Sequence):
|
|||
raise IndexError('peek from an empty deque')
|
||||
return value
|
||||
|
||||
|
||||
def peekleft(self):
|
||||
"""Peek at value at front of deque.
|
||||
|
||||
|
|
@ -450,7 +434,6 @@ class Deque(Sequence):
|
|||
raise IndexError('peek from an empty deque')
|
||||
return value
|
||||
|
||||
|
||||
def pop(self):
|
||||
"""Remove and return value at back of deque.
|
||||
|
||||
|
|
@ -477,7 +460,6 @@ class Deque(Sequence):
|
|||
raise IndexError('pop from an empty deque')
|
||||
return value
|
||||
|
||||
|
||||
def popleft(self):
|
||||
"""Remove and return value at front of deque.
|
||||
|
||||
|
|
@ -502,7 +484,6 @@ class Deque(Sequence):
|
|||
raise IndexError('pop from an empty deque')
|
||||
return value
|
||||
|
||||
|
||||
def remove(self, value):
|
||||
"""Remove first occurrence of `value` in deque.
|
||||
|
||||
|
|
@ -540,7 +521,6 @@ class Deque(Sequence):
|
|||
|
||||
raise ValueError('deque.remove(value): value not in deque')
|
||||
|
||||
|
||||
def reverse(self):
|
||||
"""Reverse deque in place.
|
||||
|
||||
|
|
@ -563,7 +543,6 @@ class Deque(Sequence):
|
|||
del temp
|
||||
rmtree(directory)
|
||||
|
||||
|
||||
def rotate(self, steps=1):
|
||||
"""Rotate deque right by `steps`.
|
||||
|
||||
|
|
@ -612,9 +591,7 @@ class Deque(Sequence):
|
|||
else:
|
||||
self.append(value)
|
||||
|
||||
|
||||
__hash__ = None
|
||||
|
||||
__hash__ = None # type: ignore
|
||||
|
||||
@contextmanager
|
||||
def transact(self):
|
||||
|
|
@ -665,6 +642,7 @@ class Index(MutableMapping):
|
|||
('c', 3)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""Initialize index in directory and update items.
|
||||
|
||||
|
|
@ -692,7 +670,6 @@ class Index(MutableMapping):
|
|||
self._cache = Cache(directory, eviction_policy='none')
|
||||
self.update(*args, **kwargs)
|
||||
|
||||
|
||||
@classmethod
|
||||
def fromcache(cls, cache, *args, **kwargs):
|
||||
"""Initialize index using `cache` and update items.
|
||||
|
|
@ -720,19 +697,16 @@ class Index(MutableMapping):
|
|||
self.update(*args, **kwargs)
|
||||
return self
|
||||
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
"Cache used by index."
|
||||
"""Cache used by index."""
|
||||
return self._cache
|
||||
|
||||
|
||||
@property
|
||||
def directory(self):
|
||||
"Directory path where items are stored."
|
||||
"""Directory path where items are stored."""
|
||||
return self._cache.directory
|
||||
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""index.__getitem__(key) <==> index[key]
|
||||
|
||||
|
|
@ -756,7 +730,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return self._cache[key]
|
||||
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""index.__setitem__(key, value) <==> index[key] = value
|
||||
|
||||
|
|
@ -774,7 +747,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
self._cache[key] = value
|
||||
|
||||
|
||||
def __delitem__(self, key):
|
||||
"""index.__delitem__(key) <==> del index[key]
|
||||
|
||||
|
|
@ -797,7 +769,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
del self._cache[key]
|
||||
|
||||
|
||||
def setdefault(self, key, default=None):
|
||||
"""Set and get value for `key` in index using `default`.
|
||||
|
||||
|
|
@ -822,7 +793,6 @@ class Index(MutableMapping):
|
|||
except KeyError:
|
||||
_cache.add(key, default, retry=True)
|
||||
|
||||
|
||||
def peekitem(self, last=True):
|
||||
"""Peek at key and value item pair in index based on iteration order.
|
||||
|
||||
|
|
@ -841,7 +811,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return self._cache.peekitem(last, retry=True)
|
||||
|
||||
|
||||
def pop(self, key, default=ENOVAL):
|
||||
"""Remove corresponding item for `key` from index and return value.
|
||||
|
||||
|
|
@ -872,7 +841,6 @@ class Index(MutableMapping):
|
|||
raise KeyError(key)
|
||||
return value
|
||||
|
||||
|
||||
def popitem(self, last=True):
|
||||
"""Remove and return item pair.
|
||||
|
||||
|
|
@ -907,7 +875,6 @@ class Index(MutableMapping):
|
|||
|
||||
return key, value
|
||||
|
||||
|
||||
def push(self, value, prefix=None, side='back'):
|
||||
"""Push `value` onto `side` of queue in index identified by `prefix`.
|
||||
|
||||
|
|
@ -939,7 +906,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return self._cache.push(value, prefix, side, retry=True)
|
||||
|
||||
|
||||
def pull(self, prefix=None, default=(None, None), side='front'):
|
||||
"""Pull key and value item pair from `side` of queue in index.
|
||||
|
||||
|
|
@ -980,7 +946,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return self._cache.pull(prefix, default, side, retry=True)
|
||||
|
||||
|
||||
def clear(self):
|
||||
"""Remove all items from index.
|
||||
|
||||
|
|
@ -994,7 +959,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
self._cache.clear(retry=True)
|
||||
|
||||
|
||||
def __iter__(self):
|
||||
"""index.__iter__() <==> iter(index)
|
||||
|
||||
|
|
@ -1003,7 +967,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return iter(self._cache)
|
||||
|
||||
|
||||
def __reversed__(self):
|
||||
"""index.__reversed__() <==> reversed(index)
|
||||
|
||||
|
|
@ -1020,7 +983,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return reversed(self._cache)
|
||||
|
||||
|
||||
def __len__(self):
|
||||
"""index.__len__() <==> len(index)
|
||||
|
||||
|
|
@ -1029,7 +991,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return len(self._cache)
|
||||
|
||||
|
||||
def keys(self):
|
||||
"""Set-like object providing a view of index keys.
|
||||
|
||||
|
|
@ -1044,7 +1005,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return KeysView(self)
|
||||
|
||||
|
||||
def values(self):
|
||||
"""Set-like object providing a view of index values.
|
||||
|
||||
|
|
@ -1059,7 +1019,6 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return ValuesView(self)
|
||||
|
||||
|
||||
def items(self):
|
||||
"""Set-like object providing a view of index items.
|
||||
|
||||
|
|
@ -1074,18 +1033,14 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return ItemsView(self)
|
||||
|
||||
|
||||
__hash__ = None
|
||||
|
||||
__hash__ = None # type: ignore
|
||||
|
||||
def __getstate__(self):
|
||||
return self.directory
|
||||
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__init__(state)
|
||||
|
||||
|
||||
def __eq__(self, other):
|
||||
"""index.__eq__(other) <==> index == other
|
||||
|
||||
|
|
@ -1119,7 +1074,6 @@ class Index(MutableMapping):
|
|||
else:
|
||||
return all(self[key] == other.get(key, ENOVAL) for key in self)
|
||||
|
||||
|
||||
def __ne__(self, other):
|
||||
"""index.__ne__(other) <==> index != other
|
||||
|
||||
|
|
@ -1143,8 +1097,7 @@ class Index(MutableMapping):
|
|||
"""
|
||||
return not self == other
|
||||
|
||||
|
||||
def memoize(self, name=None, typed=False):
|
||||
def memoize(self, name=None, typed=False, ignore=()):
|
||||
"""Memoizing cache decorator.
|
||||
|
||||
Decorator to wrap callable with memoizing function using cache.
|
||||
|
|
@ -1195,11 +1148,11 @@ class Index(MutableMapping):
|
|||
|
||||
:param str name: name given for callable (default None, automatic)
|
||||
:param bool typed: cache different types separately (default False)
|
||||
:param set ignore: positional or keyword args to ignore (default ())
|
||||
:return: callable decorator
|
||||
|
||||
"""
|
||||
return self._cache.memoize(name, typed)
|
||||
|
||||
return self._cache.memoize(name, typed, ignore=ignore)
|
||||
|
||||
@contextmanager
|
||||
def transact(self):
|
||||
|
|
@ -1228,7 +1181,6 @@ class Index(MutableMapping):
|
|||
with self._cache.transact(retry=True):
|
||||
yield
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
"""index.__repr__() <==> repr(index)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
"""Disk Cache Recipes
|
||||
|
||||
"""
|
||||
|
||||
import functools
|
||||
|
|
@ -12,7 +11,7 @@ import time
|
|||
from .core import ENOVAL, args_to_key, full_name
|
||||
|
||||
|
||||
class Averager(object):
|
||||
class Averager:
|
||||
"""Recipe for calculating a running average.
|
||||
|
||||
Sometimes known as "online statistics," the running average maintains the
|
||||
|
|
@ -32,6 +31,7 @@ class Averager(object):
|
|||
None
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cache, key, expire=None, tag=None):
|
||||
self._cache = cache
|
||||
self._key = key
|
||||
|
|
@ -39,27 +39,30 @@ class Averager(object):
|
|||
self._tag = tag
|
||||
|
||||
def add(self, value):
|
||||
"Add `value` to average."
|
||||
"""Add `value` to average."""
|
||||
with self._cache.transact(retry=True):
|
||||
total, count = self._cache.get(self._key, default=(0.0, 0))
|
||||
total += value
|
||||
count += 1
|
||||
self._cache.set(
|
||||
self._key, (total, count), expire=self._expire, tag=self._tag,
|
||||
self._key,
|
||||
(total, count),
|
||||
expire=self._expire,
|
||||
tag=self._tag,
|
||||
)
|
||||
|
||||
def get(self):
|
||||
"Get current average or return `None` if count equals zero."
|
||||
"""Get current average or return `None` if count equals zero."""
|
||||
total, count = self._cache.get(self._key, default=(0.0, 0), retry=True)
|
||||
return None if count == 0 else total / count
|
||||
|
||||
def pop(self):
|
||||
"Return current average and delete key."
|
||||
"""Return current average and delete key."""
|
||||
total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True)
|
||||
return None if count == 0 else total / count
|
||||
|
||||
|
||||
class Lock(object):
|
||||
class Lock:
|
||||
"""Recipe for cross-process and cross-thread lock.
|
||||
|
||||
>>> import diskcache
|
||||
|
|
@ -71,6 +74,7 @@ class Lock(object):
|
|||
... pass
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cache, key, expire=None, tag=None):
|
||||
self._cache = cache
|
||||
self._key = key
|
||||
|
|
@ -78,7 +82,7 @@ class Lock(object):
|
|||
self._tag = tag
|
||||
|
||||
def acquire(self):
|
||||
"Acquire lock using spin-lock algorithm."
|
||||
"""Acquire lock using spin-lock algorithm."""
|
||||
while True:
|
||||
added = self._cache.add(
|
||||
self._key,
|
||||
|
|
@ -92,11 +96,11 @@ class Lock(object):
|
|||
time.sleep(0.001)
|
||||
|
||||
def release(self):
|
||||
"Release lock by deleting key."
|
||||
"""Release lock by deleting key."""
|
||||
self._cache.delete(self._key, retry=True)
|
||||
|
||||
def locked(self):
|
||||
"Return true if the lock is acquired."
|
||||
"""Return true if the lock is acquired."""
|
||||
return self._key in self._cache
|
||||
|
||||
def __enter__(self):
|
||||
|
|
@ -106,7 +110,7 @@ class Lock(object):
|
|||
self.release()
|
||||
|
||||
|
||||
class RLock(object):
|
||||
class RLock:
|
||||
"""Recipe for cross-process and cross-thread re-entrant lock.
|
||||
|
||||
>>> import diskcache
|
||||
|
|
@ -124,6 +128,7 @@ class RLock(object):
|
|||
AssertionError: cannot release un-acquired lock
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cache, key, expire=None, tag=None):
|
||||
self._cache = cache
|
||||
self._key = key
|
||||
|
|
@ -131,7 +136,7 @@ class RLock(object):
|
|||
self._tag = tag
|
||||
|
||||
def acquire(self):
|
||||
"Acquire lock by incrementing count using spin-lock algorithm."
|
||||
"""Acquire lock by incrementing count using spin-lock algorithm."""
|
||||
pid = os.getpid()
|
||||
tid = threading.get_ident()
|
||||
pid_tid = '{}-{}'.format(pid, tid)
|
||||
|
|
@ -141,14 +146,16 @@ class RLock(object):
|
|||
value, count = self._cache.get(self._key, default=(None, 0))
|
||||
if pid_tid == value or count == 0:
|
||||
self._cache.set(
|
||||
self._key, (pid_tid, count + 1),
|
||||
expire=self._expire, tag=self._tag,
|
||||
self._key,
|
||||
(pid_tid, count + 1),
|
||||
expire=self._expire,
|
||||
tag=self._tag,
|
||||
)
|
||||
return
|
||||
time.sleep(0.001)
|
||||
|
||||
def release(self):
|
||||
"Release lock by decrementing count."
|
||||
"""Release lock by decrementing count."""
|
||||
pid = os.getpid()
|
||||
tid = threading.get_ident()
|
||||
pid_tid = '{}-{}'.format(pid, tid)
|
||||
|
|
@ -158,8 +165,10 @@ class RLock(object):
|
|||
is_owned = pid_tid == value and count > 0
|
||||
assert is_owned, 'cannot release un-acquired lock'
|
||||
self._cache.set(
|
||||
self._key, (value, count - 1),
|
||||
expire=self._expire, tag=self._tag,
|
||||
self._key,
|
||||
(value, count - 1),
|
||||
expire=self._expire,
|
||||
tag=self._tag,
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
|
|
@ -169,7 +178,7 @@ class RLock(object):
|
|||
self.release()
|
||||
|
||||
|
||||
class BoundedSemaphore(object):
|
||||
class BoundedSemaphore:
|
||||
"""Recipe for cross-process and cross-thread bounded semaphore.
|
||||
|
||||
>>> import diskcache
|
||||
|
|
@ -187,6 +196,7 @@ class BoundedSemaphore(object):
|
|||
AssertionError: cannot release un-acquired semaphore
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cache, key, value=1, expire=None, tag=None):
|
||||
self._cache = cache
|
||||
self._key = key
|
||||
|
|
@ -195,26 +205,31 @@ class BoundedSemaphore(object):
|
|||
self._tag = tag
|
||||
|
||||
def acquire(self):
|
||||
"Acquire semaphore by decrementing value using spin-lock algorithm."
|
||||
"""Acquire semaphore by decrementing value using spin-lock algorithm."""
|
||||
while True:
|
||||
with self._cache.transact(retry=True):
|
||||
value = self._cache.get(self._key, default=self._value)
|
||||
if value > 0:
|
||||
self._cache.set(
|
||||
self._key, value - 1,
|
||||
expire=self._expire, tag=self._tag,
|
||||
self._key,
|
||||
value - 1,
|
||||
expire=self._expire,
|
||||
tag=self._tag,
|
||||
)
|
||||
return
|
||||
time.sleep(0.001)
|
||||
|
||||
def release(self):
|
||||
"Release semaphore by incrementing value."
|
||||
"""Release semaphore by incrementing value."""
|
||||
with self._cache.transact(retry=True):
|
||||
value = self._cache.get(self._key, default=self._value)
|
||||
assert self._value > value, 'cannot release un-acquired semaphore'
|
||||
value += 1
|
||||
self._cache.set(
|
||||
self._key, value, expire=self._expire, tag=self._tag,
|
||||
self._key,
|
||||
value,
|
||||
expire=self._expire,
|
||||
tag=self._tag,
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
|
|
@ -224,8 +239,16 @@ class BoundedSemaphore(object):
|
|||
self.release()
|
||||
|
||||
|
||||
def throttle(cache, count, seconds, name=None, expire=None, tag=None,
|
||||
time_func=time.time, sleep_func=time.sleep):
|
||||
def throttle(
|
||||
cache,
|
||||
count,
|
||||
seconds,
|
||||
name=None,
|
||||
expire=None,
|
||||
tag=None,
|
||||
time_func=time.time,
|
||||
sleep_func=time.sleep,
|
||||
):
|
||||
"""Decorator to throttle calls to function.
|
||||
|
||||
>>> import diskcache, time
|
||||
|
|
@ -242,6 +265,7 @@ def throttle(cache, count, seconds, name=None, expire=None, tag=None,
|
|||
True
|
||||
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
rate = count / float(seconds)
|
||||
key = full_name(func) if name is None else name
|
||||
|
|
@ -298,6 +322,7 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None):
|
|||
>>> pool.terminate()
|
||||
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
key = full_name(func) if name is None else name
|
||||
lock = lock_factory(cache, key, expire=expire, tag=tag)
|
||||
|
|
@ -312,7 +337,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None):
|
|||
return decorator
|
||||
|
||||
|
||||
def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
||||
def memoize_stampede(
|
||||
cache, expire, name=None, typed=False, tag=None, beta=1, ignore=()
|
||||
):
|
||||
"""Memoizing cache decorator with cache stampede protection.
|
||||
|
||||
Cache stampedes are a type of system overload that can occur when parallel
|
||||
|
|
@ -365,16 +392,17 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
|||
:param str name: name given for callable (default None, automatic)
|
||||
:param bool typed: cache different types separately (default False)
|
||||
:param str tag: text to associate with arguments (default None)
|
||||
:param set ignore: positional or keyword args to ignore (default ())
|
||||
:return: callable decorator
|
||||
|
||||
"""
|
||||
# Caution: Nearly identical code exists in Cache.memoize
|
||||
def decorator(func):
|
||||
"Decorator created by memoize call for callable."
|
||||
"""Decorator created by memoize call for callable."""
|
||||
base = (full_name(func),) if name is None else (name,)
|
||||
|
||||
def timer(*args, **kwargs):
|
||||
"Time execution of `func` and return result and time delta."
|
||||
"""Time execution of `func` and return result and time delta."""
|
||||
start = time.time()
|
||||
result = func(*args, **kwargs)
|
||||
delta = time.time() - start
|
||||
|
|
@ -382,10 +410,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
|||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
"Wrapper for callable to cache arguments and return values."
|
||||
"""Wrapper for callable to cache arguments and return values."""
|
||||
key = wrapper.__cache_key__(*args, **kwargs)
|
||||
pair, expire_time = cache.get(
|
||||
key, default=ENOVAL, expire_time=True, retry=True,
|
||||
key,
|
||||
default=ENOVAL,
|
||||
expire_time=True,
|
||||
retry=True,
|
||||
)
|
||||
|
||||
if pair is not ENOVAL:
|
||||
|
|
@ -400,7 +431,10 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
|||
|
||||
thread_key = key + (ENOVAL,)
|
||||
thread_added = cache.add(
|
||||
thread_key, None, expire=delta, retry=True,
|
||||
thread_key,
|
||||
None,
|
||||
expire=delta,
|
||||
retry=True,
|
||||
)
|
||||
|
||||
if thread_added:
|
||||
|
|
@ -409,8 +443,13 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
|||
with cache:
|
||||
pair = timer(*args, **kwargs)
|
||||
cache.set(
|
||||
key, pair, expire=expire, tag=tag, retry=True,
|
||||
key,
|
||||
pair,
|
||||
expire=expire,
|
||||
tag=tag,
|
||||
retry=True,
|
||||
)
|
||||
|
||||
thread = threading.Thread(target=recompute)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
|
@ -422,8 +461,8 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
|
|||
return pair[0]
|
||||
|
||||
def __cache_key__(*args, **kwargs):
|
||||
"Make key for cache given function arguments."
|
||||
return args_to_key(base, args, kwargs, typed)
|
||||
"""Make key for cache given function arguments."""
|
||||
return args_to_key(base, args, kwargs, typed, ignore)
|
||||
|
||||
wrapper.__cache_key__ = __cache_key__
|
||||
return wrapper
|
||||
|
|
|
|||
Loading…
Reference in a new issue