diff --git a/CHANGES.md b/CHANGES.md index 995ec640..ff81086c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ ### 3.29.0 (2023-xx-xx xx:xx:00 UTC) +* Update diskcache 5.4.0 (1cb1425) to 5.6.1 (4d30686) * Update filelock 3.9.0 (ce3e891) to 3.11.0 (d3241b9) * Update Msgpack 1.0.4 (b5acfd5) to 1.0.5 (0516c2c) * Update Requests library 2.28.1 (ec553c2) to 2.29.0 (87d63de) diff --git a/lib/diskcache/__init__.py b/lib/diskcache/__init__.py index 2355128c..1931a0dd 100644 --- a/lib/diskcache/__init__.py +++ b/lib/diskcache/__init__.py @@ -61,8 +61,8 @@ except Exception: # pylint: disable=broad-except # pragma: no cover pass __title__ = 'diskcache' -__version__ = '5.4.0' -__build__ = 0x050400 +__version__ = '5.6.1' +__build__ = 0x050601 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2022 Grant Jenks' +__copyright__ = 'Copyright 2016-2023 Grant Jenks' diff --git a/lib/diskcache/core.py b/lib/diskcache/core.py index fadd8490..85f2e72e 100644 --- a/lib/diskcache/core.py +++ b/lib/diskcache/core.py @@ -50,14 +50,14 @@ DEFAULT_SETTINGS = { 'statistics': 0, # False 'tag_index': 0, # False 'eviction_policy': 'least-recently-stored', - 'size_limit': 2 ** 30, # 1gb + 'size_limit': 2**30, # 1gb 'cull_limit': 10, 'sqlite_auto_vacuum': 1, # FULL - 'sqlite_cache_size': 2 ** 13, # 8,192 pages + 'sqlite_cache_size': 2**13, # 8,192 pages 'sqlite_journal_mode': 'wal', - 'sqlite_mmap_size': 2 ** 26, # 64mb + 'sqlite_mmap_size': 2**26, # 64mb 'sqlite_synchronous': 1, # NORMAL - 'disk_min_file_size': 2 ** 15, # 32kb + 'disk_min_file_size': 2**15, # 32kb 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } @@ -171,7 +171,7 @@ class Disk: :return: corresponding Python key """ - # pylint: disable=no-self-use,unidiomatic-typecheck + # pylint: disable=unidiomatic-typecheck if raw: return bytes(key) if type(key) is sqlite3.Binary else key else: @@ -213,7 +213,7 @@ class Disk: size = op.getsize(full_path) return size, MODE_TEXT, filename, None elif read: - reader = ft.partial(value.read, 2 ** 22) + reader = ft.partial(value.read, 2**22) filename, full_path = self.filename(key, value) iterator = iter(reader, b'') size = self._write(full_path, iterator, 'xb') @@ -229,7 +229,6 @@ class Disk: return len(result), MODE_PICKLE, filename, None def _write(self, full_path, iterator, mode, encoding=None): - # pylint: disable=no-self-use full_dir, _ = op.split(full_path) for count in range(1, 11): @@ -265,7 +264,7 @@ class Disk: :raises: IOError if the value cannot be read """ - # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with + # pylint: disable=unidiomatic-typecheck,consider-using-with if mode == MODE_RAW: return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: @@ -435,6 +434,7 @@ class Cache: if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) @@ -1380,6 +1380,7 @@ class Cache: :raises Timeout: if database timeout occurs """ + # pylint: disable=unnecessary-dunder-call try: return self.__delitem__(key, retry=retry) except KeyError: diff --git a/lib/diskcache/djangocache.py b/lib/diskcache/djangocache.py index 8bf85ce6..5dc8ce2c 100644 --- a/lib/diskcache/djangocache.py +++ b/lib/diskcache/djangocache.py @@ -44,14 +44,15 @@ class DjangoCache(BaseCache): """ return self._cache.cache(name) - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ - return self._cache.deque(name) + return self._cache.deque(name, maxlen=maxlen) def index(self, name): """Return Index with given `name` in subdirectory. diff --git a/lib/diskcache/fanout.py b/lib/diskcache/fanout.py index dc5240c5..9822ee4e 100644 --- a/lib/diskcache/fanout.py +++ b/lib/diskcache/fanout.py @@ -30,6 +30,7 @@ class FanoutCache: """ if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) @@ -45,7 +46,7 @@ class FanoutCache: timeout=timeout, disk=disk, size_limit=size_limit, - **settings + **settings, ) for num in range(shards) ) @@ -573,9 +574,11 @@ class FanoutCache: break return result - def cache(self, name): + def cache(self, name, timeout=60, disk=None, **settings): """Return Cache with given `name` in subdirectory. + If disk is none (default), uses the fanout cache disk. + >>> fanout_cache = FanoutCache() >>> cache = fanout_cache.cache('test') >>> cache.set('abc', 123) @@ -588,6 +591,9 @@ class FanoutCache: True :param str name: subdirectory name for Cache + :param float timeout: SQLite connection timeout + :param disk: Disk type or subclass for serialization + :param settings: any of DEFAULT_SETTINGS :return: Cache with given name """ @@ -598,11 +604,16 @@ class FanoutCache: except KeyError: parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) - temp = Cache(directory=directory, disk=self._disk) + temp = Cache( + directory=directory, + timeout=timeout, + disk=self._disk if disk is None else Disk, + **settings, + ) _caches[name] = temp return temp - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. >>> cache = FanoutCache() @@ -616,6 +627,7 @@ class FanoutCache: 1 :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ @@ -626,8 +638,12 @@ class FanoutCache: except KeyError: parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) - cache = Cache(directory=directory, disk=self._disk) - deque = Deque.fromcache(cache) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) + deque = Deque.fromcache(cache, maxlen=maxlen) _deques[name] = deque return deque @@ -658,7 +674,11 @@ class FanoutCache: except KeyError: parts = name.split('/') directory = op.join(self._directory, 'index', *parts) - cache = Cache(directory=directory, disk=self._disk) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) index = Index.fromcache(cache) _indexes[name] = index return index diff --git a/lib/diskcache/persistent.py b/lib/diskcache/persistent.py index ed23398d..07507c6f 100644 --- a/lib/diskcache/persistent.py +++ b/lib/diskcache/persistent.py @@ -76,7 +76,7 @@ class Deque(Sequence): """ - def __init__(self, iterable=(), directory=None): + def __init__(self, iterable=(), directory=None, maxlen=None): """Initialize deque instance. If directory is None then temporary directory created. The directory @@ -87,10 +87,11 @@ class Deque(Sequence): """ self._cache = Cache(directory, eviction_policy='none') - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) @classmethod - def fromcache(cls, cache, iterable=()): + def fromcache(cls, cache, iterable=(), maxlen=None): """Initialize deque using `cache`. >>> cache = Cache() @@ -112,7 +113,8 @@ class Deque(Sequence): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) return self @property @@ -125,6 +127,31 @@ class Deque(Sequence): """Directory path where deque is stored.""" return self._cache.directory + @property + def maxlen(self): + """Max length of the deque.""" + return self._maxlen + + @maxlen.setter + def maxlen(self, value): + """Set max length of the deque. + + Pops items from left while length greater than max. + + >>> deque = Deque() + >>> deque.extendleft('abcde') + >>> deque.maxlen = 3 + >>> list(deque) + ['c', 'd', 'e'] + + :param value: max length + + """ + self._maxlen = value + with self._cache.transact(retry=True): + while len(self._cache) > self._maxlen: + self._popleft() + def _index(self, index, func): len_self = len(self) @@ -245,7 +272,7 @@ class Deque(Sequence): :return: deque with added items """ - self.extend(iterable) + self._extend(iterable) return self def __iter__(self): @@ -293,10 +320,11 @@ class Deque(Sequence): pass def __getstate__(self): - return self.directory + return self.directory, self.maxlen def __setstate__(self, state): - self.__init__(directory=state) + directory, maxlen = state + self.__init__(directory=directory, maxlen=maxlen) def append(self, value): """Add `value` to back of deque. @@ -311,7 +339,12 @@ class Deque(Sequence): :param value: value to add to back of deque """ - self._cache.push(value, retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, retry=True) + if len(self._cache) > self._maxlen: + self._popleft() + + _append = append def appendleft(self, value): """Add `value` to front of deque. @@ -326,7 +359,12 @@ class Deque(Sequence): :param value: value to add to front of deque """ - self._cache.push(value, side='front', retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, side='front', retry=True) + if len(self._cache) > self._maxlen: + self._pop() + + _appendleft = appendleft def clear(self): """Remove all elements from deque. @@ -341,6 +379,13 @@ class Deque(Sequence): """ self._cache.clear(retry=True) + _clear = clear + + def copy(self): + """Copy deque with same directory and max length.""" + TypeSelf = type(self) + return TypeSelf(directory=self.directory, maxlen=self.maxlen) + def count(self, value): """Return number of occurrences of `value` in deque. @@ -366,7 +411,9 @@ class Deque(Sequence): """ for value in iterable: - self.append(value) + self._append(value) + + _extend = extend def extendleft(self, iterable): """Extend front side of deque with value from `iterable`. @@ -380,7 +427,7 @@ class Deque(Sequence): """ for value in iterable: - self.appendleft(value) + self._appendleft(value) def peek(self): """Peek at value at back of deque. @@ -460,6 +507,8 @@ class Deque(Sequence): raise IndexError('pop from an empty deque') return value + _pop = pop + def popleft(self): """Remove and return value at front of deque. @@ -484,6 +533,8 @@ class Deque(Sequence): raise IndexError('pop from an empty deque') return value + _popleft = popleft + def remove(self, value): """Remove first occurrence of `value` in deque. @@ -531,15 +582,17 @@ class Deque(Sequence): ['c', 'b', 'a'] """ + # pylint: disable=protected-access # GrantJ 2019-03-22 Consider using an algorithm that swaps the values # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap # method would exchange the values at two given keys. Then, using a - # forward iterator and a reverse iterator, the reversis method could + # forward iterator and a reverse iterator, the reverse method could # avoid making copies of the values. temp = Deque(iterable=reversed(self)) - self.clear() - self.extend(temp) + self._clear() + self._extend(temp) directory = temp.directory + temp._cache.close() del temp rmtree(directory) @@ -574,22 +627,22 @@ class Deque(Sequence): for _ in range(steps): try: - value = self.pop() + value = self._pop() except IndexError: return else: - self.appendleft(value) + self._appendleft(value) else: steps *= -1 steps %= len_self for _ in range(steps): try: - value = self.popleft() + value = self._popleft() except IndexError: return else: - self.append(value) + self._append(value) __hash__ = None # type: ignore @@ -668,7 +721,9 @@ class Index(MutableMapping): args = args[1:] directory = None self._cache = Cache(directory, eviction_policy='none') - self.update(*args, **kwargs) + self._update(*args, **kwargs) + + _update = MutableMapping.update @classmethod def fromcache(cls, cache, *args, **kwargs): @@ -694,7 +749,7 @@ class Index(MutableMapping): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.update(*args, **kwargs) + self._update(*args, **kwargs) return self @property diff --git a/lib/diskcache/recipes.py b/lib/diskcache/recipes.py index b5af6dd7..babb68f6 100644 --- a/lib/diskcache/recipes.py +++ b/lib/diskcache/recipes.py @@ -17,6 +17,9 @@ class Averager: Sometimes known as "online statistics," the running average maintains the total and count. The average can then be calculated at any time. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.FanoutCache() >>> ave = Averager(cache, 'latency') @@ -65,6 +68,9 @@ class Averager: class Lock: """Recipe for cross-process and cross-thread lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> lock = Lock(cache, 'report-123') @@ -113,6 +119,9 @@ class Lock: class RLock: """Recipe for cross-process and cross-thread re-entrant lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> rlock = RLock(cache, 'user-123') @@ -181,6 +190,9 @@ class RLock: class BoundedSemaphore: """Recipe for cross-process and cross-thread bounded semaphore. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2) @@ -251,6 +263,9 @@ def throttle( ): """Decorator to throttle calls to function. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> count = 0 @@ -305,6 +320,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): Supports different kinds of locks: Lock, RLock, BoundedSemaphore. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> @barrier(cache, Lock)