From a97955ed9462174b72df768a3f48b84950c4f5bd Mon Sep 17 00:00:00 2001 From: JackDandy Date: Fri, 5 May 2023 15:08:05 +0100 Subject: [PATCH] =?UTF-8?q?Update=20filelock=203.11.0=20(d3241b9)=20?= =?UTF-8?q?=E2=86=92=203.12.0=20(b4713c9).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 3 +- lib/filelock/__init__.py | 3 +- lib/filelock/_api.py | 106 +++++++++++++++++++++++++++------------ lib/filelock/_soft.py | 14 +++--- lib/filelock/_unix.py | 10 ++-- lib/filelock/_util.py | 23 +++++++-- lib/filelock/_windows.py | 30 +++++------ lib/filelock/version.py | 4 +- 8 files changed, 126 insertions(+), 67 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4e0c6dc9..622e1a2c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ * Update attr 22.2.0 (a9960de) to 22.2.0 (683d056) * Update diskcache 5.4.0 (1cb1425) to 5.6.1 (4d30686) * Update feedparser 6.0.10 (5fcb3ae) to 6.0.10 (6d032b8) -* Update filelock 3.9.0 (ce3e891) to 3.11.0 (d3241b9) +* Update filelock 3.9.0 (ce3e891) to 3.12.0 (b4713c9) * Update Msgpack 1.0.4 (b5acfd5) to 1.0.5 (0516c2c) * Update Pytvmaze library 2.0.8 (16ed096) to 2.0.8 (81888a5) * Update Requests library 2.28.1 (ec553c2) to 2.29.0 (87d63de) @@ -27,6 +27,7 @@ [develop changelog] +* Update filelock 3.9.0 (ce3e891) to 3.11.0 (d3241b9) * Fix tv test to init recently added ReleaseMap to scene_exceptions refactor * Fix double use of var `result` overwrites the return value and causes an error in _parse_custom_exceptions * Fix name_parser_tests and webapi_tests diff --git a/lib/filelock/__init__.py b/lib/filelock/__init__.py index 31d2bce1..c7492ba5 100644 --- a/lib/filelock/__init__.py +++ b/lib/filelock/__init__.py @@ -32,11 +32,10 @@ else: # pragma: win32 no cover if warnings is not None: warnings.warn("only soft file lock is available", stacklevel=2) -#: Alias for the lock, which should be used for the current platform. On Windows, this is an alias for -# :class:`WindowsFileLock`, on Unix for :class:`UnixFileLock` and otherwise for :class:`SoftFileLock`. if TYPE_CHECKING: FileLock = SoftFileLock else: + #: Alias for the lock, which should be used for the current platform. FileLock = _FileLock diff --git a/lib/filelock/_api.py b/lib/filelock/_api.py index 958369a6..66710cc5 100644 --- a/lib/filelock/_api.py +++ b/lib/filelock/_api.py @@ -6,6 +6,7 @@ import os import time import warnings from abc import ABC, abstractmethod +from dataclasses import dataclass from threading import local from types import TracebackType from typing import Any @@ -36,7 +37,38 @@ class AcquireReturnProxy: self.lock.release() -class BaseFileLock(ABC, contextlib.ContextDecorator, local): +@dataclass +class FileLockContext: + """ + A dataclass which holds the context for a ``BaseFileLock`` object. + """ + + # The context is held in a separate class to allow optional use of thread local storage via the + # ThreadLocalFileContext class. + + #: The path to the lock file. + lock_file: str + + #: The default timeout value. + timeout: float + + #: The mode for the lock files + mode: int + + #: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held + lock_file_fd: int | None = None + + #: The lock counter is used for implementing the nested locking mechanism. + lock_counter: int = 0 # When the lock is acquired is increased and the lock is only released, when this value is 0 + + +class ThreadLocalFileContext(FileLockContext, local): + """ + A thread local version of the ``FileLockContext`` class. + """ + + +class BaseFileLock(ABC, contextlib.ContextDecorator): """Abstract base class for a file lock object.""" def __init__( @@ -44,6 +76,7 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): lock_file: str | os.PathLike[Any], timeout: float = -1, mode: int = 0o644, + thread_local: bool = True, ) -> None: """ Create a new lock object. @@ -52,29 +85,29 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a negative value. A timeout of 0 means, that there is exactly one attempt to acquire the file lock. - : param mode: file permissions for the lockfile. + :param mode: file permissions for the lockfile. + :param thread_local: Whether this object's internal context should be thread local or not. + If this is set to ``False`` then the lock will be reentrant across threads. """ - # The path to the lock file. - self._lock_file: str = os.fspath(lock_file) + self._is_thread_local = thread_local - # The file descriptor for the *_lock_file* as it is returned by the os.open() function. - # This file lock is only NOT None, if the object currently holds the lock. - self._lock_file_fd: int | None = None + # Create the context. Note that external code should not work with the context directly and should instead use + # properties of this class. + kwargs: dict[str, Any] = { + "lock_file": os.fspath(lock_file), + "timeout": timeout, + "mode": mode, + } + self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs) - # The default timeout value. - self._timeout: float = timeout - - # The mode for the lock files - self._mode: int = mode - - # The lock counter is used for implementing the nested locking mechanism. Whenever the lock is acquired, the - # counter is increased and the lock is only released, when this value is 0 again. - self._lock_counter: int = 0 + def is_thread_local(self) -> bool: + """:return: a flag indicating if this lock is thread local or not""" + return self._is_thread_local @property def lock_file(self) -> str: """:return: path to the lock file""" - return self._lock_file + return self._context.lock_file @property def timeout(self) -> float: @@ -83,7 +116,7 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): .. versionadded:: 2.0.0 """ - return self._timeout + return self._context.timeout @timeout.setter def timeout(self, value: float | str) -> None: @@ -92,16 +125,16 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): :param value: the new value, in seconds """ - self._timeout = float(value) + self._context.timeout = float(value) @abstractmethod def _acquire(self) -> None: - """If the file lock could be acquired, self._lock_file_fd holds the file descriptor of the lock file.""" + """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file.""" raise NotImplementedError @abstractmethod def _release(self) -> None: - """Releases the lock and sets self._lock_file_fd to None.""" + """Releases the lock and sets self._context.lock_file_fd to None.""" raise NotImplementedError @property @@ -114,7 +147,14 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): This was previously a method and is now a property. """ - return self._lock_file_fd is not None + return self._context.lock_file_fd is not None + + @property + def lock_counter(self) -> int: + """ + :return: The number of times this lock has been acquired (but not yet released). + """ + return self._context.lock_counter def acquire( self, @@ -132,7 +172,7 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): :param poll_interval: interval of trying to acquire the lock file :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the - first attempt. Otherwise this method will block until the timeout expires or the lock is acquired. + first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired. :raises Timeout: if fails to acquire lock within the timeout period :return: a context object that will unlock the file when the context is exited @@ -157,7 +197,7 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): """ # Use the default timeout, if no timeout is provided. if timeout is None: - timeout = self.timeout + timeout = self._context.timeout if poll_intervall is not None: msg = "use poll_interval instead of poll_intervall" @@ -165,10 +205,10 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): poll_interval = poll_intervall # Increment the number right at the beginning. We can still undo it, if something fails. - self._lock_counter += 1 + self._context.lock_counter += 1 lock_id = id(self) - lock_filename = self._lock_file + lock_filename = self.lock_file start_time = time.perf_counter() try: while True: @@ -180,16 +220,16 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): break elif blocking is False: _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename) - raise Timeout(self._lock_file) + raise Timeout(lock_filename) elif 0 <= timeout < time.perf_counter() - start_time: _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename) - raise Timeout(self._lock_file) + raise Timeout(lock_filename) else: msg = "Lock %s not acquired on %s, waiting %s seconds ..." _LOGGER.debug(msg, lock_id, lock_filename, poll_interval) time.sleep(poll_interval) except BaseException: # Something did go wrong, so decrement the counter. - self._lock_counter = max(0, self._lock_counter - 1) + self._context.lock_counter = max(0, self._context.lock_counter - 1) raise return AcquireReturnProxy(lock=self) @@ -201,14 +241,14 @@ class BaseFileLock(ABC, contextlib.ContextDecorator, local): :param force: If true, the lock counter is ignored and the lock is released in every case/ """ if self.is_locked: - self._lock_counter -= 1 + self._context.lock_counter -= 1 - if self._lock_counter == 0 or force: - lock_id, lock_filename = id(self), self._lock_file + if self._context.lock_counter == 0 or force: + lock_id, lock_filename = id(self), self.lock_file _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename) self._release() - self._lock_counter = 0 + self._context.lock_counter = 0 _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename) def __enter__(self) -> BaseFileLock: diff --git a/lib/filelock/_soft.py b/lib/filelock/_soft.py index 697f6c4c..57a6c04d 100644 --- a/lib/filelock/_soft.py +++ b/lib/filelock/_soft.py @@ -5,14 +5,14 @@ import sys from errno import EACCES, EEXIST from ._api import BaseFileLock -from ._util import raise_on_exist_ro_file +from ._util import raise_on_not_writable_file class SoftFileLock(BaseFileLock): """Simply watches the existence of the lock file.""" def _acquire(self) -> None: - raise_on_exist_ro_file(self._lock_file) + raise_on_not_writable_file(self.lock_file) # first check for exists and read-only mode as the open will mask this case as EEXIST flags = ( os.O_WRONLY # open for writing only @@ -21,7 +21,7 @@ class SoftFileLock(BaseFileLock): | os.O_TRUNC # truncate the file to zero byte ) try: - file_handler = os.open(self._lock_file, flags, self._mode) + file_handler = os.open(self.lock_file, flags, self._context.mode) except OSError as exception: # re-raise unless expected exception if not ( exception.errno == EEXIST # lock already exist @@ -29,13 +29,13 @@ class SoftFileLock(BaseFileLock): ): # pragma: win32 no cover raise else: - self._lock_file_fd = file_handler + self._context.lock_file_fd = file_handler def _release(self) -> None: - os.close(self._lock_file_fd) # type: ignore # the lock file is definitely not None - self._lock_file_fd = None + os.close(self._context.lock_file_fd) # type: ignore # the lock file is definitely not None + self._context.lock_file_fd = None try: - os.remove(self._lock_file) + os.remove(self.lock_file) except OSError: # the file is already deleted and that's what we want pass diff --git a/lib/filelock/_unix.py b/lib/filelock/_unix.py index 4c0a2603..641bd8d9 100644 --- a/lib/filelock/_unix.py +++ b/lib/filelock/_unix.py @@ -33,9 +33,9 @@ else: # pragma: win32 no cover def _acquire(self) -> None: open_flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC - fd = os.open(self._lock_file, open_flags, self._mode) + fd = os.open(self.lock_file, open_flags, self._context.mode) try: - os.fchmod(fd, self._mode) + os.fchmod(fd, self._context.mode) except PermissionError: pass # This locked is not owned by this UID try: @@ -45,14 +45,14 @@ else: # pragma: win32 no cover if exception.errno == ENOSYS: # NotImplemented error raise NotImplementedError("FileSystem does not appear to support flock; user SoftFileLock instead") else: - self._lock_file_fd = fd + self._context.lock_file_fd = fd def _release(self) -> None: # Do not remove the lockfile: # https://github.com/tox-dev/py-filelock/issues/31 # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition - fd = cast(int, self._lock_file_fd) - self._lock_file_fd = None + fd = cast(int, self._context.lock_file_fd) + self._context.lock_file_fd = None fcntl.flock(fd, fcntl.LOCK_UN) os.close(fd) diff --git a/lib/filelock/_util.py b/lib/filelock/_util.py index 238b80fa..81cbeaf2 100644 --- a/lib/filelock/_util.py +++ b/lib/filelock/_util.py @@ -2,9 +2,18 @@ from __future__ import annotations import os import stat +import sys +from errno import EACCES, EISDIR -def raise_on_exist_ro_file(filename: str) -> None: +def raise_on_not_writable_file(filename: str) -> None: + """ + Raise an exception if attempting to open the file for writing would fail. + This is done so files that will never be writable can be separated from + files that are writable but currently locked + :param filename: file to check + :raises OSError: as if the file was opened for writing + """ try: file_stat = os.stat(filename) # use stat to do exists + can write to check without race condition except OSError: @@ -12,9 +21,17 @@ def raise_on_exist_ro_file(filename: str) -> None: if file_stat.st_mtime != 0: # if os.stat returns but modification is zero that's an invalid os.stat - ignore it if not (file_stat.st_mode & stat.S_IWUSR): - raise PermissionError(f"Permission denied: {filename!r}") + raise PermissionError(EACCES, "Permission denied", filename) + + if stat.S_ISDIR(file_stat.st_mode): + if sys.platform == "win32": # pragma: win32 cover + # On Windows, this is PermissionError + raise PermissionError(EACCES, "Permission denied", filename) + else: # pragma: win32 no cover + # On linux / macOS, this is IsADirectoryError + raise IsADirectoryError(EISDIR, "Is a directory", filename) __all__ = [ - "raise_on_exist_ro_file", + "raise_on_not_writable_file", ] diff --git a/lib/filelock/_windows.py b/lib/filelock/_windows.py index cb324c94..799644c8 100644 --- a/lib/filelock/_windows.py +++ b/lib/filelock/_windows.py @@ -2,46 +2,48 @@ from __future__ import annotations import os import sys -from errno import ENOENT +from errno import EACCES from typing import cast from ._api import BaseFileLock -from ._util import raise_on_exist_ro_file +from ._util import raise_on_not_writable_file if sys.platform == "win32": # pragma: win32 cover import msvcrt class WindowsFileLock(BaseFileLock): - """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems.""" + """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.""" def _acquire(self) -> None: - raise_on_exist_ro_file(self._lock_file) + raise_on_not_writable_file(self.lock_file) flags = ( os.O_RDWR # open for read and write | os.O_CREAT # create file if not exists - | os.O_TRUNC # truncate file if not empty + | os.O_TRUNC # truncate file if not empty ) try: - fd = os.open(self._lock_file, flags, self._mode) + fd = os.open(self.lock_file, flags, self._context.mode) except OSError as exception: - if exception.errno == ENOENT: # No such file or directory + if exception.errno != EACCES: # has no access to this lock raise else: try: msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - except OSError: - os.close(fd) + except OSError as exception: + os.close(fd) # close file first + if exception.errno != EACCES: # file is already locked + raise else: - self._lock_file_fd = fd + self._context.lock_file_fd = fd def _release(self) -> None: - fd = cast(int, self._lock_file_fd) - self._lock_file_fd = None + fd = cast(int, self._context.lock_file_fd) + self._context.lock_file_fd = None msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) os.close(fd) try: - os.remove(self._lock_file) + os.remove(self.lock_file) # Probably another instance of the application hat acquired the file lock. except OSError: pass @@ -49,7 +51,7 @@ if sys.platform == "win32": # pragma: win32 cover else: # pragma: win32 no cover class WindowsFileLock(BaseFileLock): - """Uses the :func:`msvcrt.locking` function to hard lock the lock file on windows systems.""" + """Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.""" def _acquire(self) -> None: raise NotImplementedError diff --git a/lib/filelock/version.py b/lib/filelock/version.py index 16805e82..f8e7b3ea 100644 --- a/lib/filelock/version.py +++ b/lib/filelock/version.py @@ -1,4 +1,4 @@ # file generated by setuptools_scm # don't change, don't track in version control -__version__ = version = '3.11.0' -__version_tuple__ = version_tuple = (3, 11, 0) +__version__ = version = '3.12.0' +__version_tuple__ = version_tuple = (3, 12, 0)