diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fd54c4e..35caf877 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: priority: 5 args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - repo: https://github.com/gaborbernat/simplint-pre-commit - rev: "5f12b15e1199c42bdeb3e13ea1c8b45952fbd802" # frozen: v0.11.0 + rev: "a12d8eaac290be24041a3e2778c636c56453722e" # frozen: v0.12.2 hooks: - id: simplint priority: 6 diff --git a/pyproject.toml b/pyproject.toml index a1871ed6..1a37a7ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,6 @@ lint.ignore = [ "undocumented-public-init", # undocumented-public-api (simplint) covers the public API instead "undocumented-public-method", # undocumented-public-api (simplint) covers the public API instead "undocumented-public-nested-class", # undocumented-public-api (simplint) covers the public API instead - "unnecessary-regular-expression-compile", # conflicts with simplint inline-regex ] lint.per-file-ignores."tasks/**/*.py" = [ "assert", # asserts state the platform a Windows-only script already refuses to run outside of, narrowing it for ty diff --git a/src/filelock/_api.py b/src/filelock/_api.py index 87c44b8f..9b892d3b 100644 --- a/src/filelock/_api.py +++ b/src/filelock/_api.py @@ -345,19 +345,6 @@ def raise_cleanup_errors( _RESOLVE_DIR: Final[Callable[[str], str]] = os.path.abspath if sys.platform == "win32" else os.path.realpath -def canonical(path: str | os.PathLike[str]) -> str: - # Return one stable key for *path*, collapsing equivalent spellings without following a final symlink. - # - # Relative, absolute, and ``./`` spellings of one lock file must map to a single singleton instance, - # deadlock-registry entry, and removal key. Resolving the whole path with ``realpath`` would follow a final symlink - # and alias a lock target the backend deliberately rejects, so the registry identity would differ from the - # backend's. Resolving only the parent directory and re-appending the literal final component collapses the - # equivalent spellings while keeping a final symlink a distinct key. On Windows the parent is resolved with - # ``abspath`` so junctions and reparse points are not followed either. - parent, name = os.path.split(os.fspath(path)) - return os.path.join(_RESOLVE_DIR(parent or os.curdir), name) # ruff:ignore[os-path-join] # string join matches abspath/realpath - - class _ThreadLocalRegistry(local): def __init__(self) -> None: super().__init__() @@ -501,6 +488,22 @@ def __call__( # ruff:ignore[too-many-arguments] # forwards the public construc raise ValueError(msg) +@cache +def resolve_lazy_imports_before_shutdown() -> None: + if hasattr(types, "LazyImportType"): # pragma: >=3.15 cover + atexit.register(_resolve_lazy_imports) + + +def _resolve_lazy_imports() -> None: # pragma: >=3.15 cover + # Interpreter shutdown clears sys.modules and sys.meta_path before it finalizes a lock still alive at exit, so a + # lazy import (PEP 810) that the release path in __del__ reaches first can no longer load. Load them while it can. + for name, module in list(sys.modules.items()): + if name == "filelock" or name.startswith("filelock."): + for attribute, value in list(module.__dict__.items()): + if type(value) is types.LazyImportType: + getattr(module, attribute) + + def _resolve_lifetime(lifetime: float | None, cls: type[BaseFileLock], *, stacklevel: int) -> float | None: # Validate ``lifetime`` and drop a value the backend cannot honor. # @@ -544,6 +547,13 @@ def resolve_poll_interval(poll_interval: float) -> float: return poll_interval +def check_timeout_max(name: str, value: float) -> None: + # Lock.acquire, Event.wait and time.sleep raise OverflowError past this bound, which is only ~49.7 days on Windows. + if value > TIMEOUT_MAX: + msg = f"{name} must not exceed threading.TIMEOUT_MAX ({TIMEOUT_MAX}), got {value!r}" + raise ValueError(msg) + + def check_blocking(blocking: bool) -> None: # ruff:ignore[boolean-type-hint-positional-argument] # validates the option's own value if not isinstance(blocking, bool): msg = f"blocking must be a bool, not {type(blocking).__name__}" @@ -655,6 +665,59 @@ def _supported_init_params( return {name: value for name, value in params.items() if name in model.accepted_params} +@contextmanager +def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]: + # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. + if not _HAS_REGISTER_AT_FORK or sys.is_finalizing(): + yield # pragma: lacks fork + return # pragma: lacks fork + ensure_current_process() # pragma: needs fork + creator_pid = os.getpid() # pragma: needs fork + # A generator holding this transition can be finalized on another thread, which must close the entering thread's. + thread_id: Final = get_ident() # pragma: needs fork + with _FORK_STATE.gate: # pragma: needs fork + while ( + _FORK_STATE.admission_closed + and thread_id not in _FORK_STATE.fork_owner_depths + and thread_id not in _FORK_STATE.transitions + ): + _FORK_STATE.gate.wait() # pragma: no cover - exercised in isolated interpreter + token: Final = next(_TRANSITION_TOKENS) + _FORK_STATE.transitions.setdefault(thread_id, {})[token] = descriptor_owner + _FORK_STATE.active_transitions += 1 + try: # pragma: needs fork + yield + finally: + if os.getpid() == creator_pid: # pragma: needs fork + with _FORK_STATE.gate: + owned: Final = _FORK_STATE.transitions[thread_id] + del owned[token] + if not owned: + del _FORK_STATE.transitions[thread_id] + _FORK_STATE.active_transitions -= 1 + _FORK_STATE.gate.notify_all() + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _InitParameterModel: + accepted_params: frozenset[str] + accepts_kwargs: bool + default_params: dict[str, inspect.Parameter] + + +def canonical(path: str | os.PathLike[str]) -> str: + # Return one stable key for *path*, collapsing equivalent spellings without following a final symlink. + # + # Relative, absolute, and ``./`` spellings of one lock file must map to a single singleton instance, + # deadlock-registry entry, and removal key. Resolving the whole path with ``realpath`` would follow a final symlink + # and alias a lock target the backend deliberately rejects, so the registry identity would differ from the + # backend's. Resolving only the parent directory and re-appending the literal final component collapses the + # equivalent spellings while keeping a final symlink a distinct key. On Windows the parent is resolved with + # ``abspath`` so junctions and reparse points are not followed either. + parent, name = os.path.split(os.fspath(path)) + return os.path.join(_RESOLVE_DIR(parent or os.curdir), name) # ruff:ignore[os-path-join] # string join matches abspath/realpath + + def timeout_behavior(timeout: float, *, any_negative: bool) -> float | str: # inf, and for a BaseFileLock any negative value, wait without a limit like -1. if math.isnan(timeout): @@ -662,23 +725,9 @@ def timeout_behavior(timeout: float, *, any_negative: bool) -> float | str: return -1.0 if timeout == math.inf or (any_negative and timeout < 0) else timeout -def check_timeout_max(name: str, value: float) -> None: - # Lock.acquire, Event.wait and time.sleep raise OverflowError past this bound, which is only ~49.7 days on Windows. - if value > TIMEOUT_MAX: - msg = f"{name} must not exceed threading.TIMEOUT_MAX ({TIMEOUT_MAX}), got {value!r}" - raise ValueError(msg) - - _INIT_PARAMETER_MODELS: Final[WeakKeyDictionary[type[BaseFileLock], _InitParameterModel]] = WeakKeyDictionary() -@dataclass(frozen=True, slots=True, kw_only=True) -class _InitParameterModel: - accepted_params: frozenset[str] - accepts_kwargs: bool - default_params: dict[str, inspect.Parameter] - - @dataclass(frozen=True, slots=True, kw_only=True) class LockSettings: thread_local: bool @@ -865,13 +914,7 @@ def _adopt_settings(self, settings: LockSettings) -> None: #: Holder id in the deadlock registry; a stand-in that finishes the finalizer has another id(self). self._holder_id = id(self) self._transition_lock = RLock() - self._is_thread_local = settings.thread_local - self._is_singleton = settings.is_singleton - self._context_error_policy = settings.context_error_policy # already validated by the metaclass - self._close_error_policy = settings.close_error_policy # already validated by the metaclass - self._fallback_to_soft = settings.fallback_to_soft - self._preserve_lock_file = settings.preserve_lock_file # already validated by the metaclass - self._on_acquired = settings.on_acquired # already validated by the metaclass + self._settings = settings # policies and callbacks already validated by the metaclass def __enter__(self) -> Self: """ @@ -1012,7 +1055,7 @@ def _transition_admission( # state are plain attributes every thread reads and writes, so an unserialized loser clears the winner's # descriptor or double-closes it. Waiting in slices keeps cancel_check and the timeout live. A thread-local # context is never shared, so it skips the gate. - if self._is_thread_local: + if self._settings.thread_local: yield return deadline: Final = None if timeout < 0 else start_time + timeout @@ -1241,10 +1284,10 @@ def _rollback_failed_registration(self, registration_error: BaseException) -> No def _invoke_on_acquired(self) -> None: # The wrapper runs in the backend executor for async locks, preserving the callback's documented thread. - if self._on_acquired is None or self._context.lock_counter != 1: + if self._settings.on_acquired is None or self._context.lock_counter != 1: return try: - self._on_acquired(cast("int", self._context.lock_file_fd)) + self._settings.on_acquired(cast("int", self._context.lock_file_fd)) except BaseException as callback_error: # arbitrary caller code; roll back on any failure callback_context = callback_error.__context__ try: @@ -1344,7 +1387,7 @@ def _release_in_context(self, body_error: BaseException | None) -> None: try: self.release() except BaseException as release_error: - if body_error is None or self._context_error_policy == "chain": + if body_error is None or self._settings.context_error_policy == "chain": raise raise_body_and_release(body_error, release_error) @@ -1388,7 +1431,7 @@ def _release_current(self, *, force: bool) -> None: if self._context.lock_file_registry is not None: self._finish_interrupted_release() return - with contextlib.nullcontext() if self._is_thread_local else self._transition_lock: + with contextlib.nullcontext() if self._settings.thread_local else self._transition_lock: state: Final = self._hold_state() # A signal handler can land between the backend's unlock and its clearing of the descriptor, where a second # release would close the number twice and could close a file that reused it. The flag holds the releasing @@ -1481,7 +1524,7 @@ def is_singleton(self) -> bool: .. versionadded:: 3.13.0 """ - return self._is_singleton + return self._settings.is_singleton @property def context_error_policy(self) -> ContextErrorPolicy: @@ -1491,7 +1534,7 @@ def context_error_policy(self) -> ContextErrorPolicy: .. versionadded:: 3.30.0 """ - return self._context_error_policy + return self._settings.context_error_policy @property def close_error_policy(self) -> CloseErrorPolicy: @@ -1501,7 +1544,7 @@ def close_error_policy(self) -> CloseErrorPolicy: .. versionadded:: 3.30.0 """ - return self._close_error_policy + return self._settings.close_error_policy @property def fallback_to_soft(self) -> bool: @@ -1514,7 +1557,7 @@ def fallback_to_soft(self) -> bool: .. versionadded:: 3.30.0 """ - return self._fallback_to_soft + return self._settings.fallback_to_soft @property def preserve_lock_file(self) -> bool: @@ -1527,7 +1570,7 @@ def preserve_lock_file(self) -> bool: .. versionadded:: 3.30.0 """ - return self._preserve_lock_file + return self._settings.preserve_lock_file @property def on_acquired(self) -> Callable[[int], None] | None: @@ -1540,7 +1583,7 @@ def on_acquired(self) -> Callable[[int], None] | None: .. versionadded:: 3.30.0 """ - return self._on_acquired + return self._settings.on_acquired @property def timeout(self) -> float: @@ -1686,7 +1729,7 @@ def _singleton_extra_mismatches( # ruff:ignore[no-self-use] # the base class a def is_thread_local(self) -> bool: """:returns: a flag indicating if this lock is thread local or not""" - return self._is_thread_local + return self._settings.thread_local def _close_released_fd(self, fd: int, *, default_suppresses: bool) -> None: # CPython never retries close() after EINTR because the descriptor number may already be reused, so neither does @@ -1698,7 +1741,9 @@ def _close_released_fd(self, fd: int, *, default_suppresses: bool) -> None: raise def _suppresses_close_error(self, *, default_suppresses: bool) -> bool: - return self._close_error_policy == "suppress" or (self._close_error_policy == "default" and default_suppresses) + return self._settings.close_error_policy == "suppress" or ( + self._settings.close_error_policy == "default" and default_suppresses + ) def _open_mode(self) -> int: # Mode for ``os.open``: 0o666 when unset so umask and ACLs decide, otherwise the explicit mode. @@ -1761,31 +1806,33 @@ def _descriptors_for_fork(self) -> tuple[tuple[int, tuple[int, int] | None], ... return tuple(descriptors) -def resolve_read_write_timeout(timeout: float, *, blocking: bool) -> float: - # 0 would still get SQLite's full busy timeout, and a truthy "false" would turn one attempt into a blocking wait. - check_blocking(blocking) - if (timeout := resolve_timeout(timeout, blocking=blocking)) < 0 and timeout != -1 and blocking: - msg = f"timeout must be a non-negative number or -1, not {timeout!r}" - raise ValueError(msg) - return timeout +def register_fork_class(cls: _ForkResettableClass) -> None: + if not _HAS_REGISTER_AT_FORK: + return # pragma: lacks fork + with fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork + _FORK_CLASSES[id(cls)] = cls + _refresh_owner_pins() -def acquire_thread_lock( - lock: Lock, *, blocking: bool, timeout: float, poll_interval: float, cancel_check: Callable[[], bool] | None -) -> bool: - # Take a read-write lock's in-process lock; with *cancel_check*, ask it every *poll_interval* while waiting. - if not blocking: - return lock.acquire(blocking=False) - if cancel_check is None: - # Lock.acquire raises OverflowError past TIMEOUT_MAX, which is 49.7 days on Windows. - return lock.acquire() if timeout == -1 else lock.acquire(timeout=min(timeout, TIMEOUT_MAX)) - deadline = None if timeout == -1 else time.perf_counter() + timeout - while not lock.acquire( - timeout=poll_interval if deadline is None else min(poll_interval, max(deadline - time.perf_counter(), 0)) - ): - if cancel_check() or (deadline is not None and time.perf_counter() >= deadline): - return False - return True +def _refresh_owner_pins() -> None: # pragma: needs fork + with _FORK_STATE.gate: + fork_owner_thread_ids = tuple(_FORK_STATE.fork_owner_depths) + if get_ident() not in fork_owner_thread_ids: # pragma: no cover - at-fork callbacks disable tracing + return + objects = tuple(_FORK_OBJECTS.values()) + classes = tuple(_FORK_CLASSES.values()) + for thread_id in fork_owner_thread_ids: + if object_snapshots := _FORK_STATE.pinned_objects.get(thread_id): # pragma: needs fork + object_snapshots[:] = [objects] * len(object_snapshots) + _FORK_STATE.pinned_classes[thread_id][:] = [classes] * len(object_snapshots) + + +def register_fork_object(instance: _ForkResettable) -> None: + if not _HAS_REGISTER_AT_FORK: + return # pragma: lacks fork + with fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork + _FORK_OBJECTS[id(instance)] = instance + _refresh_owner_pins() def resolve_timeout(timeout: float, *, blocking: bool) -> float: @@ -1797,40 +1844,6 @@ def resolve_timeout(timeout: float, *, blocking: bool) -> float: return -1 if timeout == math.inf else timeout -@dataclass( # simplint: ignore[dataclass-frozen] # state is reset to None on finalize - slots=True, weakref_slot=True, kw_only=True -) -class _FinalizerState: - # A finalized lock's state, held for the release that runs once the lock is unreachable. - state: ( - dict[str, object] | None # simplint: ignore[imprecise-annotation] # the instance __dict__ holds any value - ) - - -#: Weak references whose callback finishes a singleton's finalizer once the cache no longer finds it. -_UNREACHABLE_LOCKS: Final[set[ref[BaseFileLock]]] = set() - - -def _finalize_unreachable(holder: ref[_FinalizerState], cls: type[BaseFileLock], reference: ref[BaseFileLock]) -> None: - _UNREACHABLE_LOCKS.discard(reference) - if (finalizer_state := holder()) is None or (state := finalizer_state.state) is None: - return # pragma: no cover # only a collector freeing a resurrected lock as part of a cycle drops the holder - # The lock's dictionary points back at the holder; that cycle ends here. - finalizer_state.state = None - # The release methods only read and write this state, so a fresh instance takes it over. Dropping that instance - # runs __del__ again, and the cache no longer finds the lock. - stand_in = object.__new__(cls) - stand_in.__dict__ = state - del stand_in - - -def is_system_call_failure(error: BaseException) -> bool: - # A failed system call sets errno. A bare TimeoutError raised by a signal handler is an OSError without one. - if _is_exception_group(error): - return all(map(is_system_call_failure, cast("_ExceptionGroupProtocol", error).exceptions)) - return isinstance(error, OSError) and error.errno is not None - - def _on_this_stack(frame: FrameType | None) -> bool: if frame is None: return False @@ -1869,8 +1882,87 @@ def __exit__( lock.release() +def is_system_call_failure(error: BaseException) -> bool: + # A failed system call sets errno. A bare TimeoutError raised by a signal handler is an OSError without one. + if _is_exception_group(error): + return all(map(is_system_call_failure, cast("_ExceptionGroupProtocol", error).exceptions)) + return isinstance(error, OSError) and error.errno is not None + + +def unregister_owned_descriptor(token: int) -> None: # pragma: needs fork + # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. + if sys.is_finalizing(): # pragma: no cover # only exit reaches it, after coverage stops measuring + return + with fork_transition(), _FORK_STATE.registry_lock: + _OWNED_DESCRIPTORS.pop(token, None) + + +def register_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int | None: + if not _HAS_REGISTER_AT_FORK: + return None # pragma: lacks fork + with fork_transition(): # pragma: needs fork + if identity is None: + stat_result = os.fstat(fd) + identity = stat_result.st_dev, stat_result.st_ino + return _record_owned_descriptor(fd, identity) + + +@dataclass( # simplint: ignore[dataclass-frozen] # state is reset to None on finalize + slots=True, weakref_slot=True, kw_only=True +) +class _FinalizerState: + # A finalized lock's state, held for the release that runs once the lock is unreachable. + state: ( + dict[str, object] | None # simplint: ignore[imprecise-annotation] # the instance __dict__ holds any value + ) + + +def _finalize_unreachable(holder: ref[_FinalizerState], cls: type[BaseFileLock], reference: ref[BaseFileLock]) -> None: + _UNREACHABLE_LOCKS.discard(reference) + if (finalizer_state := holder()) is None or (state := finalizer_state.state) is None: + return # pragma: no cover # only a collector freeing a resurrected lock as part of a cycle drops the holder + # The lock's dictionary points back at the holder; that cycle ends here. + finalizer_state.state = None + # The release methods only read and write this state, so a fresh instance takes it over. Dropping that instance + # runs __del__ again, and the cache no longer finds the lock. + stand_in = object.__new__(cls) + stand_in.__dict__ = state + del stand_in + + +def resolve_read_write_timeout(timeout: float, *, blocking: bool) -> float: + # 0 would still get SQLite's full busy timeout, and a truthy "false" would turn one attempt into a blocking wait. + check_blocking(blocking) + if (timeout := resolve_timeout(timeout, blocking=blocking)) < 0 and timeout != -1 and blocking: + msg = f"timeout must be a non-negative number or -1, not {timeout!r}" + raise ValueError(msg) + return timeout + + +def acquire_thread_lock( + lock: Lock, *, blocking: bool, timeout: float, poll_interval: float, cancel_check: Callable[[], bool] | None +) -> bool: + # Take a read-write lock's in-process lock; with *cancel_check*, ask it every *poll_interval* while waiting. + if not blocking: + return lock.acquire(blocking=False) + if cancel_check is None: + # Lock.acquire raises OverflowError past TIMEOUT_MAX, which is 49.7 days on Windows. + return lock.acquire() if timeout == -1 else lock.acquire(timeout=min(timeout, TIMEOUT_MAX)) + deadline = None if timeout == -1 else time.perf_counter() + timeout + while not lock.acquire( + timeout=poll_interval if deadline is None else min(poll_interval, max(deadline - time.perf_counter(), 0)) + ): + if cancel_check() or (deadline is not None and time.perf_counter() >= deadline): + return False + return True + + +#: Weak references whose callback finishes a singleton's finalizer once the cache no longer finds it. +_UNREACHABLE_LOCKS: Final[set[ref[BaseFileLock]]] = set() + + @dataclass # simplint: ignore[dataclass-slots, dataclass-kw-only] # base rejects slots; signature is public -class FileLockContext: +class FileLockContext: # simplint: ignore[too-many-attributes] # its fields are public API # Holds the context for a ``BaseFileLock`` object. # A separate class so ThreadLocalFileContext can make the whole context thread-local. @@ -2012,22 +2104,6 @@ def reset_synchronization(self) -> None: # pragma: forked child _SETTRACE_SURVIVES_AUDIT_HOOKS: Final[bool] = sys.implementation.name != "cpython" or sys.version_info >= (3, 12) -@cache -def resolve_lazy_imports_before_shutdown() -> None: - if hasattr(types, "LazyImportType"): # pragma: >=3.15 cover - atexit.register(_resolve_lazy_imports) - - -def _resolve_lazy_imports() -> None: # pragma: >=3.15 cover - # Interpreter shutdown clears sys.modules and sys.meta_path before it finalizes a lock still alive at exit, so a - # lazy import (PEP 810) that the release path in __del__ reaches first can no longer load. Load them while it can. - for name, module in list(sys.modules.items()): - if name == "filelock" or name.startswith("filelock."): - for attribute, value in list(module.__dict__.items()): - if type(value) is types.LazyImportType: - getattr(module, attribute) - - def _register_fork_hooks() -> None: if _REGISTER_AT_FORK is None: return # pragma: lacks fork @@ -2040,65 +2116,6 @@ def _register_fork_hooks() -> None: ) -@contextmanager -def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]: - # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. - if not _HAS_REGISTER_AT_FORK or sys.is_finalizing(): - yield # pragma: lacks fork - return # pragma: lacks fork - ensure_current_process() # pragma: needs fork - creator_pid = os.getpid() # pragma: needs fork - # A generator holding this transition can be finalized on another thread, which must close the entering thread's. - thread_id: Final = get_ident() # pragma: needs fork - with _FORK_STATE.gate: # pragma: needs fork - while ( - _FORK_STATE.admission_closed - and thread_id not in _FORK_STATE.fork_owner_depths - and thread_id not in _FORK_STATE.transitions - ): - _FORK_STATE.gate.wait() # pragma: no cover - exercised in isolated interpreter - token: Final = next(_TRANSITION_TOKENS) - _FORK_STATE.transitions.setdefault(thread_id, {})[token] = descriptor_owner - _FORK_STATE.active_transitions += 1 - try: # pragma: needs fork - yield - finally: - if os.getpid() == creator_pid: # pragma: needs fork - with _FORK_STATE.gate: - owned: Final = _FORK_STATE.transitions[thread_id] - del owned[token] - if not owned: - del _FORK_STATE.transitions[thread_id] - _FORK_STATE.active_transitions -= 1 - _FORK_STATE.gate.notify_all() - - -def register_fork_object(instance: _ForkResettable) -> None: - if not _HAS_REGISTER_AT_FORK: - return # pragma: lacks fork - with fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork - _FORK_OBJECTS[id(instance)] = instance - _refresh_owner_pins() - - -def register_fork_class(cls: _ForkResettableClass) -> None: - if not _HAS_REGISTER_AT_FORK: - return # pragma: lacks fork - with fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork - _FORK_CLASSES[id(cls)] = cls - _refresh_owner_pins() - - -def register_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int | None: - if not _HAS_REGISTER_AT_FORK: - return None # pragma: lacks fork - with fork_transition(): # pragma: needs fork - if identity is None: - stat_result = os.fstat(fd) - identity = stat_result.st_dev, stat_result.st_ino - return _record_owned_descriptor(fd, identity) - - def _record_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int: # pragma: needs fork with _FORK_STATE.registry_lock: token = next(_DESCRIPTOR_TOKENS) @@ -2111,14 +2128,6 @@ def _record_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int: return token -def unregister_owned_descriptor(token: int) -> None: # pragma: needs fork - # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. - if sys.is_finalizing(): # pragma: no cover # only exit reaches it, after coverage stops measuring - return - with fork_transition(), _FORK_STATE.registry_lock: - _OWNED_DESCRIPTORS.pop(token, None) - - def _pin_fork_objects() -> None: # pragma: needs fork ensure_current_process() thread_id = get_ident() @@ -2240,19 +2249,6 @@ def _detach_child_state( # pragma: forked child _REGISTRY.held.clear() -def _refresh_owner_pins() -> None: # pragma: needs fork - with _FORK_STATE.gate: - fork_owner_thread_ids = tuple(_FORK_STATE.fork_owner_depths) - if get_ident() not in fork_owner_thread_ids: # pragma: no cover - at-fork callbacks disable tracing - return - objects = tuple(_FORK_OBJECTS.values()) - classes = tuple(_FORK_CLASSES.values()) - for thread_id in fork_owner_thread_ids: - if object_snapshots := _FORK_STATE.pinned_objects.get(thread_id): # pragma: needs fork - object_snapshots[:] = [objects] * len(object_snapshots) - _FORK_STATE.pinned_classes[thread_id][:] = [classes] * len(object_snapshots) - - # The defaults capture the module globals: the hook outlives them at interpreter shutdown, where CPython wipes the # module dict to None before the final audit events fire. def _audit_fork_safety( # pragma: no cover - CPython disables tracing while Python audit hooks run diff --git a/src/filelock/_async.py b/src/filelock/_async.py index e1303922..56dfb9a8 100644 --- a/src/filelock/_async.py +++ b/src/filelock/_async.py @@ -55,12 +55,6 @@ TASKS_THREAD_ID: Final[int] = 0 -@dataclass(frozen=True, slots=True, kw_only=True) -class BackendOutcome(Generic[_T]): - value: _T | None = None - error: BaseException | None = None - - class AsyncTransitionGate: # First-in, first-out admission for the calls that transition one lock context. # @@ -499,6 +493,12 @@ def settle(call: asyncio.Future[BackendOutcome[_T]]) -> None: return outcome +@dataclass(frozen=True, slots=True, kw_only=True) +class BackendOutcome(Generic[_T]): + value: _T | None = None + error: BaseException | None = None + + def capture_call(func: Callable[[], _T]) -> BackendOutcome[_T]: try: return BackendOutcome(value=func()) diff --git a/src/filelock/_lease.py b/src/filelock/_lease.py index cf64f208..0182edd9 100644 --- a/src/filelock/_lease.py +++ b/src/filelock/_lease.py @@ -59,40 +59,6 @@ _DEFAULT_LEASE_DURATION: Final[float] = 30.0 -@dataclass(frozen=True) -class LeaseCompromise: - """ - Why a held lease stopped being this process's to hold, and the error that ended its refresh, if any. - - :param lock_file: the path of the lock the lease was held on - :param token: the token of the claim that was lost - :param reason: what took the claim away: the marker vanished, its owner changed, a refresh failed, or it was evicted - :param error: the error that ended the refresh, when a refresh failed with one - """ - - lock_file: str - token: str - reason: CompromiseReason - error: Exception | None = None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class _Heartbeat: - # A running heartbeat and the event that stops it, which only ever exist together. - thread: Thread - stop: Event - - -class LeaseClaim: - # The state of one claim: its token, its heartbeat, and how that claim was lost. - __slots__ = ("compromise", "heartbeat", "token") - - def __init__(self) -> None: - self.token: str | None = None - self.compromise: LeaseCompromise | None = None - self.heartbeat: _Heartbeat | None = None - - class SoftFileLease(MarkerSoftFileLock): """ Existence lock whose claim expires, so a peer may take it while the previous holder still runs. @@ -357,6 +323,16 @@ def _read_peer(self) -> tuple[OwnerRecord, os.stat_result] | None: return None +class LeaseClaim: + # The state of one claim: its token, its heartbeat, and how that claim was lost. + __slots__ = ("compromise", "heartbeat", "token") + + def __init__(self) -> None: + self.token: str | None = None + self.compromise: LeaseCompromise | None = None + self.heartbeat: _Heartbeat | None = None + + @dataclass(frozen=True, slots=True, kw_only=True) class _RefreshLoop: # What a heartbeat thread needs to refresh one claim, holding the lease itself only weakly. @@ -421,6 +397,30 @@ def _touch(self) -> tuple[_RefreshOutcome, OSError | None]: return "ok", None +@dataclass(frozen=True, slots=True, kw_only=True) +class _Heartbeat: + # A running heartbeat and the event that stops it, which only ever exist together. + thread: Thread + stop: Event + + +@dataclass(frozen=True) +class LeaseCompromise: + """ + Why a held lease stopped being this process's to hold, and the error that ended its refresh, if any. + + :param lock_file: the path of the lock the lease was held on + :param token: the token of the claim that was lost + :param reason: what took the claim away: the marker vanished, its owner changed, a refresh failed, or it was evicted + :param error: the error that ended the refresh, when a refresh failed with one + """ + + lock_file: str + token: str + reason: CompromiseReason + error: Exception | None = None + + __all__ = [ "CompromiseReason", "LeaseClaim", diff --git a/src/filelock/_marker.py b/src/filelock/_marker.py index 8e3fcb85..05af920c 100644 --- a/src/filelock/_marker.py +++ b/src/filelock/_marker.py @@ -139,38 +139,6 @@ def _published_record(self) -> OwnerRecord: ) -class OwnerRecord(NamedTuple): - """ - Keep optional metadata when reading unknown lock modes. - - :param pid: the PID of the process that wrote the marker - :param hostname: the host name recorded in the marker - :param mode: the owner mode the marker declares: ``lease``, ``exclusive``, or ``unknown`` - :param token: the claim token of a lease, or ``None`` - :param lease_duration: the lease duration in seconds, or ``None`` when the marker records none - :param start: the owner's process start token, or ``None`` when none was recorded - """ - - pid: int - hostname: str - mode: OwnerMode - token: str | None = None - lease_duration: float | None = None - start: int | None = None - - -def encode_marker(record: OwnerRecord) -> bytes: - # Render an owner record as the bytes a protocol 2 marker holds. - lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"] - if record.token is not None: - lines.append(f"token={record.token}") - if record.lease_duration is not None: - lines.append(f"duration={record.lease_duration!r}") - if record.start is not None: - lines.append(f"start={record.start}") - return "".join(f"{line}\n" for line in lines).encode() - - def parse_marker(content: str | None) -> OwnerRecord | None: # Return the owner a protocol 2 marker names, or ``None`` when the record is malformed or protocol 1. # A torn write may end in a prefix like "mode=exclu", an unknown contract nobody reclaims; malformed, it ages out. @@ -212,6 +180,38 @@ def _build_record(fields: dict[str, str]) -> OwnerRecord | None: return OwnerRecord(pid=pid, hostname=hostname, mode=mode, token=token, lease_duration=duration, start=start) +def encode_marker(record: OwnerRecord) -> bytes: + # Render an owner record as the bytes a protocol 2 marker holds. + lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"] + if record.token is not None: + lines.append(f"token={record.token}") + if record.lease_duration is not None: + lines.append(f"duration={record.lease_duration!r}") + if record.start is not None: + lines.append(f"start={record.start}") + return "".join(f"{line}\n" for line in lines).encode() + + +class OwnerRecord(NamedTuple): + """ + Keep optional metadata when reading unknown lock modes. + + :param pid: the PID of the process that wrote the marker + :param hostname: the host name recorded in the marker + :param mode: the owner mode the marker declares: ``lease``, ``exclusive``, or ``unknown`` + :param token: the claim token of a lease, or ``None`` + :param lease_duration: the lease duration in seconds, or ``None`` when the marker records none + :param start: the owner's process start token, or ``None`` when none was recorded + """ + + pid: int + hostname: str + mode: OwnerMode + token: str | None = None + lease_duration: float | None = None + start: int | None = None + + __all__ = [ "MarkerSoftFileLock", "OwnerMode", diff --git a/src/filelock/_read_write.py b/src/filelock/_read_write.py index 50454fd8..415ea08e 100644 --- a/src/filelock/_read_write.py +++ b/src/filelock/_read_write.py @@ -406,10 +406,13 @@ class _OwnedConnection: # connection while the singleton cache still hands the lock out, and a thread given the lock would nest into a hold # with no database lock behind it. The lock alone refers to this holder, so the holder is collected once the cache # no longer finds the lock. - __slots__ = ("connection",) + __slots__ = ("connection", "identity", "transaction_released") def __init__(self) -> None: self.connection: _ForkSafeConnection | None = None + #: Whether the connection's transaction is known to be over, so a release does not roll back twice. + self.transaction_released = True + self.identity: _DatabaseIdentity | None = None def __del__(self) -> None: if (connection := self.connection) is not None: @@ -417,6 +420,34 @@ def __del__(self) -> None: connection.close() +class _Hold: + # The levels an instance holds and the threads holding them; level 0 when it holds none. + __slots__ = ("level", "mode", "read_depths", "write_thread_id") + + def __init__(self) -> None: + self.level = 0 + self.mode: Literal["read", "write"] | None = None + self.write_thread_id: int | None = None + self.read_depths: dict[int, int] = {} + + def grant(self, mode: Literal["read", "write"], holder: int) -> None: + self.mode, self.level = mode, 1 + if mode == "write": + self.write_thread_id = holder + else: + self.read_depths = {holder: 1} + + def nest(self, mode: Literal["read", "write"], holder: int) -> None: + if mode == "read": + self.read_depths[holder] = self.read_depths.get(holder, 0) + 1 + self.level += 1 + + def leave_one_level(self, thread_id: int) -> None: + self.level -= 1 + if self.mode == "read" and (depth := self.read_depths.pop(thread_id) - 1): + self.read_depths[thread_id] = depth + + class _ReadWriteLockMeta(type): # Resolve singleton instances for ``is_singleton=True`` construction. # @@ -545,15 +576,10 @@ def __init__( self.timeout = timeout self.blocking = blocking self._transaction_lock = threading.Lock() # serializes the (possibly blocking) SQLite transaction work - self._internal_lock = threading.Lock() # protects _lock_level / _current_mode updates and rollback - self._lock_level = 0 - self._current_mode: Literal["read", "write"] | None = None - self._write_thread_id: int | None = None - self._read_depths: dict[int, int] = {} + self._internal_lock = threading.Lock() # protects _hold updates and rollback + self._hold = _Hold() self._acquisition_thread_ids: set[int] = set() self._owned_connection = _OwnedConnection() - self._connection_transaction_released = True - self._connection_identity: _DatabaseIdentity | None = None self._closed = False self._creator_pid = _GETPID() self._fork_invalidated = False @@ -741,12 +767,12 @@ def _inherited(self) -> bool: def _holds_level(self) -> bool: # A release interrupted after its rollback, but before it recorded that, leaves a level with no transaction. - if self._lock_level > 0 and self._transaction_ended(): + if self._hold.level > 0 and self._transaction_ended(): self._clear_lock_state() - return self._lock_level > 0 + return self._hold.level > 0 def _transaction_ended(self) -> bool: - if self._connection_transaction_released or (connection := self._con) is None: + if self._owned_connection.transaction_released or (connection := self._con) is None: return True # An interrupt can land between the rollback and the flag that records it. return not connection.in_transaction @@ -755,20 +781,13 @@ def _transaction_ended(self) -> bool: def _con(self) -> _ForkSafeConnection | None: return self._owned_connection.connection - @_con.setter - def _con(self, connection: _ForkSafeConnection | None) -> None: - self._owned_connection.connection = connection - def _clear_lock_state(self) -> None: - self._lock_level = 0 - self._current_mode = None - self._write_thread_id = None - self._read_depths = {} + self._hold = _Hold() def _validate_reentrant(self, mode: Literal["read", "write"], cur: int) -> AcquireReturnProxy: - if self._current_mode != mode: + if self._hold.mode != mode: opposite = "write" if mode == "read" else "read" - if cur == self._write_thread_id or cur in self._read_depths: + if cur == self._hold.write_thread_id or cur in self._hold.read_depths: direction = "downgrade" if mode == "read" else "upgrade" reason = f"already holding a {opposite} lock ({direction} not allowed)" else: @@ -778,15 +797,13 @@ def _validate_reentrant(self, mode: Literal["read", "write"], cur: int) -> Acqui ) msg = f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): {reason}" raise RuntimeError(msg) - if mode == "write" and cur != self._write_thread_id: + if mode == "write" and cur != self._hold.write_thread_id: msg = ( f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) " - f"from thread {cur} while it is held by thread {self._write_thread_id}" + f"from thread {cur} while it is held by thread {self._hold.write_thread_id}" ) raise RuntimeError(msg) - if mode == "read": - self._read_depths[cur] = self._read_depths.get(cur, 0) + 1 - self._lock_level += 1 + self._hold.nest(mode, cur) return AcquireReturnProxy(lock=self) def _acquire_transaction_lock( @@ -841,12 +858,7 @@ def _do_acquire_inner( raise with self._internal_lock: self._raise_if_process_changed(operation_pid) - self._current_mode = mode - self._lock_level = 1 - if mode == "write": - self._write_thread_id = holder - else: - self._read_depths = {holder: 1} + self._hold.grant(mode, holder) return AcquireReturnProxy(lock=self) def _finish_connection(self) -> None: @@ -854,28 +866,28 @@ def _finish_connection(self) -> None: if (connection := self._con) is None: return rollback_error: sqlite3.Error | None = None - if not self._connection_transaction_released: + if not self._owned_connection.transaction_released: if connection.in_transaction: try: connection.rollback() except sqlite3.Error as error: if connection.in_transaction: raise - self._connection_transaction_released = True + self._owned_connection.transaction_released = True rollback_error = error else: - self._connection_transaction_released = True + self._owned_connection.transaction_released = True else: - self._connection_transaction_released = True + self._owned_connection.transaction_released = True try: connection.close() except sqlite3.Error as close_error: if rollback_error is not None: _raise_chained_errors(rollback_error, close_error) raise - self._con = None - self._connection_transaction_released = True - self._connection_identity = None + self._owned_connection.connection = None + self._owned_connection.transaction_released = True + self._owned_connection.identity = None if rollback_error is not None: raise rollback_error @@ -890,7 +902,8 @@ def _open_for_acquisition(self, timeout: float, *, blocking: bool, operation_pid / 1000 ) connection = self._open_connection(sqlite_timeout=sqlite_timeout) - self._con, self._connection_transaction_released, self._connection_identity = ( + owned = self._owned_connection + owned.connection, owned.transaction_released, owned.identity = ( connection, False, _FORKED_DATABASES.identity(self._canonical_path), @@ -1003,7 +1016,7 @@ def _release(self, *, force: bool, close: bool, thread_id: int | None = None) -> with self._transaction_lock, self._internal_lock: if not force: self._raise_if_not_holder(thread_id) - if self._lock_level == 0: + if self._hold.level == 0: if force and self._con is None: if close: self._closed = True @@ -1011,8 +1024,8 @@ def _release(self, *, force: bool, close: bool, thread_id: int | None = None) -> if not force: msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held" raise RuntimeError(msg) - if not force and self._lock_level > 1: - self._leave_one_level(thread_id) + if not force and self._hold.level > 1: + self._hold.leave_one_level(thread_id) return try: self._finish_connection() @@ -1029,25 +1042,20 @@ def _release(self, *, force: bool, close: bool, thread_id: int | None = None) -> def _raise_if_not_holder(self, thread_id: int) -> None: # The connection runs with check_same_thread=False, so a release from another thread would end the owner's # transaction under it and let a second writer in. - if self._current_mode == "write" and thread_id != self._write_thread_id: + if self._hold.mode == "write" and thread_id != self._hold.write_thread_id: msg = ( f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) " - f"from thread {thread_id} while it is held by thread {self._write_thread_id}" + f"from thread {thread_id} while it is held by thread {self._hold.write_thread_id}" ) raise RuntimeError(msg) # Readers share that one transaction too, so a thread that holds no read level would end another's hold. - if self._current_mode == "read" and thread_id not in self._read_depths: + if self._hold.mode == "read" and thread_id not in self._hold.read_depths: msg = ( f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) " f"from thread {thread_id}, which does not hold it" ) raise RuntimeError(msg) - def _leave_one_level(self, thread_id: int) -> None: - self._lock_level -= 1 - if self._current_mode == "read" and (depth := self._read_depths.pop(thread_id) - 1): - self._read_depths[thread_id] = depth - @contextmanager def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: """ @@ -1132,7 +1140,7 @@ def _holds_level_of(self, thread_id: int) -> bool: # Read without the internal lock. The async wrapper asks from its event loop, which must not wait out a release # on another thread. A stale answer is what a caller asking a moment earlier would have had. A level whose # transaction ended is one an interrupted release left behind; it counts as gone, as _holds_level() treats it. - if thread_id != self._write_thread_id and thread_id not in self._read_depths: + if thread_id != self._hold.write_thread_id and thread_id not in self._hold.read_depths: return False try: return not self._transaction_ended() @@ -1142,10 +1150,10 @@ def _holds_level_of(self, thread_id: int) -> bool: def _reset_after_fork_in_child(self) -> None: # pragma: forked child if self._con is not None: - _FORKED_DATABASES.poison_after_fork(self._canonical_path, self._connection_identity) - self._con = None - self._connection_transaction_released = True - self._connection_identity = None + _FORKED_DATABASES.poison_after_fork(self._canonical_path, self._owned_connection.identity) + self._owned_connection.connection = None + self._owned_connection.transaction_released = True + self._owned_connection.identity = None self._transaction_lock = threading.Lock() self._internal_lock = threading.Lock() self._clear_lock_state() diff --git a/src/filelock/_soft_rw/_protocol.py b/src/filelock/_soft_rw/_protocol.py index 151cfeca..310946ec 100644 --- a/src/filelock/_soft_rw/_protocol.py +++ b/src/filelock/_soft_rw/_protocol.py @@ -105,66 +105,6 @@ def prepare(self, root: str, *, force: bool = False) -> None: ... -@dataclass(frozen=True, slots=True, kw_only=True) -class Snapshot: - # The lock's state at one generation: which token writes, which tokens read. - generation: int - writer: str | None - readers: frozenset[str] - - @property - def members(self) -> frozenset[str]: - return self.readers if self.writer is None else self.readers | {self.writer} - - def encode(self) -> bytes: - lines = [_PROTOCOL, f"generation={self.generation}"] - if self.writer is not None: - lines.append(f"writer={self.writer}") - lines.extend(f"reader={token}" for token in sorted(self.readers)) - return "".join(f"{line}\n" for line in lines).encode("ascii") - - -def parse_snapshot(data: bytes) -> Snapshot | None: - # The snapshot a ``gen/`` record encodes, or ``None`` when the record is malformed. - if not data or len(data) > MAX_RECORD_SIZE: - return None - try: - lines = data.decode("ascii").split("\n") - except UnicodeDecodeError: - return None - if lines[0] != _PROTOCOL or lines[-1]: - return None - generation: int | None = None - writer: str | None = None - readers: set[str] = set() - for line in lines[1:-1]: - key, _, value = line.partition("=") - if ( - key == "generation" - and generation is None - and len(value) <= _GENERATION_DIGITS - and value.isdigit() - and str(int(value)) == value - ): - generation = int(value) - elif key == "writer" and writer is None and _is_token(value): - writer = value - elif key == "reader" and _is_token(value) and value not in readers: - readers.add(value) - else: - return None - return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers)) - - -def _is_token(value: str) -> bool: - # Whether *value* is a claim token: exactly 32 lowercase hex digits. - return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value) - - -def new_token() -> str: - return secrets.token_hex(_TOKEN_HEX_LENGTH // 2) - - class Ledger: # How long each observed value has stayed the same, measured on this observer's monotonic clock. # @@ -391,6 +331,66 @@ def _compact_through(self, expired: int) -> None: self._files.unlink(self._path(generation)) +@dataclass(frozen=True, slots=True, kw_only=True) +class Snapshot: + # The lock's state at one generation: which token writes, which tokens read. + generation: int + writer: str | None + readers: frozenset[str] + + @property + def members(self) -> frozenset[str]: + return self.readers if self.writer is None else self.readers | {self.writer} + + def encode(self) -> bytes: + lines = [_PROTOCOL, f"generation={self.generation}"] + if self.writer is not None: + lines.append(f"writer={self.writer}") + lines.extend(f"reader={token}" for token in sorted(self.readers)) + return "".join(f"{line}\n" for line in lines).encode("ascii") + + +def new_token() -> str: + return secrets.token_hex(_TOKEN_HEX_LENGTH // 2) + + +def parse_snapshot(data: bytes) -> Snapshot | None: + # The snapshot a ``gen/`` record encodes, or ``None`` when the record is malformed. + if not data or len(data) > MAX_RECORD_SIZE: + return None + try: + lines = data.decode("ascii").split("\n") + except UnicodeDecodeError: + return None + if lines[0] != _PROTOCOL or lines[-1]: + return None + generation: int | None = None + writer: str | None = None + readers: set[str] = set() + for line in lines[1:-1]: + key, _, value = line.partition("=") + if ( + key == "generation" + and generation is None + and len(value) <= _GENERATION_DIGITS + and value.isdigit() + and str(int(value)) == value + ): + generation = int(value) + elif key == "writer" and writer is None and _is_token(value): + writer = value + elif key == "reader" and _is_token(value) and value not in readers: + readers.add(value) + else: + return None + return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers)) + + +def _is_token(value: str) -> bool: + # Whether *value* is a claim token: exactly 32 lowercase hex digits. + return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value) + + class Participant: # One acquisition, from publishing a claim through holding the lock to leaving. # @@ -419,14 +419,10 @@ def __init__( # ruff:ignore[too-many-arguments] # one value per protocol input self._clock = clock self._ledger = ledger self._log = log - self._holder = self._holder_path(self.token) self._holder_identity: tuple[int, int] | None = None self._entered = False self._next_sweep = clock() - def _holder_path(self, token: str) -> str: - return str(Path(self._root, HOLDERS_DIRECTORY, token)) - def publish(self) -> None: # Create the holder record peers watch for liveness; it exists before the token appears in any snapshot. self._files.prepare(self._root) @@ -434,11 +430,14 @@ def publish(self) -> None: def _create_holder(self) -> None: try: - self._holder_identity = self._files.create(self._holder, self._record()) + self._holder_identity = self._files.create(self._holder_path(self.token), self._record()) except FileNotFoundError: # Removing the whole log takes the directories with it. self._files.prepare(self._root, force=True) - self._holder_identity = self._files.create(self._holder, self._record()) + self._holder_identity = self._files.create(self._holder_path(self.token), self._record()) + + def _holder_path(self, token: str) -> str: + return str(Path(self._root, HOLDERS_DIRECTORY, token)) def _record(self) -> bytes: return encode_holder(self.token, new_token(), self._stale_threshold) @@ -495,7 +494,7 @@ def _keep_claim_fresh(self, latest: Snapshot) -> None: def _refresh_nonce(self) -> bool: if (identity := self._holder_identity) is None: return False - return self._files.overwrite(self._holder, self._record(), identity) + return self._files.overwrite(self._holder_path(self.token), self._record(), identity) def _sweep(self, latest: Snapshot) -> None: # A participant that crashed before its first commit leaves a record no snapshot names, and a commit that @@ -580,7 +579,7 @@ def leave(self) -> None: finally: self._entered = False self.generation = None - self._files.unlink(self._holder) + self._files.unlink(self._holder_path(self.token)) def encode_holder(token: str, nonce: str, lease: float) -> bytes: diff --git a/src/filelock/_soft_rw/_sync.py b/src/filelock/_soft_rw/_sync.py index a0b6363c..1e642fac 100644 --- a/src/filelock/_soft_rw/_sync.py +++ b/src/filelock/_soft_rw/_sync.py @@ -12,6 +12,7 @@ __lazy_modules__ = [ "filelock._error", "filelock._lease", + "functools", "math", f"{__spec__.parent}._protocol", f"{__spec__.parent}._storage", @@ -23,6 +24,7 @@ import time from contextlib import contextmanager, suppress from dataclasses import dataclass +from functools import partial from math import isfinite from typing import TYPE_CHECKING, ClassVar, Final, NamedTuple from weakref import WeakValueDictionary @@ -153,7 +155,7 @@ def __call__( # ruff:ignore[too-many-arguments] # forwards the public construc f" cannot be changed to timeout={timeout}, blocking={blocking}" ) raise ValueError(msg) - elif on_compromise is not None and instance._on_compromise != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares + elif on_compromise is not None and instance._losses.callback != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares # A tuning difference keeps the first caller's values; a different loss callback is a different safety # contract, and returning the cached instance would silently drop it. msg = f"Singleton lock created with a different on_compromise callback for {lock_file!s}" @@ -256,14 +258,19 @@ def __init__( # ruff:ignore[too-many-arguments] # public constructor: one para # Resolved once: a relative lock path must keep naming the same log after the process changes directory. The # final component stays unresolved, as in 4.0.12, so every release shares one log on a symlinked lock path. - self._root = f"{_canonical(self.lock_file)}.rw" - self._files = OsFiles(self.lock_file) - self._ledger = Ledger(time.monotonic) - self._log = GenerationLog(self._files, self.lock_file, self._root) - self._on_compromise = on_compromise + root = f"{_canonical(self.lock_file)}.rw" + files = OsFiles(self.lock_file) + # The ledger and log outlive one acquisition, so every participant of this instance shares them. + self._new_participant: Final = partial( + Participant, + files, + root, + ledger=Ledger(time.monotonic), + log=GenerationLog(files, self.lock_file, root), + ) + self._losses = _Losses(callback=on_compromise) self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock()) self._hold: _Hold | None = None - self._compromise: LeaseCompromise | None = None self._closed: bool = False with _ALL_INSTANCES_LOCK: @@ -321,7 +328,7 @@ def compromise(self) -> LeaseCompromise | None: """ with self._locks.internal: - return self._compromise + return self._losses.latest @classmethod def _reset_class_after_fork(cls) -> None: # pragma: forked child @@ -441,7 +448,7 @@ def _validate_reentrant(self, mode: Mode, holder: int) -> AcquireReturnProxy: raise RuntimeError(msg) # Publishing a hold clears the loss recorded for the one before, so a loss seen here is this hold's. Its # heartbeat has stopped and a peer may hold the lock, so a new level must not bypass the log. - if (compromise := self._compromise) is not None: + if (compromise := self._losses.latest) is not None: raise lost_hold_error(self.lock_file, mode, compromise) if mode == "read": hold.read_depths[holder] = hold.read_depths.get(holder, 0) + 1 @@ -460,15 +467,7 @@ def _do_acquire_inner( with self._locks.internal: if self._hold is not None: return self._validate_reentrant(mode, holder) - participant = Participant( - self._files, - self._root, - mode, - stale_threshold=self.stale_threshold, - clock=time.monotonic, - ledger=self._ledger, - log=self._log, - ) + participant = self._new_participant(mode, stale_threshold=self.stale_threshold, clock=time.monotonic) participant.publish() try: self._wait_for(participant.advance, deadline=deadline, blocking=blocking, cancel_check=cancel_check) @@ -506,7 +505,7 @@ def _do_acquire_inner( start_error = self._closed_error() else: self._hold = hold - self._compromise = None + self._losses.latest = None try: hold.heartbeat_thread.start() except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise @@ -569,9 +568,9 @@ def _report_compromise(self, hold: _Hold, reason: CompromiseReason, error: Excep compromise = LeaseCompromise( lock_file=self.lock_file, token=hold.participant.token, reason=reason, error=error ) - self._compromise = compromise - if self._on_compromise is not None: - self._on_compromise(compromise) + self._losses.latest = compromise + if self._losses.callback is not None: + self._losses.callback(compromise) def release(self, *, force: bool = False) -> None: """ @@ -792,6 +791,13 @@ def _validate_intervals(heartbeat_interval: float, stale_threshold: float | None return _Intervals(heartbeat_interval, stale_threshold, poll_interval) +@dataclass(slots=True, kw_only=True) +class _Losses: + # The caller's handler for a lost hold and the latest loss, which stays readable until the next hold is published. + callback: Callable[[LeaseCompromise], None] | None + latest: LeaseCompromise | None = None + + @dataclass(frozen=True, slots=True, kw_only=True) class _Locks: internal: threading.Lock diff --git a/src/filelock/_strict.py b/src/filelock/_strict.py index 9dc4ea48..77c1467b 100644 --- a/src/filelock/_strict.py +++ b/src/filelock/_strict.py @@ -1,5 +1,6 @@ from __future__ import annotations +from types import MappingProxyType from typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -101,7 +102,10 @@ _CLAIM_READ_RETRY: Final[float] = 0.002 #: Rides out a peer replacing the lock path mid-inspection; a foreign node or unreadable record outlasting it is damage. _SENTINEL_READ_GRACE: Final[float] = 0.5 -_NODE_DESCRIPTIONS: Final[dict[int, str]] = {stat.S_IFDIR: "a directory", stat.S_IFLNK: "a symlink"} +_NODE_DESCRIPTIONS: Final[MappingProxyType[int, str]] = MappingProxyType({ + stat.S_IFDIR: "a directory", + stat.S_IFLNK: "a symlink", +}) #: Windows opens descriptors in text mode by default, which rewrites newlines and truncates a record at a control byte. #: The claim and sentinel records are exact binary, so every record descriptor must be binary; POSIX ignores the flag. _O_BINARY: Final[int] = getattr(os, "O_BINARY", 0) @@ -408,121 +412,36 @@ def _release(self) -> None: _raise_recorded_errors("strict release cleanup failed", errors) -@dataclass(frozen=True) -class StrictSoftFileClaim: - """ - One parsed strict soft-lock claim. - - :param name: the claim's file name in the claim directory - :param state: whether the claim is an ``intent`` or a ``held`` record - :param token: the random token that identifies the claim - :param pid: the PID of the process that wrote the claim - :param hostname: the host name of the process that wrote the claim - :param start: the owner's process start token, or ``None`` when the platform exposes no proven start time - """ - - name: str - state: StrictSoftFileClaimState - token: str - pid: int - hostname: str - #: The owner's process start token, or ``None`` when the platform exposes no proven start time. A strict lock never - #: reclaims a claim on its own, so this identifies the owner for tooling rather than driving any automatic break. - start: int | None = None - - -class _PrivateRecordReclaimedError(Exception): - pass - - -def _open_or_create_sentinel( - lock_file: str, path: Path, mode: int, unchanged_for: Callable[[os.stat_result], float] -) -> int | None: - try: - return _open_sentinel(lock_file, path, unchanged_for) - except FileNotFoundError: - pass - - _reclaim_sentinel_private_records(lock_file, path) - try: - publication_cleanup_error = _publish_record(os.fspath(path), _LEGACY_SENTINEL, mode) - except _PrivateRecordReclaimedError: - return None - except (NotImplementedError, OSError) as error: - raise_if_hard_links_unsupported(lock_file, error) - if not isinstance(error, OSError) or error.errno != EEXIST: - raise - else: - if publication_cleanup_error is not None: # pragma: needs dir-fd - raise publication_cleanup_error - try: - return _open_sentinel(lock_file, path, unchanged_for) - except FileNotFoundError: - return None - - -def _open_sentinel(lock_file: str, path: Path, unchanged_for: Callable[[os.stat_result], float]) -> int | None: - # A live soft-lock owner blocks like any holder. Anything else raises: claims cannot show it and force_break() - # cannot remove it, so waiting would wedge every strict contender with no diagnostic. A file no lock parses may be a - # soft lock between its create and its write, so it raises only once unchanged as long as a soft contender waits. - deadline: float | None = None - while True: - fd, state, occupant = _inspect_lock_path(path) - if fd is not None or state == "held": - return fd - reason = f"expected the strict sentinel at the lock path, found {occupant}" - if state == "unparsable": - if unchanged_for(path.lstat()) < MALFORMED_LOCK_AGE_THRESHOLD: - return None - raise SoftFileLockProtocolError( - lock_file, None, f"{reason}, unchanged for {MALFORMED_LOCK_AGE_THRESHOLD:g} seconds" +def _read_existing_claims( + lock_file: str, directory: Path, *, probe_clock: bool = True, verify_server: bool = sys.platform != "win32" +) -> tuple[StrictSoftFileClaim, ...]: + if not _OPEN_SUPPORTS_DIR_FD: # pragma: lacks dir-fd + try: + pinned = _pin_claim_directory(directory) + except FileNotFoundError: + return () + except OSError as error: + reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + with pinned: + return _read_claims( + lock_file, (os.fspath(directory), None), probe_clock=probe_clock, verify_server=verify_server ) - if deadline is None: - deadline = time.monotonic() + _SENTINEL_READ_GRACE - if state == "exited" or time.monotonic() >= deadline: - raise SoftFileLockProtocolError(lock_file, None, reason) - time.sleep(_CLAIM_READ_RETRY) - - -class _LockPathInspection(NamedTuple): - fd: int | None - state: _OccupantState - occupant: str - - -def _inspect_lock_path(path: Path) -> _LockPathInspection: - # "held" may be a soft-lock owner that still runs, "exited" is one this host proves gone, "unparsable" is content no - # lock parses, and "damaged" is a node no lock writes. - if not stat.S_ISREG(mode := path.lstat().st_mode): - return _LockPathInspection(None, "damaged", _NODE_DESCRIPTIONS.get(stat.S_IFMT(mode), "a special file")) - try: - fd, record = _open_record((os.fspath(path.parent), None), path.name, _CLAIM_RECORD_LIMIT) - except FileNotFoundError: - raise - except PermissionError as error: - # Windows denies the read while a legacy holder keeps its marker open, and a holder running as another user - # may leave it unreadable here, so a marker that cannot be read may still be held. - return _LockPathInspection(None, "held", f"a file it cannot read ({error.strerror or error})") - except OSError as error: - return _LockPathInspection(None, "damaged", f"a file it cannot read ({error.strerror or error})") - if record == _LEGACY_SENTINEL: - return _LockPathInspection(fd, "sentinel", "the strict sentinel") - os.close(fd) - return _LockPathInspection(None, *_describe_soft_marker(record.decode("utf-8", errors="replace"))) - - -def _describe_soft_marker(content: str) -> tuple[_OccupantState, str]: - # A SoftFileLock, MarkerSoftFileLock or SoftFileLease owner holds the path legitimately until it releases, so its - # marker is contention. One this host proves gone is named, never removed: strict never acts on an inferred death. - if (record := parse_marker(content)) is not None: - pid, hostname, start = record.pid, record.hostname, record.start - elif (holder := _parse_lock_holder(content)) is not None: - pid, hostname, start = holder - else: - return "unparsable", "a file that is neither the sentinel nor a soft-lock marker it can parse" - if owner_is_stale(pid, hostname, start): - return "exited", f"a soft-lock marker left by pid {pid} on {hostname}, which has exited" - return "held", f"a soft-lock marker held by pid {pid} on {hostname}" + try: # pragma: needs dir-fd + directory_fd = _open_claim_directory(os.fspath(directory)) + except FileNotFoundError: # pragma: needs dir-fd + return () + except OSError as error: # pragma: needs dir-fd + reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + try: # pragma: needs dir-fd + return _read_claims( + lock_file, (os.fspath(directory), directory_fd), probe_clock=probe_clock, verify_server=verify_server + ) + finally: # pragma: needs dir-fd + # A failed close has no result to report after the scan and proof cleanup. + with contextlib.suppress(OSError): + os.close(directory_fd) def _read_claims( @@ -542,29 +461,6 @@ def _read_claims( ) -def _claim_exists_on_server(lock_file: str, directory_ref: tuple[str, int | None], name: str) -> bool: - # Reading an NFS claim can succeed from a client cache after its server-side name is gone. LINK checks the source - # on the server; a private proof is reclaimed if this process exits before unlinking it. - proof = f"{_CLAIM_PROBE_PREFIX}{secrets.token_hex(_TOKEN_HEX_LENGTH // 2)}{_PRIVATE_RECORD_SUFFIX}" - try: - _link_or_find_committed(directory_ref, name, proof) - except OSError as error: - if error.errno in {ENOENT, ESTALE}: - return False - reason = f"cannot verify claim: {error.strerror or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, name, reason) from error - except NotImplementedError as error: - raise SoftFileLockProtocolError(lock_file, name, "cannot verify claim: hard links unavailable") from error - try: - _unlink_relative(directory_ref, proof) - except FileNotFoundError: - pass - except OSError as error: - reason = f"cannot remove claim probe: {error.strerror or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, name, reason) from error - return True - - def _list_claim_names(lock_file: str, directory_ref: tuple[str, int | None], *, probe_clock: bool) -> list[str]: try: with _scandir_relative(directory_ref) as entries: @@ -574,72 +470,6 @@ def _list_claim_names(lock_file: str, directory_ref: tuple[str, int | None], *, raise SoftFileLockProtocolError(lock_file, None, reason) from error -def _read_claim_records( - lock_file: str, directory_ref: tuple[str, int | None], names: list[str], *, probe_clock: bool -) -> dict[str, bytes]: - # A contended scan can list a claim that is not yet cleanly readable, in two ways that both resolve on a brief - # retry. Windows fails an open with EACCES while a peer's unlink holds the claim; once the deletion is pending, - # _attempt_claim_read skips the claim outright, as another process's handle can keep the name listed for seconds. - # - # On NFS, a peer that unlinks its own claim leaves this client's cached filehandle stale, so the next open returns - # ESTALE rather than a clean ENOENT until the lookup revalidates against the server. Retrying re-runs the path - # lookup, which turns the vanished claim into ENOENT (skip) or reads it if it still exists. A genuinely unreadable - # record (a locked-down file, an EIO fault) still fails closed. - # - # The grace starts at the first pending result rather than before the first attempt, so a slow first read cannot use - # up the window and leave no retry. One deadline covers the scan: a deadline per claim let claims that clear one - # after another stretch a scan, and the caller's timeout with it, by a grace each. - records: dict[str, bytes] = {} - deadline: float | None = None - while True: - pending: dict[str, OSError] = {} - for name in names: - record, error = _attempt_claim_read(lock_file, directory_ref, name) - if error is not None: - pending[name] = error - elif record is not None: - records[name] = record - if not pending: - return records - if deadline is None: - deadline = time.monotonic() + _CLAIM_READ_GRACE - elif time.monotonic() >= deadline: - break - names = list(pending) - time.sleep(_CLAIM_READ_RETRY) - # A stale handle that outlives revalidation is a claim the server no longer has (RFC 1813 NFS3ERR_STALE): skip it - # like ENOENT. Skipping a vanished claim overcounts contention at worst; it cannot free a held lock. - if denied := [name for name, error in pending.items() if error.errno != ESTALE]: - # Windows can deny opens after a peer removes the claim from the directory. - listed: Final = set(_list_claim_names(lock_file, directory_ref, probe_clock=probe_clock)) - if still_listed := [name for name in denied if name in listed]: - error = pending[still_listed[0]] - reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, still_listed[0], reason) from error - return records - - -def _attempt_claim_read( - lock_file: str, directory_ref: tuple[str, int | None], name: str -) -> tuple[bytes | None, OSError | None]: - # Return the record, or (None, None) when the claim has already gone, or (None, error) for an open the caller may - # retry: a Windows EACCES while a peer's unlink holds the claim, or an NFS ESTALE from a peer unlinking its own - # claim out from under this client's cached filehandle. Any other OSError is a real fault and fails closed. - try: - return _read_record(directory_ref, name, _CLAIM_RECORD_LIMIT), None - except FileNotFoundError: - return None, None - except PermissionError as error: - if sys.platform == "win32" and _is_unlinked(str(Path(directory_ref[0], name))): # pragma: win32 cover - return None, None - return None, error - except OSError as error: - if error.errno == ESTALE: - return None, error - reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, name, reason) from error - - def _public_claim_names( directory_ref: tuple[str, int | None], entries: Iterator[os.DirEntry[str]], *, probe_clock: bool ) -> list[str]: @@ -663,93 +493,576 @@ def _public_claim_names( return sorted(names) -def _read_existing_claims( - lock_file: str, directory: Path, *, probe_clock: bool = True, verify_server: bool = sys.platform != "win32" -) -> tuple[StrictSoftFileClaim, ...]: - if not _OPEN_SUPPORTS_DIR_FD: # pragma: lacks dir-fd - try: - pinned = _pin_claim_directory(directory) - except FileNotFoundError: - return () - except OSError as error: - reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - with pinned: - return _read_claims( - lock_file, (os.fspath(directory), None), probe_clock=probe_clock, verify_server=verify_server - ) - try: # pragma: needs dir-fd - directory_fd = _open_claim_directory(os.fspath(directory)) - except FileNotFoundError: # pragma: needs dir-fd - return () - except OSError as error: # pragma: needs dir-fd - reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - try: # pragma: needs dir-fd - return _read_claims( - lock_file, (os.fspath(directory), directory_fd), probe_clock=probe_clock, verify_server=verify_server - ) - finally: # pragma: needs dir-fd - # A failed close has no result to report after the scan and proof cleanup. +def _filesystem_now(directory_ref: tuple[str, int | None], public_name: str) -> float | None: + # A network filesystem stamps mtimes with the server's clock, which can run seconds off this client's, so read "now" + # as the mtime of a probe written like a record (liblockfile uses a read's atime). The probe has a private-record + # name, so a scan reaps one left behind. Without a stamp no age is trusted. + probe = _private_record_name(public_name) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = _open_relative(directory_ref, probe, flags, 0o600) + except OSError: + return None + try: + write_all(fd, b"\n") + return os.fstat(fd).st_mtime + except OSError: + return None + finally: with contextlib.suppress(OSError): - os.close(directory_fd) + os.close(fd) + with contextlib.suppress(OSError): + _unlink_private_record_once(directory_ref, probe) + + +def _private_record_name(public_name: str) -> str: + return f".{public_name}{_PRIVATE_RECORD_MARKER}{secrets.token_hex(_PRIVATE_RECORD_RANDOM_HEX_LENGTH // 2)}.tmp" + + +def _open_relative(directory_ref: tuple[str, int | None], name: str, flags: int, mode: int) -> int: + directory, directory_fd = directory_ref + return ( + os.open(name, flags, mode, dir_fd=directory_fd) + if directory_fd is not None + else os.open(Path(directory, name), flags, mode) + ) + + +def _unlink_private_record_once(directory_ref: tuple[str, int | None], private_name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + os.unlink(private_name, dir_fd=directory_fd) + else: + Path(directory, private_name).unlink() + + +def _is_claim_probe_name(name: str) -> bool: + if not name.startswith(_CLAIM_PROBE_PREFIX) or not name.endswith(_PRIVATE_RECORD_SUFFIX): + return False + token = name[len(_CLAIM_PROBE_PREFIX) : -len(_PRIVATE_RECORD_SUFFIX)] + return len(token) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in token) + + +def _private_public_name(private_name: str) -> str | None: + if not private_name.startswith(".") or not private_name.endswith(_PRIVATE_RECORD_SUFFIX): + return None + public_name, marker, random_hex = private_name[1 : -len(_PRIVATE_RECORD_SUFFIX)].rpartition(_PRIVATE_RECORD_MARKER) + if ( + marker != _PRIVATE_RECORD_MARKER + or len(random_hex) != _PRIVATE_RECORD_RANDOM_HEX_LENGTH + or any(character not in "0123456789abcdef" for character in random_hex) + ): + return None + return public_name + + +def _reclaim_private_record( + directory_ref: tuple[str, int | None], private_name: str, now: Callable[[], float | None] +) -> None: + directory, directory_fd = directory_ref + try: + private_stat = ( + os.stat(private_name, dir_fd=directory_fd, follow_symlinks=False) + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD + else Path(directory, private_name).lstat() + ) + except FileNotFoundError: + return + if not stat.S_ISREG(private_stat.st_mode): + msg = f"{Path(directory, private_name)} is not a regular private record" + raise OSError(msg) + private_record_grace: Final = 2.0 + if private_stat.st_nlink == 1 and ( + (current := now()) is None or current - private_stat.st_mtime < private_record_grace + ): + return + try: + _unlink_private_record_once(directory_ref, private_name) + except FileNotFoundError: + pass + except OSError as error: + if error.errno not in {EACCES, EPERM}: + raise + + +def _parse_claim_name(name: str) -> tuple[StrictSoftFileClaimState, str] | None: + if not name.endswith(".claim"): + return None + parts = name.removesuffix(".claim").split("-") + claim_states: Final = frozenset({"intent", "held"}) + claim_name_part_count: Final = 3 + if len(parts) != claim_name_part_count or parts[0] not in claim_states or parts[1] != "v1": + return None + token = parts[2] + if len(token) != _TOKEN_HEX_LENGTH or any(character not in "0123456789abcdef" for character in token): + return None + return cast("StrictSoftFileClaimState", parts[0]), token + + +def _read_claim_records( + lock_file: str, directory_ref: tuple[str, int | None], names: list[str], *, probe_clock: bool +) -> dict[str, bytes]: + # A contended scan can list a claim that is not yet cleanly readable, in two ways that both resolve on a brief + # retry. Windows fails an open with EACCES while a peer's unlink holds the claim; once the deletion is pending, + # _attempt_claim_read skips the claim outright, as another process's handle can keep the name listed for seconds. + # + # On NFS, a peer that unlinks its own claim leaves this client's cached filehandle stale, so the next open returns + # ESTALE rather than a clean ENOENT until the lookup revalidates against the server. Retrying re-runs the path + # lookup, which turns the vanished claim into ENOENT (skip) or reads it if it still exists. A genuinely unreadable + # record (a locked-down file, an EIO fault) still fails closed. + # + # The grace starts at the first pending result rather than before the first attempt, so a slow first read cannot use + # up the window and leave no retry. One deadline covers the scan: a deadline per claim let claims that clear one + # after another stretch a scan, and the caller's timeout with it, by a grace each. + records: dict[str, bytes] = {} + deadline: float | None = None + while True: + pending: dict[str, OSError] = {} + for name in names: + record, error = _attempt_claim_read(lock_file, directory_ref, name) + if error is not None: + pending[name] = error + elif record is not None: + records[name] = record + if not pending: + return records + if deadline is None: + deadline = time.monotonic() + _CLAIM_READ_GRACE + elif time.monotonic() >= deadline: + break + names = list(pending) + time.sleep(_CLAIM_READ_RETRY) + # A stale handle that outlives revalidation is a claim the server no longer has (RFC 1813 NFS3ERR_STALE): skip it + # like ENOENT. Skipping a vanished claim overcounts contention at worst; it cannot free a held lock. + if denied := [name for name, error in pending.items() if error.errno != ESTALE]: + # Windows can deny opens after a peer removes the claim from the directory. + listed: Final = set(_list_claim_names(lock_file, directory_ref, probe_clock=probe_clock)) + if still_listed := [name for name in denied if name in listed]: + error = pending[still_listed[0]] + reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, still_listed[0], reason) from error + return records + + +def _attempt_claim_read( + lock_file: str, directory_ref: tuple[str, int | None], name: str +) -> tuple[bytes | None, OSError | None]: + # Return the record, or (None, None) when the claim has already gone, or (None, error) for an open the caller may + # retry: a Windows EACCES while a peer's unlink holds the claim, or an NFS ESTALE from a peer unlinking its own + # claim out from under this client's cached filehandle. Any other OSError is a real fault and fails closed. + try: + return _read_record(directory_ref, name, _CLAIM_RECORD_LIMIT), None + except FileNotFoundError: + return None, None + except PermissionError as error: + if sys.platform == "win32" and _is_unlinked(str(Path(directory_ref[0], name))): # pragma: win32 cover + return None, None + return None, error + except OSError as error: + if error.errno == ESTALE: + return None, error + reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, name, reason) from error + + +def _read_record(directory_ref: tuple[str, int | None], name: str, limit: int) -> bytes: + fd, record = _open_record(directory_ref, name, limit, deletable=True) + os.close(fd) + return record + + +def _open_record( + directory_ref: tuple[str, int | None], name: str, limit: int, *, deletable: bool = False +) -> tuple[int, bytes]: + path = Path(directory_ref[0], name) + path_stat = _lstat_relative(directory_ref, name) + if not stat.S_ISREG(path_stat.st_mode): + msg = f"{path} is not a regular file" + raise OSError(msg) + if deletable and sys.platform == "win32": # pragma: win32 cover + # A claim's owner must be able to unlink it while another process reads it. The sentinel stays for the lock's + # lifetime, so it keeps os.open. + fd = _open_deletable_for_reading(str(path)) + else: + flags = os.O_RDONLY | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + fd = _open_relative(directory_ref, name, flags, 0) + try: + record = _read_opened_record(fd, path, path_stat, limit) + except BaseException as read_error: # preserve read and descriptor cleanup errors + try: + os.close(fd) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve read and descriptor cleanup errors + _raise_cleanup_errors("strict record read cleanup failed", read_error, close_error) + raise + return fd, record + + +def _lstat_relative(directory_ref: tuple[str, int | None], name: str) -> os.stat_result: + directory, directory_fd = directory_ref + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd + return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + return Path(directory, name).lstat() + + +def _read_opened_record(fd: int, path: Path, path_stat: os.stat_result, limit: int) -> bytes: + opened_stat = os.fstat(fd) + if not stat.S_ISREG(opened_stat.st_mode): + msg = f"{path} is not a regular file" + raise OSError(msg) + if _file_identity(opened_stat) != _file_identity(path_stat): + msg = f"{path} changed while opening" + raise OSError(msg) + record = os.read(fd, limit + 1) + if len(record) > limit: + msg = f"{path} exceeds {limit} bytes" + raise OSError(msg) + return record + + +def _parse_claim( + lock_file: str, + name: str, + name_parts: tuple[StrictSoftFileClaimState, str], + record: bytes, +) -> StrictSoftFileClaim: + try: + magic, token, pid_text, hostname_hex, start_text, trailing = record.decode("ascii").split("\n") + pid = int(pid_text) + start = int(start_text) if start_text else None + hostname = bytes.fromhex(hostname_hex).decode("utf-8") + except ValueError as error: + raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") from error + if not all(( + not trailing, + magic == _CLAIM_MAGIC, + token == name_parts[1], + 1 <= pid <= 2**31 - 1, + str(pid) == pid_text, + start is None or (start >= 0 and str(start) == start_text), + hostname.encode().hex() == hostname_hex + and hostname.isprintable() + and not any(character.isspace() for character in hostname), + )): + raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") + return StrictSoftFileClaim(name=name, state=name_parts[0], token=token, pid=pid, hostname=hostname, start=start) + + +@dataclass(frozen=True) +class StrictSoftFileClaim: + """ + One parsed strict soft-lock claim. + + :param name: the claim's file name in the claim directory + :param state: whether the claim is an ``intent`` or a ``held`` record + :param token: the random token that identifies the claim + :param pid: the PID of the process that wrote the claim + :param hostname: the host name of the process that wrote the claim + :param start: the owner's process start token, or ``None`` when the platform exposes no proven start time + """ + + name: str + state: StrictSoftFileClaimState + token: str + pid: int + hostname: str + #: The owner's process start token, or ``None`` when the platform exposes no proven start time. A strict lock never + #: reclaims a claim on its own, so this identifies the owner for tooling rather than driving any automatic break. + start: int | None = None + + +def _claim_exists_on_server(lock_file: str, directory_ref: tuple[str, int | None], name: str) -> bool: + # Reading an NFS claim can succeed from a client cache after its server-side name is gone. LINK checks the source + # on the server; a private proof is reclaimed if this process exits before unlinking it. + proof = f"{_CLAIM_PROBE_PREFIX}{secrets.token_hex(_TOKEN_HEX_LENGTH // 2)}{_PRIVATE_RECORD_SUFFIX}" + try: + _link_or_find_committed(directory_ref, name, proof) + except OSError as error: + if error.errno in {ENOENT, ESTALE}: + return False + reason = f"cannot verify claim: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, name, reason) from error + except NotImplementedError as error: + raise SoftFileLockProtocolError(lock_file, name, "cannot verify claim: hard links unavailable") from error + try: + _unlink_relative(directory_ref, proof) + except FileNotFoundError: + pass + except OSError as error: + reason = f"cannot remove claim probe: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, name, reason) from error + return True + + +def _link_or_find_committed( + directory_ref: tuple[str, int | None], + source_name: str, + destination_name: str, + source_identity: tuple[int, int] | None = None, +) -> None: + try: + _link_relative(directory_ref, source_name, destination_name) + except FileExistsError: + # NFS retransmits a LINK whose reply was lost and reports EEXIST for the committed link; as in open(2) NOTES + # and liblockfile, it succeeded when the destination names the source. Prefer the fstat-ed identity: a peer's + # scan reaps a private record once it has two links, so its name may already resolve to nothing. + if (identity := _relative_identity(directory_ref, destination_name)) is None or identity != ( + source_identity or _relative_identity(directory_ref, source_name) + ): + raise + + +def _link_relative(directory_ref: tuple[str, int | None], source_name: str, destination_name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + link_no_follow(source_name, destination_name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) + return + link_no_follow(Path(directory, source_name), Path(directory, destination_name)) # pragma: win32 cover + + +def link_no_follow( + source: str | Path, + destination: str | Path, + *, + src_dir_fd: int | None = None, + dst_dir_fd: int | None = None, +) -> None: + if not _HAS_LINK: + # No os.link at all (Termux/Android): report it as unsupported like a filesystem that refuses hard links, so + # the acquire path raises SoftFileLockProtocolError instead of a bare AttributeError. + msg = "os.link is unavailable on this platform" + raise NotImplementedError(msg) + # The source is a private record this process created with O_CREAT | O_EXCL, so follow_symlinks guards nothing an + # attacker can reach. Pass the option only when the runtime honors it: PyPy advertises it through + # os.supports_follow_symlinks yet its linkat rejects it with EINVAL, so probe once rather than trust the set. + if _LINK_HONORS_FOLLOW_SYMLINKS: + os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False) + else: # pragma: lacks link-follow-symlinks + os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + +def _relative_identity(directory_ref: tuple[str, int | None], name: str) -> tuple[int, int] | None: + directory, directory_fd = directory_ref + try: + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd + path_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + else: # pragma: win32 cover + path_stat = Path(directory, name).lstat() + except FileNotFoundError: + return None + return _file_identity(path_stat) + + +def _unlink_relative(directory_ref: tuple[str, int | None], name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + os.unlink(name, dir_fd=directory_fd) + elif sys.platform == "win32": # pragma: win32 cover + _unlink_in_directory(Path(directory), name) + else: + Path(directory, name).unlink() # pragma: win32 no cover + + +def _inspect_lock_path(path: Path) -> _LockPathInspection: + # "held" may be a soft-lock owner that still runs, "exited" is one this host proves gone, "unparsable" is content no + # lock parses, and "damaged" is a node no lock writes. + if not stat.S_ISREG(mode := path.lstat().st_mode): + return _LockPathInspection(None, "damaged", _NODE_DESCRIPTIONS.get(stat.S_IFMT(mode), "a special file")) + try: + fd, record = _open_record((os.fspath(path.parent), None), path.name, _CLAIM_RECORD_LIMIT) + except FileNotFoundError: + raise + except PermissionError as error: + # Windows denies the read while a legacy holder keeps its marker open, and a holder running as another user + # may leave it unreadable here, so a marker that cannot be read may still be held. + return _LockPathInspection(None, "held", f"a file it cannot read ({error.strerror or error})") + except OSError as error: + return _LockPathInspection(None, "damaged", f"a file it cannot read ({error.strerror or error})") + if record == _LEGACY_SENTINEL: + return _LockPathInspection(fd, "sentinel", "the strict sentinel") + os.close(fd) + return _LockPathInspection(None, *_describe_soft_marker(record.decode("utf-8", errors="replace"))) + + +class _LockPathInspection(NamedTuple): + fd: int | None + state: _OccupantState + occupant: str + + +def _describe_soft_marker(content: str) -> tuple[_OccupantState, str]: + # A SoftFileLock, MarkerSoftFileLock or SoftFileLease owner holds the path legitimately until it releases, so its + # marker is contention. One this host proves gone is named, never removed: strict never acts on an inferred death. + if (record := parse_marker(content)) is not None: + pid, hostname, start = record.pid, record.hostname, record.start + elif (holder := _parse_lock_holder(content)) is not None: + pid, hostname, start = holder + else: + return "unparsable", "a file that is neither the sentinel nor a soft-lock marker it can parse" + if owner_is_stale(pid, hostname, start): + return "exited", f"a soft-lock marker left by pid {pid} on {hostname}, which has exited" + return "held", f"a soft-lock marker held by pid {pid} on {hostname}" + + +def _ensure_protocol_directory(lock_file: str, directory: Path, mode: int, parent_fd: int | None = None) -> None: + try: + if parent_fd is None: + directory.mkdir(mode) + else: # pragma: needs dir-fd + os.mkdir(directory.name, mode, dir_fd=parent_fd) + except FileExistsError: + try: + found = _lstat_relative((os.fspath(directory.parent), parent_fd), directory.name) + owner = foreign_directory_owner( + found, directory.parent.stat() if parent_fd is None else os.fstat(parent_fd) + ) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot inspect {directory}") from error + if ( + not stat.S_ISDIR(found.st_mode) + or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ): + raise SoftFileLockProtocolError(lock_file, None, f"{directory} is not a real directory") from None + if owner is not None: # pragma: needs foreign-sticky-tmp + reason = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner" + raise SoftFileLockProtocolError(lock_file, None, reason) from None + _settle_directory_mode(lock_file, directory, mode, parent_fd) + + +def _settle_directory_mode(lock_file: str, directory: Path, mode: int, parent_fd: int | None) -> None: + # The umask or a default ACL can leave the directory wider than the mode, now or in an earlier release, so narrow a + # directory this account owns to the mode. The sticky bit (a directory flag that lets only a file's owner remove it) + # then protects each claim from other accounts even where the mode admits them. Windows has neither mode bits nor a + # sticky bit. + if sys.platform != "win32": # pragma: win32 no cover + try: + fd = os.open( + directory.name if parent_fd is not None else os.fspath(directory), + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_fd, + ) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory}") from error + try: + found = os.fstat(fd) + settled = found.st_mode & 0o7000 | found.st_mode & mode & 0o777 | stat.S_ISVTX + if found.st_uid == os.geteuid() and settled != stat.S_IMODE(found.st_mode): + os.fchmod(fd, settled) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot set the mode of {directory}") from error + finally: + os.close(fd) + + +def _ensure_claim_directory(lock_file: str, directory: Path, mode: int) -> None: + if not (_OPEN_SUPPORTS_DIR_FD and _MKDIR_SUPPORTS_DIR_FD): # pragma: lacks dir-fd + try: + pinned = pin_directories(directory.parent) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error + with pinned: + _ensure_protocol_directory(lock_file, directory, mode) + else: # pragma: needs dir-fd + # mkdir follows a symlink in every component but the last, so create against a descriptor of the real parent. + try: + coordination_fd = _open_directory(os.fspath(directory.parent)) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error + try: + _ensure_protocol_directory(lock_file, directory, mode, coordination_fd) + finally: + with contextlib.suppress(OSError): + os.close(coordination_fd) + + +def pin_directories(*directories: Path) -> contextlib.ExitStack[bool | None]: + # Windows has no dir_fd, so each operation resolves every path component by name again. An open handle on a + # directory blocks a rename of it, and with it a swap for a junction (a directory that points at another directory), + # until the operation ends. Other platforms work through descriptors and hold nothing. + stack: contextlib.ExitStack[bool | None] = contextlib.ExitStack() + if sys.platform == "win32": # pragma: win32 cover + with stack: + for directory in directories: + stack.callback(_unpin_directory, _pin_directory(str(directory))) + return stack.pop_all() + return stack # pragma: win32 no cover + + +def _open_directory(directory: str, dir_fd: int | None = None) -> int: # pragma: needs dir-fd + return os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) + + +def _open_or_create_sentinel( + lock_file: str, path: Path, mode: int, unchanged_for: Callable[[os.stat_result], float] +) -> int | None: + try: + return _open_sentinel(lock_file, path, unchanged_for) + except FileNotFoundError: + pass + + _reclaim_sentinel_private_records(lock_file, path) + try: + publication_cleanup_error = _publish_record(os.fspath(path), _LEGACY_SENTINEL, mode) + except _PrivateRecordReclaimedError: + return None + except (NotImplementedError, OSError) as error: + raise_if_hard_links_unsupported(lock_file, error) + if not isinstance(error, OSError) or error.errno != EEXIST: + raise + else: + if publication_cleanup_error is not None: # pragma: needs dir-fd + raise publication_cleanup_error + try: + return _open_sentinel(lock_file, path, unchanged_for) + except FileNotFoundError: + return None + + +def _open_sentinel(lock_file: str, path: Path, unchanged_for: Callable[[os.stat_result], float]) -> int | None: + # A live soft-lock owner blocks like any holder. Anything else raises: claims cannot show it and force_break() + # cannot remove it, so waiting would wedge every strict contender with no diagnostic. A file no lock parses may be a + # soft lock between its create and its write, so it raises only once unchanged as long as a soft contender waits. + deadline: float | None = None + while True: + fd, state, occupant = _inspect_lock_path(path) + if fd is not None or state == "held": + return fd + reason = f"expected the strict sentinel at the lock path, found {occupant}" + if state == "unparsable": + if unchanged_for(path.lstat()) < MALFORMED_LOCK_AGE_THRESHOLD: + return None + raise SoftFileLockProtocolError( + lock_file, None, f"{reason}, unchanged for {MALFORMED_LOCK_AGE_THRESHOLD:g} seconds" + ) + if deadline is None: + deadline = time.monotonic() + _SENTINEL_READ_GRACE + if state == "exited" or time.monotonic() >= deadline: + raise SoftFileLockProtocolError(lock_file, None, reason) + time.sleep(_CLAIM_READ_RETRY) -def _parse_claim( - lock_file: str, - name: str, - name_parts: tuple[StrictSoftFileClaimState, str], - record: bytes, -) -> StrictSoftFileClaim: +def _reclaim_sentinel_private_records(lock_file: str, path: Path) -> None: + directory_ref = os.fspath(path.parent), None + now = functools.cache(functools.partial(_filesystem_now, directory_ref, path.name)) try: - magic, token, pid_text, hostname_hex, start_text, trailing = record.decode("ascii").split("\n") - pid = int(pid_text) - start = int(start_text) if start_text else None - hostname = bytes.fromhex(hostname_hex).decode("utf-8") - except (UnicodeDecodeError, ValueError) as error: - raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") from error - if not all(( - not trailing, - magic == _CLAIM_MAGIC, - token == name_parts[1], - 1 <= pid <= 2**31 - 1, - str(pid) == pid_text, - start is None or (start >= 0 and str(start) == start_text), - hostname.encode().hex() == hostname_hex - and hostname.isprintable() - and not any(character.isspace() for character in hostname), - )): - raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") - return StrictSoftFileClaim(name=name, state=name_parts[0], token=token, pid=pid, hostname=hostname, start=start) + with os.scandir(path.parent) as entries: + for entry in entries: + if _private_public_name(entry.name) == path.name: + _reclaim_private_record(directory_ref, entry.name, now) + except OSError as error: + reason = f"cannot reclaim sentinel private records: {error.strerror or error}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error -def _parse_claim_name(name: str) -> tuple[StrictSoftFileClaimState, str] | None: - if not name.endswith(".claim"): - return None - parts = name.removesuffix(".claim").split("-") - claim_states: Final = frozenset({"intent", "held"}) - claim_name_part_count: Final = 3 - if len(parts) != claim_name_part_count or parts[0] not in claim_states or parts[1] != "v1": - return None - token = parts[2] - if len(token) != _TOKEN_HEX_LENGTH or any(character not in "0123456789abcdef" for character in token): - return None - return cast("StrictSoftFileClaimState", parts[0]), token +def _file_identity(st: os.stat_result) -> tuple[int, int]: + return st.st_dev, st.st_ino def _claim_name(state: StrictSoftFileClaimState, token: str) -> str: return f"{state}-v1-{token}.claim" -def _claim_token_key(name: str) -> str: - return name.removeprefix("held-").removeprefix("intent-") - - -def _claim_record(token: str) -> bytes: - hostname_hex = host_name().encode().hex() - start = process_start_token(os.getpid()) - start_text = "" if start is None else str(start) - return f"{_CLAIM_MAGIC}\n{token}\n{os.getpid()}\n{hostname_hex}\n{start_text}\n".encode("ascii") +def _pin_claim_directory(directory: Path) -> contextlib.ExitStack[bool | None]: + return pin_directories(directory.parent, directory) def _publish_record( @@ -820,6 +1133,24 @@ def _publish_record_in_directory( _raise_record_finalization_errors(close_error, unlink_error) +def _link_private_record( + directory_ref: tuple[str, int | None], + names: tuple[str, str], + private_identity: tuple[int, int], +) -> None: + try: + _link_or_find_committed(directory_ref, *names, private_identity) + except FileNotFoundError as error: + if _relative_identity(directory_ref, names[0]) is not None: + raise + msg = "private publication record was reclaimed" + raise _PrivateRecordReclaimedError(msg) from error + if _relative_identity(directory_ref, names[1]) == private_identity: + return + msg_0 = "private publication record was replaced" + raise _PrivateRecordReclaimedError(msg_0) + + def _close_and_unlink_private_record( directory_ref: tuple[str, int | None], private_name: str, @@ -845,24 +1176,6 @@ def _close_and_unlink_private_record( return close_error, unlink_error -def _link_private_record( - directory_ref: tuple[str, int | None], - names: tuple[str, str], - private_identity: tuple[int, int], -) -> None: - try: - _link_or_find_committed(directory_ref, *names, private_identity) - except FileNotFoundError as error: - if _relative_identity(directory_ref, names[0]) is not None: - raise - msg = "private publication record was reclaimed" - raise _PrivateRecordReclaimedError(msg) from error - if _relative_identity(directory_ref, names[1]) == private_identity: - return - msg_0 = "private publication record was replaced" - raise _PrivateRecordReclaimedError(msg_0) - - def _raise_record_finalization_errors( close_error: BaseException | None, unlink_error: BaseException | None, @@ -873,178 +1186,41 @@ def _raise_record_finalization_errors( raise errors[0] -def _private_record_name(public_name: str) -> str: - return f".{public_name}{_PRIVATE_RECORD_MARKER}{secrets.token_hex(_PRIVATE_RECORD_RANDOM_HEX_LENGTH // 2)}.tmp" - - -def _private_public_name(private_name: str) -> str | None: - if not private_name.startswith(".") or not private_name.endswith(_PRIVATE_RECORD_SUFFIX): - return None - public_name, marker, random_hex = private_name[1 : -len(_PRIVATE_RECORD_SUFFIX)].rpartition(_PRIVATE_RECORD_MARKER) - if ( - marker != _PRIVATE_RECORD_MARKER - or len(random_hex) != _PRIVATE_RECORD_RANDOM_HEX_LENGTH - or any(character not in "0123456789abcdef" for character in random_hex) - ): - return None - return public_name - - -def _is_claim_probe_name(name: str) -> bool: - if not name.startswith(_CLAIM_PROBE_PREFIX) or not name.endswith(_PRIVATE_RECORD_SUFFIX): - return False - token = name[len(_CLAIM_PROBE_PREFIX) : -len(_PRIVATE_RECORD_SUFFIX)] - return len(token) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in token) - - -def _reclaim_sentinel_private_records(lock_file: str, path: Path) -> None: - directory_ref = os.fspath(path.parent), None - now = functools.cache(functools.partial(_filesystem_now, directory_ref, path.name)) - try: - with os.scandir(path.parent) as entries: - for entry in entries: - if _private_public_name(entry.name) == path.name: - _reclaim_private_record(directory_ref, entry.name, now) - except OSError as error: - reason = f"cannot reclaim sentinel private records: {error.strerror or error}" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - - -def _reclaim_private_record( - directory_ref: tuple[str, int | None], private_name: str, now: Callable[[], float | None] -) -> None: - directory, directory_fd = directory_ref - try: - private_stat = ( - os.stat(private_name, dir_fd=directory_fd, follow_symlinks=False) - if directory_fd is not None and _STAT_SUPPORTS_DIR_FD - else Path(directory, private_name).lstat() - ) - except FileNotFoundError: - return - if not stat.S_ISREG(private_stat.st_mode): - msg = f"{Path(directory, private_name)} is not a regular private record" - raise OSError(msg) - private_record_grace: Final = 2.0 - if private_stat.st_nlink == 1 and ( - (current := now()) is None or current - private_stat.st_mtime < private_record_grace - ): - return - try: - _unlink_private_record_once(directory_ref, private_name) - except FileNotFoundError: - pass - except OSError as error: - if error.errno not in {EACCES, EPERM}: - raise - - -def _filesystem_now(directory_ref: tuple[str, int | None], public_name: str) -> float | None: - # A network filesystem stamps mtimes with the server's clock, which can run seconds off this client's, so read "now" - # as the mtime of a probe written like a record (liblockfile uses a read's atime). The probe has a private-record - # name, so a scan reaps one left behind. Without a stamp no age is trusted. - probe = _private_record_name(public_name) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) - try: - fd = _open_relative(directory_ref, probe, flags, 0o600) - except OSError: - return None - try: - write_all(fd, b"\n") - return os.fstat(fd).st_mtime - except OSError: - return None - finally: - with contextlib.suppress(OSError): - os.close(fd) - with contextlib.suppress(OSError): - _unlink_private_record_once(directory_ref, probe) - - -def _unlink_private_record_once(directory_ref: tuple[str, int | None], private_name: str) -> None: - directory, directory_fd = directory_ref - if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - os.unlink(private_name, dir_fd=directory_fd) - else: - Path(directory, private_name).unlink() - - -def _relative_identity(directory_ref: tuple[str, int | None], name: str) -> tuple[int, int] | None: - directory, directory_fd = directory_ref - try: - if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd - path_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) - else: # pragma: win32 cover - path_stat = Path(directory, name).lstat() - except FileNotFoundError: - return None - return _file_identity(path_stat) - - -def _open_relative(directory_ref: tuple[str, int | None], name: str, flags: int, mode: int) -> int: - directory, directory_fd = directory_ref - return ( - os.open(name, flags, mode, dir_fd=directory_fd) - if directory_fd is not None - else os.open(Path(directory, name), flags, mode) - ) - - -def _link_relative(directory_ref: tuple[str, int | None], source_name: str, destination_name: str) -> None: - directory, directory_fd = directory_ref - if directory_fd is not None and _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - link_no_follow(source_name, destination_name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) - return - link_no_follow(Path(directory, source_name), Path(directory, destination_name)) # pragma: win32 cover +def _claim_record(token: str) -> bytes: + hostname_hex = host_name().encode().hex() + start = process_start_token(os.getpid()) + start_text = "" if start is None else str(start) + return f"{_CLAIM_MAGIC}\n{token}\n{os.getpid()}\n{hostname_hex}\n{start_text}\n".encode("ascii") -def _link_or_find_committed( - directory_ref: tuple[str, int | None], - source_name: str, - destination_name: str, - source_identity: tuple[int, int] | None = None, -) -> None: - try: - _link_relative(directory_ref, source_name, destination_name) - except FileExistsError: - # NFS retransmits a LINK whose reply was lost and reports EEXIST for the committed link; as in open(2) NOTES - # and liblockfile, it succeeded when the destination names the source. Prefer the fstat-ed identity: a peer's - # scan reaps a private record once it has two links, so its name may already resolve to nothing. - if (identity := _relative_identity(directory_ref, destination_name)) is None or identity != ( - source_identity or _relative_identity(directory_ref, source_name) - ): - raise +def _open_claim_directory(directory: str) -> int: # pragma: needs dir-fd + # O_NOFOLLOW applies to the last path component. A process that can rename entries in the lock's directory can + # swap the coordination directory for a symlink after any check of its name, so open the claim directory relative + # to a descriptor of the coordination directory: a symlink at either level then fails the open. + coordination_directory, name = os.path.split(directory) + coordination_fd = _open_directory(coordination_directory) + try: + return _open_directory(name, coordination_fd) + finally: + # The walk wrote nothing through it, and an error here would leak the descriptor this function returns. + with contextlib.suppress(OSError): + os.close(coordination_fd) -def link_no_follow( - source: str | Path, - destination: str | Path, - *, - src_dir_fd: int | None = None, - dst_dir_fd: int | None = None, -) -> None: - if not _HAS_LINK: - # No os.link at all (Termux/Android): report it as unsupported like a filesystem that refuses hard links, so - # the acquire path raises SoftFileLockProtocolError instead of a bare AttributeError. - msg = "os.link is unavailable on this platform" - raise NotImplementedError(msg) - # The source is a private record this process created with O_CREAT | O_EXCL, so follow_symlinks guards nothing an - # attacker can reach. Pass the option only when the runtime honors it: PyPy advertises it through - # os.supports_follow_symlinks yet its linkat rejects it with EINVAL, so probe once rather than trust the set. - if _LINK_HONORS_FOLLOW_SYMLINKS: - os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False) - else: # pragma: lacks link-follow-symlinks - os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) +class _PrivateRecordReclaimedError(Exception): + pass -def _unlink_relative(directory_ref: tuple[str, int | None], name: str) -> None: - directory, directory_fd = directory_ref - if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - os.unlink(name, dir_fd=directory_fd) - elif sys.platform == "win32": # pragma: win32 cover - _unlink_in_directory(Path(directory), name) - else: - Path(directory, name).unlink() # pragma: win32 no cover +def raise_if_hard_links_unsupported(lock_file: str, error: NotImplementedError | OSError) -> None: + windows_hard_link_unsupported: Final = frozenset({1, 17, 50}) + unsupported = ( + isinstance(error, NotImplementedError) + or error.errno in _HARD_LINK_UNSUPPORTED_ERRNOS + or (getattr(error, "winerror", None) in windows_hard_link_unsupported) + ) + if unsupported: + reason = "filesystem does not support atomic no-replace hard-link publication" + raise SoftFileLockProtocolError(lock_file, None, reason) from error def _link_no_replace(directory: Path, source_name: str, destination_name: str) -> BaseException | None: @@ -1068,6 +1244,16 @@ def _link_no_replace(directory: Path, source_name: str, destination_name: str) - return None # pragma: win32 cover +def _claim_token_key(name: str) -> str: + return name.removeprefix("held-").removeprefix("intent-") + + +def _raise_recorded_errors(message: str, errors: list[BaseException]) -> None: + if len(errors) > 1: + _raise_cleanup_errors(message, errors[0], *errors[1:]) + raise errors[0] + + def _unlink_owner_path_result(path: str) -> tuple[bool, BaseException | None]: directory, name = os.path.split(path) try: @@ -1080,10 +1266,13 @@ def _unlink_owner_path_result(path: str) -> tuple[bool, BaseException | None]: return True, cleanup_error -def _raise_recorded_errors(message: str, errors: list[BaseException]) -> None: - if len(errors) > 1: - _raise_cleanup_errors(message, errors[0], *errors[1:]) - raise errors[0] +def _scandir_relative( + directory_ref: tuple[str, int | None], +) -> contextlib.AbstractContextManager[Iterator[os.DirEntry[str]]]: + directory, directory_fd = directory_ref + if directory_fd is not None and _SCANDIR_SUPPORTS_FD: # pragma: needs dir-fd + return os.scandir(directory_fd) + return os.scandir(directory) # pragma: lacks dir-fd def _unlink_in_directory(directory: Path, name: str) -> BaseException | None: @@ -1125,191 +1314,6 @@ def _unlink_error(path: Path) -> OSError | None: # pragma: win32 cover return None -def _pin_claim_directory(directory: Path) -> contextlib.ExitStack[bool | None]: - return pin_directories(directory.parent, directory) - - -def pin_directories(*directories: Path) -> contextlib.ExitStack[bool | None]: - # Windows has no dir_fd, so each operation resolves every path component by name again. An open handle on a - # directory blocks a rename of it, and with it a swap for a junction (a directory that points at another directory), - # until the operation ends. Other platforms work through descriptors and hold nothing. - stack: contextlib.ExitStack[bool | None] = contextlib.ExitStack() - if sys.platform == "win32": # pragma: win32 cover - with stack: - for directory in directories: - stack.callback(_unpin_directory, _pin_directory(str(directory))) - return stack.pop_all() - return stack # pragma: win32 no cover - - -def _open_directory(directory: str, dir_fd: int | None = None) -> int: # pragma: needs dir-fd - return os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) - - -def _open_claim_directory(directory: str) -> int: # pragma: needs dir-fd - # O_NOFOLLOW applies to the last path component. A process that can rename entries in the lock's directory can - # swap the coordination directory for a symlink after any check of its name, so open the claim directory relative - # to a descriptor of the coordination directory: a symlink at either level then fails the open. - coordination_directory, name = os.path.split(directory) - coordination_fd = _open_directory(coordination_directory) - try: - return _open_directory(name, coordination_fd) - finally: - # The walk wrote nothing through it, and an error here would leak the descriptor this function returns. - with contextlib.suppress(OSError): - os.close(coordination_fd) - - -def _scandir_relative( - directory_ref: tuple[str, int | None], -) -> contextlib.AbstractContextManager[Iterator[os.DirEntry[str]]]: - directory, directory_fd = directory_ref - if directory_fd is not None and _SCANDIR_SUPPORTS_FD: # pragma: needs dir-fd - return os.scandir(directory_fd) - return os.scandir(directory) # pragma: lacks dir-fd - - -def _lstat_relative(directory_ref: tuple[str, int | None], name: str) -> os.stat_result: - directory, directory_fd = directory_ref - if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd - return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) - return Path(directory, name).lstat() - - -def _read_record(directory_ref: tuple[str, int | None], name: str, limit: int) -> bytes: - fd, record = _open_record(directory_ref, name, limit, deletable=True) - os.close(fd) - return record - - -def _open_record( - directory_ref: tuple[str, int | None], name: str, limit: int, *, deletable: bool = False -) -> tuple[int, bytes]: - path = Path(directory_ref[0], name) - path_stat = _lstat_relative(directory_ref, name) - if not stat.S_ISREG(path_stat.st_mode): - msg = f"{path} is not a regular file" - raise OSError(msg) - if deletable and sys.platform == "win32": # pragma: win32 cover - # A claim's owner must be able to unlink it while another process reads it. The sentinel stays for the lock's - # lifetime, so it keeps os.open. - fd = _open_deletable_for_reading(str(path)) - else: - flags = os.O_RDONLY | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) - fd = _open_relative(directory_ref, name, flags, 0) - try: - record = _read_opened_record(fd, path, path_stat, limit) - except BaseException as read_error: # preserve read and descriptor cleanup errors - try: - os.close(fd) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve read and descriptor cleanup errors - _raise_cleanup_errors("strict record read cleanup failed", read_error, close_error) - raise - return fd, record - - -def _read_opened_record(fd: int, path: Path, path_stat: os.stat_result, limit: int) -> bytes: - opened_stat = os.fstat(fd) - if not stat.S_ISREG(opened_stat.st_mode): - msg = f"{path} is not a regular file" - raise OSError(msg) - if _file_identity(opened_stat) != _file_identity(path_stat): - msg = f"{path} changed while opening" - raise OSError(msg) - record = os.read(fd, limit + 1) - if len(record) > limit: - msg = f"{path} exceeds {limit} bytes" - raise OSError(msg) - return record - - -def _ensure_claim_directory(lock_file: str, directory: Path, mode: int) -> None: - if not (_OPEN_SUPPORTS_DIR_FD and _MKDIR_SUPPORTS_DIR_FD): # pragma: lacks dir-fd - try: - pinned = pin_directories(directory.parent) - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error - with pinned: - _ensure_protocol_directory(lock_file, directory, mode) - else: # pragma: needs dir-fd - # mkdir follows a symlink in every component but the last, so create against a descriptor of the real parent. - try: - coordination_fd = _open_directory(os.fspath(directory.parent)) - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error - try: - _ensure_protocol_directory(lock_file, directory, mode, coordination_fd) - finally: - with contextlib.suppress(OSError): - os.close(coordination_fd) - - -def _ensure_protocol_directory(lock_file: str, directory: Path, mode: int, parent_fd: int | None = None) -> None: - try: - if parent_fd is None: - directory.mkdir(mode) - else: # pragma: needs dir-fd - os.mkdir(directory.name, mode, dir_fd=parent_fd) - except FileExistsError: - try: - found = _lstat_relative((os.fspath(directory.parent), parent_fd), directory.name) - owner = foreign_directory_owner( - found, directory.parent.stat() if parent_fd is None else os.fstat(parent_fd) - ) - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot inspect {directory}") from error - if ( - not stat.S_ISDIR(found.st_mode) - or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT - ): - raise SoftFileLockProtocolError(lock_file, None, f"{directory} is not a real directory") from None - if owner is not None: # pragma: needs foreign-sticky-tmp - reason = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner" - raise SoftFileLockProtocolError(lock_file, None, reason) from None - _settle_directory_mode(lock_file, directory, mode, parent_fd) - - -def _settle_directory_mode(lock_file: str, directory: Path, mode: int, parent_fd: int | None) -> None: - # The umask or a default ACL can leave the directory wider than the mode, now or in an earlier release, so narrow a - # directory this account owns to the mode. The sticky bit (a directory flag that lets only a file's owner remove it) - # then protects each claim from other accounts even where the mode admits them. Windows has neither mode bits nor a - # sticky bit. - if sys.platform != "win32": # pragma: win32 no cover - try: - fd = os.open( - directory.name if parent_fd is not None else os.fspath(directory), - os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), - dir_fd=parent_fd, - ) - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory}") from error - try: - found = os.fstat(fd) - settled = found.st_mode & 0o7000 | found.st_mode & mode & 0o777 | stat.S_ISVTX - if found.st_uid == os.geteuid() and settled != stat.S_IMODE(found.st_mode): - os.fchmod(fd, settled) - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot set the mode of {directory}") from error - finally: - os.close(fd) - - -def raise_if_hard_links_unsupported(lock_file: str, error: NotImplementedError | OSError) -> None: - windows_hard_link_unsupported: Final = frozenset({1, 17, 50}) - unsupported = ( - isinstance(error, NotImplementedError) - or error.errno in _HARD_LINK_UNSUPPORTED_ERRNOS - or (getattr(error, "winerror", None) in windows_hard_link_unsupported) - ) - if unsupported: - reason = "filesystem does not support atomic no-replace hard-link publication" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - - -def _file_identity(st: os.stat_result) -> tuple[int, int]: - return st.st_dev, st.st_ino - - __all__ = [ "StrictSoftFileClaim", "StrictSoftFileClaimState", diff --git a/src/filelock/_unix.py b/src/filelock/_unix.py index c75e7333..f082921f 100644 --- a/src/filelock/_unix.py +++ b/src/filelock/_unix.py @@ -188,7 +188,11 @@ def _switch_to_soft_lock(self, fd: int, missing_flock: OSError, *, created: bool # The filesystem does not implement flock. Fail closed when the caller opted out of existence-lock # semantics (#603), asked to preserve the pathname (#605), or set an on_acquired hook (#607), none of which # a soft lock can honor. - falls_back: Final = self._fallback_to_soft and not self._preserve_lock_file and self._on_acquired is None + falls_back: Final = ( + self._settings.fallback_to_soft + and not self._settings.preserve_lock_file + and self._settings.on_acquired is None + ) try: # A file this attempt did not create may be a peer's live marker, so it stays. Our own placeholder goes # before the close: the open descriptor keeps its inode number from a peer's marker, and an exchange diff --git a/src/filelock/_windows.py b/src/filelock/_windows.py index 51ab1355..5d7dd756 100644 --- a/src/filelock/_windows.py +++ b/src/filelock/_windows.py @@ -315,7 +315,8 @@ def _release(self) -> None: unlock_fd(fd) self._mark_descriptor_released() self._close_released_fd(fd, default_suppresses=False) - if not self._preserve_lock_file: # preserve_lock_file keeps a stable file identity for the caller (#605) + # preserve_lock_file keeps a stable file identity for the caller (#605) + if not self._settings.preserve_lock_file: with suppress(OSError): unlink_without_oplock_wait(self._lock_path) diff --git a/src/filelock/asyncio.py b/src/filelock/asyncio.py index 62cfb957..3591f0b3 100644 --- a/src/filelock/asyncio.py +++ b/src/filelock/asyncio.py @@ -608,7 +608,7 @@ async def _rollback_failed_acquire_async(self, acquisition_error: BaseException) ) def _raise_acquire_rollback_errors(self, acquire_error: BaseException, rollback_error: BaseException) -> NoReturn: - if self._context_error_policy == "group": + if self._settings.context_error_policy == "group": _raise_grouped_errors("lock acquisition backend and rollback both failed", acquire_error, rollback_error) _raise_chained_errors(acquire_error, rollback_error) @@ -629,10 +629,10 @@ async def _rollback_failed_registration_async(self, registration_error: BaseExce _raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error) async def _invoke_on_acquired_async(self) -> None: - if self._on_acquired is None or self._context.lock_counter != 1: + if self._settings.on_acquired is None or self._context.lock_counter != 1: return try: - self._on_acquired(cast("int", self._context.lock_file_fd)) + self._settings.on_acquired(cast("int", self._context.lock_file_fd)) except BaseException as callback_error: # caller control-flow errors must release the lock callback_context = callback_error.__context__ try: @@ -668,7 +668,7 @@ def _raise_cancelled_errors( first_error: BaseException, second_error: BaseException | None = None, ) -> NoReturn: - if self._context_error_policy == "group": + if self._settings.context_error_policy == "group": marker = ( (_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER) if message == _ASYNC_RELEASE_CANCELLATION_ERRORS @@ -717,7 +717,7 @@ async def _release_in_context( # ty: ignore[incompatible-method-override] # th except BaseException as release_error: if body_error is None: raise - if self._context_error_policy == "chain": + if self._settings.context_error_policy == "chain": _append_exception_context(release_error, body_error) raise if ( @@ -839,7 +839,7 @@ def _release_abandoned_holds(self) -> None: with contextlib.suppress(Exception): # A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches # threads whose loops it cannot schedule on. - if self._is_thread_local and not iscoroutinefunction(self._release): + if self._settings.thread_local and not iscoroutinefunction(self._release): self._release_without_loop() return try: @@ -886,6 +886,28 @@ def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future self._release_without_loop() +class AsyncAcquireReturnProxy: + """ + A context-aware object that will release the lock file when exiting. + + :param lock: the lock to release on exit + """ + + def __init__(self, lock: BaseAsyncFileLock) -> None: # trivial release-on-exit proxy + self.lock = lock + + async def __aenter__(self) -> BaseAsyncFileLock: # returns the wrapped lock + return self.lock + + async def __aexit__( # releases the wrapped lock on exit + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context + + def _new_transition_gate() -> _AsyncTransitionGate: # Called per context rather than passed as the factory, so the lazy import resolves on first use, not at import. return _AsyncTransitionGate() @@ -949,28 +971,6 @@ def __init__( # ruff:ignore[too-many-arguments] # mirrors the AsyncFileLockCon setattr(AsyncThreadLocalFileContext, _field.name, _delegate_to_current(_field.name)) -class AsyncAcquireReturnProxy: - """ - A context-aware object that will release the lock file when exiting. - - :param lock: the lock to release on exit - """ - - def __init__(self, lock: BaseAsyncFileLock) -> None: # trivial release-on-exit proxy - self.lock = lock - - async def __aenter__(self) -> BaseAsyncFileLock: # returns the wrapped lock - return self.lock - - async def __aexit__( # releases the wrapped lock on exit - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context - - class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock): """Simply watches the existence of the lock file.""" diff --git a/tasks/coverage_pragmas.py b/tasks/coverage_pragmas.py index 9ef11617..fefcc755 100644 --- a/tasks/coverage_pragmas.py +++ b/tasks/coverage_pragmas.py @@ -14,6 +14,7 @@ from __future__ import annotations +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast from capabilities import CAPABILITIES @@ -58,7 +59,7 @@ def _extend(config: TConfigurable, option: str, patterns: list[str]) -> None: #: Modules a missing capability makes unrunnable in full; marking every line would restate one module-level gate. -_CAPABILITY_MODULES: Final[dict[str, tuple[str, ...]]] = { +_CAPABILITY_MODULES: Final[MappingProxyType[str, tuple[str, ...]]] = MappingProxyType({ "posix-hard-link": ( "*/tests/test_read_write_private_link.py", "*\\tests\\test_read_write_private_link.py", @@ -73,7 +74,7 @@ def _extend(config: TConfigurable, option: str, patterns: list[str]) -> None: "*/filelock/_soft_rw/*.py", "*\\filelock\\_soft_rw\\*.py", ), -} +}) #: A forked child exits through os._exit without writing coverage data, so no job in the matrix can see these. _ALWAYS_EXCLUDED: Final[tuple[str, ...]] = (r"# pragma: forked child\b",) diff --git a/tests/conftest.py b/tests/conftest.py index 1897660e..37d60d34 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,16 @@ _LockT = TypeVar("_LockT", bound=BaseFileLock) +@contextmanager +def held_after_reclaim(lock: _LockT) -> Generator[_LockT]: + # Hold *lock* through :func:`acquire_after_reclaim` and release it on exit. + acquire_after_reclaim(lock) + try: + yield lock + finally: + lock.release() + + def acquire_after_reclaim(lock: BaseFileLock) -> None: # Take a path whose stale marker the lock has to break first, without a wall-clock budget. # @@ -46,16 +56,6 @@ def acquire_after_reclaim(lock: BaseFileLock) -> None: return -@contextmanager -def held_after_reclaim(lock: _LockT) -> Generator[_LockT]: - # Hold *lock* through :func:`acquire_after_reclaim` and release it on exit. - acquire_after_reclaim(lock) - try: - yield lock - finally: - lock.release() - - def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: # StrictSoftFileLock publishes claims with hard links, so tests marked requires_hard_links cannot run where # os.link is missing (Termux/Android CPython ships without it). The no-hard-link degradation itself is covered by diff --git a/tests/read_write_helpers.py b/tests/read_write_helpers.py index 281d0746..1da93f4c 100644 --- a/tests/read_write_helpers.py +++ b/tests/read_write_helpers.py @@ -8,6 +8,7 @@ from filelock import ReadWriteLock, Timeout if TYPE_CHECKING: + import os import sys from collections.abc import Callable @@ -86,6 +87,17 @@ def _probe_read_write_lock(lock_file: str, mode: Literal["read", "write"], sendi sending.send_bytes(bytes([acquired])) +def writer_gets_in(lock_file: str | os.PathLike[str]) -> bool: + contender: Final = ReadWriteLock(lock_file, is_singleton=False) + try: + contender.acquire_write(blocking=False) + except Timeout: + return False + finally: + contender.close() + return True + + _T = TypeVar("_T") @@ -116,4 +128,5 @@ class Interrupt(BaseException): "Interrupt", "assert_read_write_lock_state", "interrupt_once", + "writer_gets_in", ] diff --git a/tests/soft_rw/test_soft_rw_async.py b/tests/soft_rw/test_soft_rw_async.py index b89af8a1..f8a94c09 100644 --- a/tests/soft_rw/test_soft_rw_async.py +++ b/tests/soft_rw/test_soft_rw_async.py @@ -48,7 +48,8 @@ async def test_async_lock_context_manager(tmp_path: Path, mode: Literal["read", lock = _make(tmp_path) try: async with getattr(lock, f"{mode}_lock")(timeout=2): - pass + assert lock.generation is not None + assert lock.generation is None finally: await lock.close() @@ -231,17 +232,36 @@ def park() -> None: finally: gate.set() await parked - async with lock.write_lock(timeout=2): - pass + assert lock.generation is None + assert await asyncio.to_thread(_writer_gets_in, tmp_path) await lock.close() +def _writer_gets_in(tmp_path: Path) -> bool: + contender: Final = _make_sync(tmp_path) + try: + contender.acquire_write(blocking=False) + except Timeout: + return False + finally: + contender.close() + return True + + +def _make_sync(tmp_path: Path) -> SoftReadWriteLock: + return SoftReadWriteLock( + str(tmp_path / "foo.lock"), is_singleton=False, heartbeat_interval=0.1, stale_threshold=0.5 + ) + + @pytest.mark.asyncio async def test_async_release_force(tmp_path: Path) -> None: lock = _make(tmp_path) try: await lock.acquire_read(timeout=2) await lock.release(force=True) + assert lock.generation is None + assert await asyncio.to_thread(_writer_gets_in, tmp_path) finally: await lock.close() @@ -256,6 +276,7 @@ async def test_async_release_and_close_skip_when_pid_differs(tmp_path: Path, moc await lock.release(force=True) await lock.close() mocker.stopall() + assert not await asyncio.to_thread(_writer_gets_in, tmp_path) await lock.release() finally: mocker.stopall() @@ -603,7 +624,7 @@ async def test_async_soft_close_after_a_sync_end_of_the_shared_singleton_returns await lock.close() - assert _members(tmp_path, "foo.lock") == [] + assert _members(tmp_path) == [] @pytest.mark.asyncio @@ -635,23 +656,6 @@ async def _read_and_probe(lock: AsyncSoftReadWriteLock) -> bool: return await asyncio.to_thread(_writer_gets_in, Path(lock.lock_file).parent) -def _writer_gets_in(tmp_path: Path) -> bool: - contender: Final = _make_sync(tmp_path) - try: - contender.acquire_write(blocking=False) - except Timeout: - return False - finally: - contender.close() - return True - - -def _make_sync(tmp_path: Path) -> SoftReadWriteLock: - return SoftReadWriteLock( - str(tmp_path / "foo.lock"), is_singleton=False, heartbeat_interval=0.1, stale_threshold=0.5 - ) - - @pytest.mark.asyncio async def test_async_soft_read_through_another_wrapper_during_close_holds_the_lock(tmp_path: Path) -> None: closing, other = _make(tmp_path, is_singleton=True), _make(tmp_path, is_singleton=True) diff --git a/tests/soft_rw/test_soft_rw_protocol.py b/tests/soft_rw/test_soft_rw_protocol.py index 39499df3..7122ad84 100644 --- a/tests/soft_rw/test_soft_rw_protocol.py +++ b/tests/soft_rw/test_soft_rw_protocol.py @@ -224,6 +224,15 @@ def test_rescan_skips_a_generation_removed_after_listing( assert fresh.latest().generation == 1 +def _committed(files: OsFiles, lock_file: str, root: str, writers: list[str | None]) -> GenerationLog: + # One log that read the head before each commit, as a participant does, and so remembers the last one. + files.prepare(root) + log = GenerationLog(files, lock_file, root) + for writer in writers: + assert log.commit(Snapshot(generation=log.latest().generation + 1, writer=writer, readers=frozenset())) + return log + + def test_malformed_generation_raises(lock_file: str, files: OsFiles, root: str) -> None: files.prepare(root) Path(root, "gen", f"{1:020d}").write_bytes(b"filelock-rw/1\ngeneration=5\n") @@ -274,7 +283,7 @@ def _participant( def test_leave_retries_a_commit_a_peer_won(lock_file: str, files: OsFiles, root: str, mocker: MockerFixture) -> None: # A peer can publish the next generation between leave() reading the latest one and linking its successor; the # holder then re-reads and commits itself out of whatever the peer published. - reader = _participant(files, lock_file, "read", stale_threshold=100) + reader = _participant(files, lock_file, "read") reader.publish() assert reader.advance() real_commit = GenerationLog.commit @@ -298,10 +307,10 @@ def commit_after_a_peer(log: GenerationLog, successor: Snapshot) -> bool: def test_waiting_contender_republishes_a_swept_record(lock_file: str, files: OsFiles, root: str) -> None: # A record removed out from under a contender (a sweeper that mistook it, an operator) comes back on its next poll, # so peers always have a record to watch by the time the contender is admitted. - writer = _participant(files, lock_file, "write", stale_threshold=100) + writer = _participant(files, lock_file, "write") writer.publish() assert writer.advance() - reader = _participant(files, lock_file, "read", stale_threshold=100) + reader = _participant(files, lock_file, "read") reader.publish() record = Path(root, "holders", reader.token) record.unlink() @@ -314,7 +323,7 @@ def test_waiting_contender_keeps_an_undeletable_record( lock_file: str, files: OsFiles, root: str, mocker: MockerFixture ) -> None: # Windows can leave the record in place while refusing the rewrite; the contender carries on. - reader = _participant(files, lock_file, "read", stale_threshold=100) + reader = _participant(files, lock_file, "read") reader.publish() mocker.patch.object(OsFiles, "overwrite", return_value=False, autospec=True) assert reader.advance() @@ -431,15 +440,6 @@ def test_a_listing_compacted_from_under_the_reader_is_taken_again( assert GenerationLog(files, lock_file, root).latest().writer == _FIRST -def _committed(files: OsFiles, lock_file: str, root: str, writers: list[str | None]) -> GenerationLog: - # One log that read the head before each commit, as a participant does, and so remembers the last one. - files.prepare(root) - log = GenerationLog(files, lock_file, root) - for writer in writers: - assert log.commit(Snapshot(generation=log.latest().generation + 1, writer=writer, readers=frozenset())) - return log - - @pytest.mark.requires_hard_links @pytest.mark.parametrize( ("writers", "expected"), @@ -992,6 +992,13 @@ def test_heartbeat_leaves_a_file_linked_over_the_holder_record_alone(lock_file: _ROOT: Final[str] = "/lock.rw" +#: Each seed with its crash probability and stale-listing rate: every third seed runs without crashes, every other one +#: sees stale directory listings. +_MODEL_CASES: Final[tuple[tuple[int, float, float], ...]] = tuple( + (seed, 0.0 if seed % 3 == 0 else 0.02, 0.3 if seed % 2 else 0.0) for seed in range(24) +) + + @pytest.mark.parametrize( ("seed", "crash_probability", "stale_listings"), [ @@ -1001,9 +1008,7 @@ def test_heartbeat_leaves_a_file_linked_over_the_holder_record_alone(lock_file: stale, id=f"seed{seed}-{'crashes' if probability else 'clean'}-{'stale-listings' if stale else 'fresh-listings'}", ) - for seed in range(24) - for probability in (0.0 if seed % 3 == 0 else 0.02,) - for stale in (0.3 if seed % 2 else 0.0,) + for seed, probability, stale in _MODEL_CASES ], ) @pytest.mark.timeout(120) diff --git a/tests/soft_rw/test_soft_rw_storage.py b/tests/soft_rw/test_soft_rw_storage.py index 8bee72b7..2ebdbeee 100644 --- a/tests/soft_rw/test_soft_rw_storage.py +++ b/tests/soft_rw/test_soft_rw_storage.py @@ -196,8 +196,9 @@ def test_overwrite_does_not_block_on_a_fifo_with_a_reader(tmp_path: Path) -> Non def test_unlink_tolerates_a_refused_removal(tmp_path: Path, mocker: MockerFixture) -> None: - mocker.patch.object(storage_mod.os, "unlink", side_effect=PermissionError("held open"), autospec=True) + unlink = mocker.patch.object(storage_mod.os, "unlink", side_effect=PermissionError("held open"), autospec=True) OsFiles(str(tmp_path / "x.lock")).unlink(str(tmp_path / "record")) + unlink.assert_called_once() def test_unlink_other_errors_propagate(tmp_path: Path, mocker: MockerFixture) -> None: diff --git a/tests/soft_rw/test_soft_rw_sync.py b/tests/soft_rw/test_soft_rw_sync.py index fe4e0847..f6b82f37 100644 --- a/tests/soft_rw/test_soft_rw_sync.py +++ b/tests/soft_rw/test_soft_rw_sync.py @@ -75,6 +75,25 @@ def lock(lock_file: str) -> Generator[SoftReadWriteLock]: instance.close() +def _make_lock( + path: str, + *, + heartbeat_interval: float = 0.1, + stale_threshold: float = 0.5, + poll_interval: float = 0.02, + is_singleton: bool = False, + on_compromise: Callable[[LeaseCompromise], None] | None = None, +) -> SoftReadWriteLock: + return SoftReadWriteLock( + path, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=poll_interval, + is_singleton=is_singleton, + on_compromise=on_compromise, + ) + + def test_atexit_releases_held_lock(lock_file: str, tmp_path: Path) -> None: script = """ import atexit @@ -107,10 +126,6 @@ def check_released(): assert (tmp_path / "released").read_text() == "released" -def _state(lock_file: str) -> Snapshot: - return GenerationLog(OsFiles(lock_file), lock_file, f"{lock_file}.rw").latest() - - def test_release_interrupted_while_joining_the_heartbeat_still_leaves(lock_file: str, mocker: MockerFixture) -> None: lock = _make_lock(lock_file) lock.acquire_write(timeout=2) @@ -414,11 +429,25 @@ def test_leaked_acquired_singleton_is_closed_on_teardown(lock_file: str) -> None assert _state(lock_file).writer is not None -def test_reentrant_read_holds_and_releases(lock: SoftReadWriteLock) -> None: - with lock.read_lock(timeout=2), lock.read_lock(timeout=2): - pass +def _state(lock_file: str) -> Snapshot: + return GenerationLog(OsFiles(lock_file), lock_file, f"{lock_file}.rw").latest() + + +def test_reentrant_read_holds_and_releases(lock_file: str, lock: SoftReadWriteLock) -> None: with lock.read_lock(timeout=2): - pass + with lock.read_lock(timeout=2): + pass + assert not _writer_gets_in(lock_file) + assert _writer_gets_in(lock_file) + + +def _writer_gets_in(lock_file: str) -> bool: + with closing(_make_lock(lock_file)) as contender: + try: + contender.acquire_write(blocking=False) + except Timeout: + return False + return True def test_reentrant_write_holds_and_releases(lock_file: str, lock: SoftReadWriteLock) -> None: @@ -480,11 +509,10 @@ def test_write_lock_release_from_different_thread_prohibited(lock_file: str, loc def test_write_lock_drop_from_different_thread_needs_an_explicit_override( lock_file: str, lock: SoftReadWriteLock, drop: Callable[[SoftReadWriteLock], None] ) -> None: - with closing(_make_lock(lock_file)) as contender, ThreadPoolExecutor(max_workers=1) as other_thread: + with ThreadPoolExecutor(max_workers=1) as other_thread: lock.acquire_write(timeout=2) other_thread.submit(drop, lock).result() - with contender.write_lock(blocking=False): - pass + assert _writer_gets_in(lock_file) def test_blocking_acquire_without_timeout_waits_for_release(lock_file: str) -> None: @@ -526,6 +554,7 @@ def test_release_force_without_hold_is_noop(lock_file: str) -> None: lock = SoftReadWriteLock(lock_file) try: lock.release(force=True) + assert lock.generation is None finally: lock.close() @@ -534,14 +563,19 @@ def test_release_force_on_reentrant_lock_drops_all(lock: SoftReadWriteLock) -> N lock.acquire_read(timeout=2) lock.acquire_read(timeout=2) lock.release(force=True) + assert lock.generation is None with lock.write_lock(timeout=2): - pass + assert lock.generation is not None def test_close_is_idempotent(lock_file: str) -> None: lock = SoftReadWriteLock(lock_file) + lock.acquire_write(timeout=2) lock.close() lock.close() + assert _writer_gets_in(lock_file) + with pytest.raises(RuntimeError, match="has been closed"): + lock.acquire_read(timeout=0) def test_acquire_on_closed_raises(lock_file: str) -> None: @@ -624,6 +658,35 @@ def test_multiple_readers_can_hold_simultaneously(lock_file: str) -> None: p2.join(timeout=_PROCESS_DEADLINE) +def _worker( + lock_file: str, + mode: Literal["read", "write"], + acquired_event: EventType, + release_event: EventType | None = None, + timeout: float = -1, + blocking: bool = True, + heartbeat_interval: float = 0.1, + stale_threshold: float = 1.0, + poll_interval: float = 0.02, +) -> None: + lock = SoftReadWriteLock( + lock_file, + timeout=timeout, + blocking=blocking, + is_singleton=False, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=poll_interval, + ) + try: + with lock.read_lock() if mode == "read" else lock.write_lock(): + acquired_event.set() + if release_event is not None: + release_event.wait(timeout=_PROCESS_DEADLINE) + finally: + lock.close() + + @SKIP_ON_UNRELIABLE_PROCESS_SYNC @pytest.mark.timeout(_PROCESS_DEADLINE * 4) @pytest.mark.parametrize("contender_mode", [pytest.param("write", id="writer"), pytest.param("read", id="reader")]) @@ -759,17 +822,19 @@ def test_two_readers_in_same_process_share_slot(lock_file: str) -> None: try: barrier = threading.Barrier(8) overlap = threading.Barrier(8) + arrivals: list[int] = [] def target() -> None: barrier.wait(timeout=_PROCESS_DEADLINE) with lock.read_lock(timeout=5): - overlap.wait(timeout=_PROCESS_DEADLINE) + arrivals.append(overlap.wait(timeout=_PROCESS_DEADLINE)) threads = [threading.Thread(target=target) for _ in range(8)] for thread in threads: thread.start() for thread in threads: thread.join(timeout=_PROCESS_DEADLINE) + assert sorted(arrivals) == list(range(8)) finally: lock.close() @@ -832,6 +897,28 @@ def test_dead_holder_evicted_by_contender( assert _state(lock_file).members == {hold.participant.token} +def _sigkill_worker( # pragma: forked child + lock_file: str, + mode: Literal["read", "write"], + acquired_event: EventType, + heartbeat_interval: float, + stale_threshold: float, +) -> None: + lock = SoftReadWriteLock( + lock_file, + is_singleton=False, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=0.05, + ) + if mode == "read": + lock.acquire_read() + else: + lock.acquire_write() + acquired_event.set() + threading.Event().wait() + + def test_heartbeat_reports_eviction_and_stops(lock_file: str) -> None: # A peer that waited out the stale threshold commits a snapshot without this holder. The next heartbeat sees the # snapshot no longer naming it, records the loss, and stops refreshing a claim that no longer exists. @@ -999,7 +1086,6 @@ def hook(_seconds: float) -> None: swapped.set() # The peer evicts the draining writer and takes the slot; the reader leaves so only the slot blocks the writer. log = GenerationLog(OsFiles(lock_file), lock_file, f"{lock_file}.rw") - latest = log.latest() peer.append(_plant_holder(lock_file, mode="read")) latest = log.latest() assert log.commit(Snapshot(generation=latest.generation + 1, writer=peer[0], readers=frozenset())) @@ -1082,11 +1168,28 @@ def test_acquire_hands_back_the_slot_when_the_heartbeat_cannot_start( lock.release(force=True) # a handed-back slot leaves nothing to release, so this must not raise +def test_participant_reads_the_clock_patched_after_construction(lock_file: str, mocker: MockerFixture) -> None: + lock = _make_lock(lock_file) + clock = mocker.create_autospec(time.monotonic, side_effect=time.monotonic) + mocker.patch( + "filelock._soft_rw._sync.time", + new=SimpleNamespace(monotonic=clock, perf_counter=time.perf_counter, sleep=time.sleep), + ) + try: + lock.acquire_write(timeout=2) + assert lock._hold is not None + assert lock._hold.participant._clock is clock + finally: + mocker.stopall() + lock.release(force=True) + lock.close() + + def test_short_attempts_still_evict_a_dead_holder(lock_file: str) -> None: # How long a peer's record has stayed unchanged is knowledge the lock keeps across attempts: a caller whose every # attempt is shorter than the stale threshold (a retry loop, blocking=False) must still get past a crashed holder. _plant_holder(lock_file, mode="write") - lock = _make_lock(lock_file, heartbeat_interval=0.1, stale_threshold=0.3) + lock = _make_lock(lock_file, stale_threshold=0.3) try: def attempt() -> bool: @@ -1170,7 +1273,7 @@ def test_lock_file_in_a_missing_directory_is_created(tmp_path: Path) -> None: lock = _make_lock(str(tmp_path / "nested" / "deeper" / "x.lock")) try: with lock.write_lock(timeout=1): - pass + assert (lock.generation, (tmp_path / "nested" / "deeper").is_dir()) == (1, True) finally: lock.close() @@ -1239,7 +1342,7 @@ def test_foreign_host_holder_is_evicted_after_the_stale_threshold(lock_file: str # The report behind #725: a pod dies on another host holding the lock, and a replacement pod under a different # hostname must still get in. Liveness is a nonce that stopped changing, so the hostname plays no part in it. _plant_holder(lock_file, mode="write") - lock = _make_lock(lock_file, heartbeat_interval=0.1, stale_threshold=0.3) + lock = _make_lock(lock_file, stale_threshold=0.3) try: started = time.monotonic() with pytest.raises(Timeout): @@ -1327,7 +1430,7 @@ def remove_once_blocked(participant: Participant) -> bool: def test_malformed_holder_record_is_evicted(lock_file: str) -> None: # A record that cannot be parsed still identifies nobody who refreshes it, so it ages out like any other. Path(f"{lock_file}.rw", "holders", _plant_holder(lock_file, mode="write")).write_bytes(b"\x00garbage") - lock = _make_lock(lock_file, heartbeat_interval=0.1, stale_threshold=0.3) + lock = _make_lock(lock_file, stale_threshold=0.3) try: with lock.write_lock(timeout=5): pass @@ -1537,7 +1640,7 @@ def test_stray_files_in_the_protocol_directories_are_ignored(lock_file: str, loc (root / "gen" / "not-a-generation").write_bytes(b"ignored") (root / "holders" / "not-a-token").write_bytes(b"ignored") with lock.write_lock(timeout=2): - pass + assert lock.generation is not None @SKIP_ON_UNRELIABLE_PROCESS_SYNC @@ -1569,110 +1672,6 @@ def test_child_fork_lock_behavior(tmp_path: Path, scenario: str, names: tuple[st assert result.is_set() -@NEEDS_FORK -@pytest.mark.timeout(_PROCESS_DEADLINE * 2) -# Holding the write lock keeps the heartbeat thread alive, so this fork is necessarily from a multi-threaded -# process and Python 3.15 warns that it may deadlock. That is the scenario under test, and it is already safe: -# register_at_fork resets inherited state in the child (any child use raises "invalidated by fork()"). Expected. -@SKIP_ON_UNRELIABLE_PROCESS_SYNC -@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of fork.*:DeprecationWarning") -def test_parent_retains_lock_across_fork(tmp_path: Path) -> None: # pragma: needs fork - path = str(tmp_path / "foo.lock") - lock = SoftReadWriteLock(path, heartbeat_interval=0.2, stale_threshold=1.0, poll_interval=0.02) - lock.acquire_write(timeout=5) - try: - child = _fork_process(target=time.sleep, args=(0.05,)) - child.start() - child.join(timeout=_PROCESS_DEADLINE) - assert _state(path).writer is not None - peer = SoftReadWriteLock( - path, - heartbeat_interval=0.2, - stale_threshold=1.0, - poll_interval=0.02, - is_singleton=False, - ) - try: - with pytest.raises(Timeout): - peer.acquire_write(timeout=0.3) - finally: - peer.close() - finally: - lock.release() - lock.close() - assert _state(path).writer is None - - -def _make_lock( - path: str, - *, - heartbeat_interval: float = 0.1, - stale_threshold: float = 0.5, - poll_interval: float = 0.02, - is_singleton: bool = False, - on_compromise: Callable[[LeaseCompromise], None] | None = None, -) -> SoftReadWriteLock: - return SoftReadWriteLock( - path, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=poll_interval, - is_singleton=is_singleton, - on_compromise=on_compromise, - ) - - -def _worker( - lock_file: str, - mode: Literal["read", "write"], - acquired_event: EventType, - release_event: EventType | None = None, - timeout: float = -1, - blocking: bool = True, - heartbeat_interval: float = 0.1, - stale_threshold: float = 1.0, - poll_interval: float = 0.02, -) -> None: - lock = SoftReadWriteLock( - lock_file, - timeout=timeout, - blocking=blocking, - is_singleton=False, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=poll_interval, - ) - try: - with lock.read_lock() if mode == "read" else lock.write_lock(): - acquired_event.set() - if release_event is not None: - release_event.wait(timeout=_PROCESS_DEADLINE) - finally: - lock.close() - - -def _sigkill_worker( # pragma: forked child - lock_file: str, - mode: Literal["read", "write"], - acquired_event: EventType, - heartbeat_interval: float, - stale_threshold: float, -) -> None: - lock = SoftReadWriteLock( - lock_file, - is_singleton=False, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=0.05, - ) - if mode == "read": - lock.acquire_read() - else: - lock.acquire_write() - acquired_event.set() - threading.Event().wait() - - def _reuse_inherited_lock(lock_file: str, result: EventType, failure: EventType) -> None: # pragma: needs fork lock = SoftReadWriteLock(lock_file, heartbeat_interval=0.2, stale_threshold=1.0, poll_interval=0.02) lock.acquire_write(timeout=5) @@ -1696,6 +1695,22 @@ def child_entry() -> None: lock.close() +def _fork_event() -> EventType: # pragma: needs fork + if sys.platform == "win32": # pragma: win32 cover + msg = "fork context is POSIX only" + raise RuntimeError(msg) + return mp.get_context("fork").Event() + + +def _fork_process( # pragma: needs fork + target: Callable[[], object] | Callable[[float], object], args: tuple[float, ...] = () +) -> mp.process.BaseProcess: + if sys.platform == "win32": # pragma: win32 cover + msg = "fork context is POSIX only" + raise RuntimeError(msg) + return mp.get_context("fork").Process(target=target, args=args) + + def _release_inherited_lock(lock_file: str, result: EventType, failure: EventType) -> None: # pragma: needs fork lock = SoftReadWriteLock(lock_file, heartbeat_interval=0.2, stale_threshold=1.0, poll_interval=0.02) lock.acquire_read(timeout=5) @@ -1751,20 +1766,38 @@ def child_entry() -> None: parent_lock.close() -def _fork_process( # pragma: needs fork - target: Callable[[], object] | Callable[[float], object], args: tuple[float, ...] = () -) -> mp.process.BaseProcess: - if sys.platform == "win32": # pragma: win32 cover - msg = "fork context is POSIX only" - raise RuntimeError(msg) - return mp.get_context("fork").Process(target=target, args=args) - - -def _fork_event() -> EventType: # pragma: needs fork - if sys.platform == "win32": # pragma: win32 cover - msg = "fork context is POSIX only" - raise RuntimeError(msg) - return mp.get_context("fork").Event() +@NEEDS_FORK +@pytest.mark.timeout(_PROCESS_DEADLINE * 2) +# Holding the write lock keeps the heartbeat thread alive, so this fork is necessarily from a multi-threaded +# process and Python 3.15 warns that it may deadlock. That is the scenario under test, and it is already safe: +# register_at_fork resets inherited state in the child (any child use raises "invalidated by fork()"). Expected. +@SKIP_ON_UNRELIABLE_PROCESS_SYNC +@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of fork.*:DeprecationWarning") +def test_parent_retains_lock_across_fork(tmp_path: Path) -> None: # pragma: needs fork + path = str(tmp_path / "foo.lock") + lock = SoftReadWriteLock(path, heartbeat_interval=0.2, stale_threshold=1.0, poll_interval=0.02) + lock.acquire_write(timeout=5) + try: + child = _fork_process(target=time.sleep, args=(0.05,)) + child.start() + child.join(timeout=_PROCESS_DEADLINE) + assert _state(path).writer is not None + peer = SoftReadWriteLock( + path, + heartbeat_interval=0.2, + stale_threshold=1.0, + poll_interval=0.02, + is_singleton=False, + ) + try: + with pytest.raises(Timeout): + peer.acquire_write(timeout=0.3) + finally: + peer.close() + finally: + lock.release() + lock.close() + assert _state(path).writer is None @SKIP_ON_UNRELIABLE_PROCESS_SYNC @@ -1926,15 +1959,6 @@ def test_soft_read_release_from_a_thread_without_a_read_level_raises( lock.release() -def _writer_gets_in(lock_file: str) -> bool: - with closing(_make_lock(lock_file)) as contender: - try: - contender.acquire_write(blocking=False) - except Timeout: - return False - return True - - def test_soft_read_release_of_a_dropped_hold_leaves_a_newer_one_held( lock: SoftReadWriteLock, other_thread: ThreadPoolExecutor ) -> None: diff --git a/tests/test_async_filelock.py b/tests/test_async_filelock.py index aabf94e4..f11157dc 100644 --- a/tests/test_async_filelock.py +++ b/tests/test_async_filelock.py @@ -735,12 +735,14 @@ def _signal_poll(polling: asyncio.Event) -> bool: @pytest.mark.asyncio -async def test_async_relative_release_uses_acquisition_directory(working_directories: tuple[Path, Path]) -> None: +async def test_async_relative_release_uses_acquisition_directory( + working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: first, second = working_directories lock = AsyncSoftFileLock("resource.lock") await lock.acquire() - os.chdir(second) + monkeypatch.chdir(second) await lock.release() assert not (first / "resource.lock").exists() @@ -749,12 +751,12 @@ async def test_async_relative_release_uses_acquisition_directory(working_directo @pytest.mark.parametrize("lock_type", [AsyncFileLock, AsyncSoftFileLock]) @pytest.mark.asyncio async def test_async_singleton_held_before_chdir_re_enters_by_absolute_path( - lock_type: type[BaseAsyncFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseAsyncFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: first, second = working_directories held = lock_type("resource.lock", is_singleton=True) await held.acquire() - os.chdir(second) + monkeypatch.chdir(second) async with lock_type(first / "resource.lock", is_singleton=True) as again: assert (again is held, held.lock_counter) == (True, 2) await held.release(force=True) @@ -980,16 +982,6 @@ async def test_soft_close_error_policy_cleans_marker( assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) -@pytest.mark.asyncio -async def test_soft_close_error_suppress_policy_cleans_marker(tmp_path: Path, mocker: MockerFixture) -> None: - lock_path = tmp_path / "a" - lock = AsyncSoftFileLock(lock_path, close_error_policy="suppress", run_in_executor=False) - await lock.acquire() - with _close_after_commit(mocker, lock) as (_, attempts): - await lock.release() - assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) - - @contextmanager def _close_after_commit(mocker: MockerFixture, lock: AsyncSoftFileLock) -> Generator[tuple[OSError, list[int]]]: real_close = os.close @@ -1021,6 +1013,16 @@ def close(fd: int) -> None: mocker.stop(close_mock) +@pytest.mark.asyncio +async def test_soft_close_error_suppress_policy_cleans_marker(tmp_path: Path, mocker: MockerFixture) -> None: + lock_path = tmp_path / "a" + lock = AsyncSoftFileLock(lock_path, close_error_policy="suppress", run_in_executor=False) + await lock.acquire() + with _close_after_commit(mocker, lock) as (_, attempts): + await lock.release() + assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) + + @pytest.mark.asyncio async def test_soft_close_error_suppression_releases_nested_acquisitions_once( tmp_path: Path, mocker: MockerFixture diff --git a/tests/test_async_filelock_runner_shutdown.py b/tests/test_async_filelock_runner_shutdown.py index 42f746b4..4c15bc5d 100644 --- a/tests/test_async_filelock_runner_shutdown.py +++ b/tests/test_async_filelock_runner_shutdown.py @@ -19,26 +19,6 @@ _T = TypeVar("_T") -class _CancellationObservedTask(asyncio.Task[_T]): # pragma: needs fcntl - def __init__( - self, - coroutine: Coroutine[None, None, _T], - cancellation_seen: threading.Event, - *, - loop: asyncio.AbstractEventLoop, - ) -> None: - self._cancellation_seen = cancellation_seen - super().__init__(coroutine, loop=loop) - - def cancel( - self, - msg: object | None = None, # simplint: ignore[imprecise-annotation] # Task.cancel accepts arbitrary payloads - ) -> bool: - # Task.cancel accepts arbitrary payloads; object is the narrowest accurate type for its public contract. - self._cancellation_seen.set() - return super().cancel(msg) - - @NEEDS_FCNTL def test_runner_shutdown_waits_for_executor_acquire_rollback(tmp_path: Path) -> None: # pragma: needs fcntl hook_started = threading.Event() @@ -87,6 +67,26 @@ async def start_acquire() -> None: asyncio.run(start_acquire()) +class _CancellationObservedTask(asyncio.Task[_T]): # pragma: needs fcntl + def __init__( + self, + coroutine: Coroutine[None, None, _T], + cancellation_seen: threading.Event, + *, + loop: asyncio.AbstractEventLoop, + ) -> None: + self._cancellation_seen = cancellation_seen + super().__init__(coroutine, loop=loop) + + def cancel( + self, + msg: object | None = None, # simplint: ignore[imprecise-annotation] # Task.cancel accepts arbitrary payloads + ) -> bool: + # Task.cancel accepts arbitrary payloads; object is the narrowest accurate type for its public contract. + self._cancellation_seen.set() + return super().cancel(msg) + + @NEEDS_FCNTL @XFAIL_WITHOUT_COROUTINE_CANCELLATION def test_runner_shutdown_chains_body_cancellation_and_release_errors( # pragma: needs fcntl diff --git a/tests/test_async_read_write.py b/tests/test_async_read_write.py index 55544ebb..af1ee5a3 100644 --- a/tests/test_async_read_write.py +++ b/tests/test_async_read_write.py @@ -12,7 +12,14 @@ import pytest_asyncio from tests.capability_marks import NEEDS_COLLECTED_FINALIZATION, XFAIL_WITHOUT_COROUTINE_CANCELLATION -from tests.read_write_helpers import ACQUIRE_SETTINGS, INVALID_TIMEOUTS, OPPOSITE_MODE, Interrupt, interrupt_once +from tests.read_write_helpers import ( + ACQUIRE_SETTINGS, + INVALID_TIMEOUTS, + OPPOSITE_MODE, + Interrupt, + interrupt_once, + writer_gets_in, +) pytest.importorskip("sqlite3") @@ -182,9 +189,19 @@ async def test_release_force(lock_file: str) -> None: async def test_release_force_unheld_is_noop(lock_file: str) -> None: lock = AsyncReadWriteLock(lock_file, is_singleton=False) await lock.release(force=True) + assert await _try_write(lock) await lock.close() +async def _try_write(lock: AsyncReadWriteLock) -> bool: + try: + await lock.acquire_write(blocking=False) + except Timeout: + return False + await lock.release() + return True + + @pytest.mark.asyncio async def test_close_rejects_later_acquisition_idle_owned_executor(lock_file: str) -> None: lock = AsyncReadWriteLock(lock_file, is_singleton=False) @@ -263,8 +280,7 @@ def park() -> None: finally: gate.set() await parked - async with lock.write_lock(timeout=_TASK_WAIT): - pass + assert await _try_write(lock) await lock.close() executor.shutdown() @@ -356,12 +372,13 @@ async def test_context_manager_overrides_defaults(lock_file: str) -> None: async def test_sequential_mode_switch(lock_file: str) -> None: lock = AsyncReadWriteLock(lock_file, is_singleton=False) async with lock.read_lock(): - pass + assert not await asyncio.to_thread(writer_gets_in, lock_file) async with lock.write_lock(): - pass + assert not await asyncio.to_thread(writer_gets_in, lock_file) async with lock.read_lock(): - pass + assert not await asyncio.to_thread(writer_gets_in, lock_file) await lock.close() + assert writer_gets_in(lock_file) @pytest.mark.asyncio @@ -426,15 +443,6 @@ async def test_another_task_waits_for_the_last_release_of_a_nested_write(lock_fi assert (entered_while_nested, entered_after_release) == (False, True) -async def _try_write(lock: AsyncReadWriteLock) -> bool: - try: - await lock.acquire_write(blocking=False) - except Timeout: - return False - await lock.release() - return True - - @pytest.mark.asyncio async def test_force_release_drops_the_callers_whole_nest(lock_file: str) -> None: lock = AsyncReadWriteLock(lock_file, is_singleton=False) @@ -774,18 +782,7 @@ async def test_async_close_after_a_sync_end_of_the_shared_singleton_returns( await lock.close() - assert _writer_gets_in(lock_file) - - -def _writer_gets_in(lock_file: str) -> bool: - contender: Final = ReadWriteLock(lock_file, is_singleton=False) - try: - contender.acquire_write(blocking=False) - except Timeout: - return False - finally: - contender.close() - return True + assert writer_gets_in(lock_file) @pytest.mark.asyncio @@ -813,7 +810,7 @@ async def test_async_read_after_a_sync_forced_release_takes_the_lock_again(lock_ async def _read_and_probe(lock: AsyncReadWriteLock) -> bool: async with lock.read_lock(timeout=_TASK_WAIT): - return _writer_gets_in(lock.lock_file) + return writer_gets_in(lock.lock_file) @pytest.mark.parametrize( @@ -903,9 +900,9 @@ async def test_async_close_keeps_the_hold_taken_through_another_wrapper( await asyncio.create_task(closing.close()) - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) await holder.release() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) await holder.close() @@ -917,9 +914,9 @@ async def test_async_close_of_the_entering_wrapper_keeps_a_level_taken_through_a await entering.close() - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) await nested.release() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) await nested.close() @@ -932,7 +929,7 @@ async def test_async_close_of_the_nested_wrapper_ends_only_its_own_level(lock_fi await nested.close() await entering.release() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) await entering.close() @@ -944,9 +941,9 @@ async def test_async_release_through_each_wrapper_ends_the_hold_after_the_last(l await nested.release() - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) await entering.release() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) await entering.close() await nested.close() @@ -958,9 +955,9 @@ async def test_async_forced_release_through_a_wrapper_without_a_level_releases_n await other.release(force=True) - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) await holder.release() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) await holder.close() await other.close() @@ -973,7 +970,7 @@ async def test_async_release_through_a_wrapper_without_a_level_raises(lock_file: with pytest.raises(RuntimeError, match="through this wrapper"): await other.release() - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) await holder.release() await holder.close() await other.close() @@ -987,7 +984,7 @@ async def test_async_close_keeps_a_sync_reader_of_the_shared_singleton(lock_file pass await lock.close() - assert not _writer_gets_in(lock_file) + assert not writer_gets_in(lock_file) @pytest.mark.asyncio @@ -997,7 +994,7 @@ async def test_async_close_of_a_singleton_wrapper_ends_the_holds_taken_through_i await lock.close() - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) with pytest.raises(sqlite3.ProgrammingError, match="closed database"): await lock.acquire_read() @@ -1019,7 +1016,7 @@ async def test_async_close_waits_out_an_acquire_in_flight_even_when_canceled(loc with pytest.raises(asyncio.CancelledError): await closing - assert _writer_gets_in(lock_file) + assert writer_gets_in(lock_file) holder.close() diff --git a/tests/test_async_read_write_cancel_wait.py b/tests/test_async_read_write_cancel_wait.py index 87c13585..1aecca1e 100644 --- a/tests/test_async_read_write_cancel_wait.py +++ b/tests/test_async_read_write_cancel_wait.py @@ -69,17 +69,6 @@ def __call__( ) -async def _acquire_within_deadline( - lock: AsyncReadWriteLock | AsyncSoftReadWriteLock, mode: Literal["read", "write"], timeout: float -) -> float: - start = time.perf_counter() - with pytest.raises(TimeoutError) as info: - async with asyncio.timeout(_DEADLINE): - await getattr(lock, f"acquire_{mode}")(timeout) - assert not isinstance(info.value, Timeout) - return time.perf_counter() - start - - @_FAMILIES @_MODES @pytest.mark.parametrize( @@ -103,6 +92,17 @@ async def test_async_acquire_behind_another_process_honors_the_callers_deadline( await lock.close() +async def _acquire_within_deadline( + lock: AsyncReadWriteLock | AsyncSoftReadWriteLock, mode: Literal["read", "write"], timeout: float +) -> float: + start = time.perf_counter() + with pytest.raises(TimeoutError) as info: + async with asyncio.timeout(_DEADLINE): + await getattr(lock, f"acquire_{mode}")(timeout) + assert not isinstance(info.value, Timeout) + return time.perf_counter() - start + + @_FAMILIES @pytest.mark.asyncio async def test_async_acquire_behind_a_sync_acquire_of_the_same_instance_honors_the_callers_deadline( diff --git a/tests/test_async_transition_gate.py b/tests/test_async_transition_gate.py index d39864a0..10714b09 100644 --- a/tests/test_async_transition_gate.py +++ b/tests/test_async_transition_gate.py @@ -204,6 +204,7 @@ async def second() -> None: async with asyncio.timeout(1): await second_entered.wait() await asyncio.gather(first_task, second_task) + assert calls > 1 @pytest.mark.asyncio diff --git a/tests/test_filelock.py b/tests/test_filelock.py index 799509c5..d7f1693c 100644 --- a/tests/test_filelock.py +++ b/tests/test_filelock.py @@ -1133,15 +1133,16 @@ def _run_shared_threads(target: Callable[[int], None]) -> None: def test_shared_instance_released_on_another_thread_frees_the_acquirer(tmp_path: Path) -> None: shared = FileLock(tmp_path / "a.lock", thread_local=False) - def acquire_fresh() -> None: - with FileLock(tmp_path / "a.lock"): - pass + def acquire_fresh() -> bool: + with FileLock(tmp_path / "a.lock") as fresh: + return fresh.is_locked with ThreadPoolExecutor(max_workers=1) as acquirer: acquirer.submit(shared.acquire).result(_SHARED_WAIT) shared.release() - acquirer.submit(acquire_fresh).result(_SHARED_WAIT) + assert not shared.is_locked + assert acquirer.submit(acquire_fresh).result(_SHARED_WAIT) def test_shared_instance_release_behind_a_failed_acquire_is_a_no_op(tmp_path: Path) -> None: @@ -3154,6 +3155,15 @@ def test_singleton_rejects_different_context_policy(tmp_path: Path) -> None: first.release(force=True) +@NEEDS_FCNTL # pragma: needs fcntl +def test_close_error_default_suppressed_on_unix(tmp_path: Path, mocker: MockerFixture) -> None: + lock = FileLock(str(tmp_path / "a")) # default policy + lock.acquire() + _fail_close_of(mocker, lock, OSError(EIO, "close failed")) + lock.release() # Unix default drops a FUSE/Docker EIO + assert not lock.is_locked + + def _fail_close_of(mocker: MockerFixture, lock: BaseFileLock, error: OSError) -> list[int]: # Fail os.close only for this lock's own descriptor, and record each attempt on it. A blanket patch would also hit # the close another test's lock runs from __del__ during this test, turning an unrelated garbage collection into a @@ -3172,15 +3182,6 @@ def close(target: int) -> None: return attempts -@NEEDS_FCNTL # pragma: needs fcntl -def test_close_error_default_suppressed_on_unix(tmp_path: Path, mocker: MockerFixture) -> None: - lock = FileLock(str(tmp_path / "a")) # default policy - lock.acquire() - _fail_close_of(mocker, lock, OSError(EIO, "close failed")) - lock.release() # Unix default drops a FUSE/Docker EIO - assert not lock.is_locked - - @_WINDOWS_ONLY # pragma: win32 cover def test_close_error_default_raises_on_windows(tmp_path: Path, mocker: MockerFixture) -> None: lock = FileLock(str(tmp_path / "a")) # default policy @@ -3337,13 +3338,13 @@ def _signal_poll(polling: threading.Event) -> bool: @pytest.mark.parametrize("lock_type", [SoftFileLock, MarkerSoftFileLock, SoftFileLease]) def test_relative_release_uses_acquisition_directory( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: first, second = working_directories lock = lock_type("resource.lock") lock.acquire() - os.chdir(second) + monkeypatch.chdir(second) with lock: # re-entry joins the hold, so it keeps that hold's path pass lock.release() @@ -3421,14 +3422,14 @@ def test_reentrant_acquire_after_directory_rename( @pytest.mark.parametrize("lock_type", _RELATIVE_LOCK_TYPES) def test_each_acquisition_resolves_the_working_directory( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: _, second = working_directories lock = lock_type("resource.lock") with lock: pass - os.chdir(second) + monkeypatch.chdir(second) with lock: assert (lock.lock_file, (second / "resource.lock").exists()) == ("resource.lock", True) @@ -3438,7 +3439,7 @@ def test_each_acquisition_resolves_the_working_directory( ) # pragma: win32 no cover @pytest.mark.parametrize("lock_type", _RELATIVE_LOCK_TYPES) def test_acquire_without_working_directory_leaves_no_hold( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: lock: Final[BaseFileLock] = lock_type("resource.lock") working_directories[0].rmdir() @@ -3446,7 +3447,7 @@ def test_acquire_without_working_directory_leaves_no_hold( with pytest.raises(FileNotFoundError): lock.acquire() second: Final[Path] = working_directories[1] - os.chdir(second) + monkeypatch.chdir(second) with lock: assert (lock.lock_counter, lock.is_locked, (second / "resource.lock").exists()) == (1, True, True) @@ -3458,7 +3459,7 @@ def test_acquire_without_working_directory_leaves_no_hold( ) # pragma: win32 no cover @pytest.mark.parametrize("lock_type", _RELATIVE_LOCK_TYPES) def test_nested_acquire_without_working_directory_preserves_hold( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: lock: Final[BaseFileLock] = lock_type("resource.lock") lock.acquire() @@ -3470,18 +3471,18 @@ def test_nested_acquire_without_working_directory_preserves_hold( lock.acquire() assert (lock.lock_counter, lock.is_locked) == (1, True) - os.chdir(working_directories[1]) + monkeypatch.chdir(working_directories[1]) lock.release() assert (lock.lock_counter, lock.is_locked) == (0, False) @pytest.mark.parametrize("lock_type", _RELATIVE_LOCK_TYPES) def test_singleton_after_chdir_locks_the_requested_file( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: first, second = working_directories relative = lock_type("resource.lock", is_singleton=True) - os.chdir(second) + monkeypatch.chdir(second) with lock_type(first / "resource.lock", is_singleton=True): assert ((first / "resource.lock").exists(), (second / "resource.lock").exists()) == (True, False) del relative @@ -3489,12 +3490,12 @@ def test_singleton_after_chdir_locks_the_requested_file( @pytest.mark.parametrize("lock_type", _RELATIVE_LOCK_TYPES) def test_singleton_held_before_chdir_re_enters_by_absolute_path( - lock_type: type[BaseFileLock], working_directories: tuple[Path, Path] + lock_type: type[BaseFileLock], working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: first, second = working_directories held = lock_type("resource.lock", is_singleton=True) held.acquire() - os.chdir(second) + monkeypatch.chdir(second) with lock_type(first / "resource.lock", is_singleton=True) as again: assert (again is held, held.lock_counter) == (True, 2) held.release(force=True) diff --git a/tests/test_fork_coordination.py b/tests/test_fork_coordination.py index 1611602e..2e402f59 100644 --- a/tests/test_fork_coordination.py +++ b/tests/test_fork_coordination.py @@ -11,7 +11,7 @@ from capabilities import CAPABILITIES from filelock import BaseFileLock, Timeout, has_fcntl -from filelock._api import fork_transition +from filelock._api import _FORK_STATE, fork_transition from tests.capability_marks import ( NEEDS_AUDIT_EVENTS, NEEDS_FORK, @@ -1392,3 +1392,4 @@ def test_fork_transition_finished_on_another_thread_releases_the_entering_thread transition.__enter__() # ruff:ignore[unnecessary-dunder-call] # entered and left on different threads with ThreadPoolExecutor(max_workers=1) as pool: pool.submit(transition.__exit__, None, None, None).result() + assert threading.get_ident() not in _FORK_STATE.transitions diff --git a/tests/test_fork_ownership.py b/tests/test_fork_ownership.py index d8724efe..5aa3753b 100644 --- a/tests/test_fork_ownership.py +++ b/tests/test_fork_ownership.py @@ -69,6 +69,49 @@ def cleanup_child() -> NoReturn: assert _probe_lock(lock_type, path) +def _run_child_cleanup( + lock: BaseFileLock, + proxy: AcquireReturnProxy, + action: Literal["release", "context", "collect"], +) -> NoReturn: + assert (lock.is_locked, lock.lock_counter) == (False, 0) + if action == "release": + lock.release(force=True) + elif action == "context": + proxy.__exit__(None, None, None) + else: + del proxy + del lock + gc.collect() + exit_child(0) + + +def _probe_lock(lock_type: type[BaseFileLock], path: str) -> bool: # pragma: needs fork + read_fd, write_fd = os.pipe() + + def probe_child() -> NoReturn: + os.close(read_fd) + candidate = lock_type(path, is_singleton=False) + try: + candidate.acquire(timeout=0) + except Timeout: + acquired = False + else: + acquired = True + candidate.release() + os.write(write_fd, b"1" if acquired else b"0") + os.close(write_fd) + exit_child(0) + + child_pid = fork_process(probe_child) + os.close(write_fd) + result = os.read(read_fd, 1) + os.close(read_fd) + _, status = os.waitpid(child_pid, 0) + assert os.waitstatus_to_exitcode(status) == 0 + return result == b"1" + + @NEEDS_FORK # pragma: needs fork @_FORK_WARNING @pytest.mark.parametrize( @@ -155,6 +198,14 @@ def descriptor_child() -> NoReturn: assert (os.waitstatus_to_exitcode(status), worker.is_alive()) == (0, False) +def _assert_descriptor_closed(fd: int) -> NoReturn: + try: + os.fstat(fd) + except OSError as exception: + exit_child(0 if exception.errno == EBADF else 1) + exit_child(1) + + @NEEDS_FORK # pragma: needs fork @_FORK_WARNING def test_fork_waits_for_descriptor_registration(tmp_path: Path, mocker: MockerFixture) -> None: @@ -438,54 +489,3 @@ def release(lock: BaseFileLock | SoftReadWriteLock) -> None: ) assert result.returncode == 0, result.stderr - - -def _run_child_cleanup( - lock: BaseFileLock, - proxy: AcquireReturnProxy, - action: Literal["release", "context", "collect"], -) -> NoReturn: - assert (lock.is_locked, lock.lock_counter) == (False, 0) - if action == "release": - lock.release(force=True) - elif action == "context": - proxy.__exit__(None, None, None) - else: - del proxy - del lock - gc.collect() - exit_child(0) - - -def _probe_lock(lock_type: type[BaseFileLock], path: str) -> bool: # pragma: needs fork - read_fd, write_fd = os.pipe() - - def probe_child() -> NoReturn: - os.close(read_fd) - candidate = lock_type(path, is_singleton=False) - try: - candidate.acquire(timeout=0) - except Timeout: - acquired = False - else: - acquired = True - candidate.release() - os.write(write_fd, b"1" if acquired else b"0") - os.close(write_fd) - exit_child(0) - - child_pid = fork_process(probe_child) - os.close(write_fd) - result = os.read(read_fd, 1) - os.close(read_fd) - _, status = os.waitpid(child_pid, 0) - assert os.waitstatus_to_exitcode(status) == 0 - return result == b"1" - - -def _assert_descriptor_closed(fd: int) -> NoReturn: - try: - os.fstat(fd) - except OSError as exception: - exit_child(0 if exception.errno == EBADF else 1) - exit_child(1) diff --git a/tests/test_marker_records.py b/tests/test_marker_records.py index ccafddf9..77435a46 100644 --- a/tests/test_marker_records.py +++ b/tests/test_marker_records.py @@ -192,6 +192,18 @@ def test_lease_ages_out_a_malformed_record(tmp_path: Path, mocker: MockerFixture lease.release() +def _acquire_after_reclaim(lock: MarkerSoftFileLock) -> None: + # One poll per call: the first sees the marker, the second evicts a stale or aged-out one and the third takes the + # free path. Counting polls instead of timing them keeps a stalled runner from failing a reclaim that happens. + assert any(_polled_once_and_holds(lock) for _ in range(3)), "the marker was not reclaimed within three polls" + + +def _polled_once_and_holds(lock: MarkerSoftFileLock) -> bool: + with suppress(Timeout): + lock.acquire(blocking=False) + return lock.is_locked + + def test_marker_pid_and_owner_are_none_without_marker(tmp_path: Path) -> None: lease = SoftFileLease(tmp_path / "a") assert (lease.pid, lease.owner, lease.is_lock_held_by_us) == (None, None, False) @@ -282,6 +294,11 @@ def test_marker_lock_reclaims_dead_owner(marker: Path) -> None: lock.release() +def _leave_marker(marker: Path) -> None: + with MarkerSoftFileLock(marker): + os._exit(0) + + @pytest.mark.timeout(_PROCESS_DEADLINE * 2) @NEEDS_STALE_RECLAIM def test_marker_lock_reclaims_dead_owner_before_its_process_is_closed(marker: Path) -> None: @@ -323,23 +340,6 @@ def test_marker_lock_reclaims_malformed_record(marker: Path, record: str, mocker lock.release() -def _acquire_after_reclaim(lock: MarkerSoftFileLock) -> None: - # One poll per call: the first sees the marker, the second evicts a stale or aged-out one and the third takes the - # free path. Counting polls instead of timing them keeps a stalled runner from failing a reclaim that happens. - assert any(_polled_once_and_holds(lock) for _ in range(3)), "the marker was not reclaimed within three polls" - - -def _polled_once_and_holds(lock: MarkerSoftFileLock) -> bool: - with suppress(Timeout): - lock.acquire(blocking=False) - return lock.is_locked - - @pytest.fixture def marker(tmp_path: Path) -> Path: return tmp_path / "a" - - -def _leave_marker(marker: Path) -> None: - with MarkerSoftFileLock(marker): - os._exit(0) diff --git a/tests/test_read_write.py b/tests/test_read_write.py index db33cd10..98493f1d 100644 --- a/tests/test_read_write.py +++ b/tests/test_read_write.py @@ -25,7 +25,7 @@ from filelock._read_write import _ForkSafeConnection from tests.capability_marks import NEEDS_FIFO, SKIP_ON_UNRELIABLE_PROCESS_SYNC from tests.process_helpers import cleanup_processes -from tests.read_write_helpers import Interrupt, assert_read_write_lock_state, interrupt_once +from tests.read_write_helpers import Interrupt, assert_read_write_lock_state, interrupt_once, writer_gets_in if sys.platform == "win32": # pragma: win32 cover from filelock._identity import host_name @@ -95,6 +95,28 @@ def test_read_locks_are_shared(lock_file: str) -> None: assert not reader2.is_alive(), "Reader 2 did not exit cleanly" +def acquire_lock( + lock_file: str, + mode: Literal["read", "write"], + acquired_event: EventType, + release_event: EventType | None = None, + timeout: float = -1, + blocking: bool = True, + ready_event: EventType | None = None, + contending_event: EventType | None = None, +) -> None: + if ready_event: + ready_event.wait(timeout=_PROCESS_DEADLINE) + + lock = ReadWriteLock(lock_file, timeout=timeout, blocking=blocking) + if contending_event: + contending_event.set() # interpreter start-up is over, so a caller can time the contention itself + with getattr(lock, f"{mode}_lock")(): + acquired_event.set() + if release_event: + release_event.wait(timeout=_PROCESS_DEADLINE) + + @pytest.mark.timeout(_PROCESS_DEADLINE * 5) def test_write_lock_excludes_other_write_locks(lock_file: str) -> None: write1_acquired = Event() @@ -263,6 +285,50 @@ def test_write_non_starvation(lock_file: str) -> None: assert not reader.is_alive(), f"Reader {idx} did not exit cleanly" +def chain_reader( + reader_index: int, + lock_file: str, + release_count: Synchronized[int], + start_signal: EventType, + release_signal: EventType, + next_reader_signal: EventType | None, + writer_or_prev_signal: EventType, +) -> None: + start_signal.wait(timeout=_PROCESS_DEADLINE) + + with ReadWriteLock(lock_file).read_lock(): + if reader_index > 0: + time.sleep( # simplint: ignore[sleep-in-test] # needs a write-intent signal + 2 + ) # overlap with earlier readers so the writer contends mid-chain + + if next_reader_signal is not None: + next_reader_signal.set() + + if reader_index == 0: + time.sleep( # simplint: ignore[sleep-in-test] # needs a write-intent signal + 1 + ) # hold briefly before releasing the writer + + writer_or_prev_signal.set() + + release_signal.wait(timeout=_PROCESS_DEADLINE) + + with release_count.get_lock(): + release_count.value += 1 + + +def _write_intent_pending(prober: ReadWriteLock, deadline: float) -> bool: + try: + prober.acquire_read(blocking=False) + except Timeout: + return False + prober.release() + assert time.monotonic() < deadline, "Writer never registered its write intent" + time.sleep(0.05) # simplint: ignore[sleep-in-test] # needs a write-intent signal + return True + + @pytest.mark.parametrize( "hold_mode", [pytest.param("read", id="upgrade"), pytest.param("write", id="downgrade")], @@ -281,6 +347,18 @@ def test_lock_mode_transition_prohibited(lock_file: str, hold_mode: Literal["rea assert result.value == 0, f"Illegal {hold_mode} transition was permitted" +def try_illegal_transition(lock_file: str, hold_mode: Literal["read", "write"], result: Synchronized[int]) -> None: + lock = ReadWriteLock(lock_file) + with getattr(lock, f"{hold_mode}_lock")(): + try: + if hold_mode == "read": + lock.acquire_write() + else: + lock.acquire_read() + except RuntimeError: + result.value = 0 + + @pytest.mark.timeout(_PROCESS_DEADLINE * 3) def test_timeout_behavior(lock_file: str) -> None: write_acquired = Event() @@ -294,13 +372,13 @@ def test_timeout_behavior(lock_file: str) -> None: writer.start() assert write_acquired.wait(timeout=_PROCESS_DEADLINE), "Write lock not acquired" - start_time = time.time() + start_time = time.perf_counter() reader.start() assert not read_acquired.wait(timeout=1), "Read lock should not be acquired" reader.join(timeout=_PROCESS_DEADLINE) - elapsed = time.time() - start_time + elapsed = time.perf_counter() - start_time assert 0.4 <= elapsed <= 10.0, f"Timeout was not respected: {elapsed}s" release_write.set() @@ -318,12 +396,12 @@ def test_non_blocking_behavior(lock_file: str) -> None: writer.start() assert write_acquired.wait(timeout=_PROCESS_DEADLINE), "Write lock not acquired" - start_time = time.time() + start_time = time.perf_counter() with pytest.raises(Timeout): ReadWriteLock(lock_file).acquire_read(blocking=False) - elapsed = time.time() - start_time + elapsed = time.perf_counter() - start_time assert elapsed < 0.1, f"Non-blocking took too long: {elapsed}s" @@ -347,6 +425,33 @@ def test_recursive_lock_acquisition(lock_file: str, mode: Literal["read", "write assert success.value == 1, "Recursive lock acquisition failed" +def recursive_lock(lock_file: str, mode: Literal["read", "write"], success_flag: Synchronized[int]) -> None: + lock = ReadWriteLock(lock_file) + acquire = getattr(lock, f"{mode}_lock") + with acquire(): + assert lock._hold.level == 1 + assert lock._hold.mode == mode + + with acquire(): + assert lock._hold.level == 2 + assert lock._hold.mode == mode + + with acquire(): + assert lock._hold.level == 3 + assert lock._hold.mode == mode + + assert lock._hold.level == 2 + assert lock._hold.mode == mode + + assert lock._hold.level == 1 + assert lock._hold.mode == mode + + assert lock._hold.level == 0 + assert lock._hold.mode is None + + success_flag.value = 1 + + @pytest.mark.parametrize( "mode", [pytest.param("read", id="read"), pytest.param("write", id="write")], @@ -375,6 +480,14 @@ def test_lock_release_on_process_termination(lock_file: str, mode: Literal["read assert not successor.is_alive(), "Successor did not exit cleanly" +def acquire_lock_and_crash(lock_file: str, mode: Literal["read", "write"], acquired_event: EventType) -> None: + lock = ReadWriteLock(lock_file) # pragma: win32 no cover + with getattr(lock, f"{mode}_lock")(): # pragma: win32 no cover + acquired_event.set() + while True: # pragma: win32 no cover + time.sleep(0.1) # simplint: ignore[sleep-in-test] # a blocked lock wait can swallow SIGTERM + + @pytest.mark.parametrize( "mode", [pytest.param("read", id="read"), pytest.param("write", id="write")], @@ -383,11 +496,12 @@ def test_lock_release_on_process_termination(lock_file: str, mode: Literal["read def test_single_lock_reentrant(lock_file: str, mode: Literal["read", "write"]) -> None: acquire = getattr(ReadWriteLock(lock_file), f"{mode}_lock") - with acquire(), acquire(): - pass - with acquire(): - pass + with acquire(): + pass + assert not writer_gets_in(lock_file) + + assert writer_gets_in(lock_file) @pytest.mark.parametrize( @@ -406,7 +520,8 @@ def test_sequential_lock_modes( lock = ReadWriteLock(lock_file) for mode in (first, second): with getattr(lock, f"{mode}_lock")(): - pass + assert not writer_gets_in(lock_file) + assert writer_gets_in(lock_file) def test_read_write_lock_keeps_constructed_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -781,28 +896,6 @@ def test_acquire_proxy_releases_the_read_write_lock_on_exit(lock_file: str, mode assert_read_write_lock_state(lock_file, "write", available=True) -def acquire_lock( - lock_file: str, - mode: Literal["read", "write"], - acquired_event: EventType, - release_event: EventType | None = None, - timeout: float = -1, - blocking: bool = True, - ready_event: EventType | None = None, - contending_event: EventType | None = None, -) -> None: - if ready_event: - ready_event.wait(timeout=_PROCESS_DEADLINE) - - lock = ReadWriteLock(lock_file, timeout=timeout, blocking=blocking) - if contending_event: - contending_event.set() # interpreter start-up is over, so a caller can time the contention itself - with getattr(lock, f"{mode}_lock")(): - acquired_event.set() - if release_event: - release_event.wait(timeout=_PROCESS_DEADLINE) - - def test_cleanup_reports_the_failure_that_escaped_before_a_start() -> None: unstarted = Process(target=time.sleep, args=(0,)) @@ -815,97 +908,6 @@ def fail_before_start() -> None: fail_before_start() -def _write_intent_pending(prober: ReadWriteLock, deadline: float) -> bool: - try: - prober.acquire_read(blocking=False) - except Timeout: - return False - prober.release() - assert time.monotonic() < deadline, "Writer never registered its write intent" - time.sleep(0.05) # simplint: ignore[sleep-in-test] # needs a write-intent signal - return True - - -def chain_reader( - reader_index: int, - lock_file: str, - release_count: Synchronized[int], - start_signal: EventType, - release_signal: EventType, - next_reader_signal: EventType | None, - writer_or_prev_signal: EventType, -) -> None: - start_signal.wait(timeout=_PROCESS_DEADLINE) - - with ReadWriteLock(lock_file).read_lock(): - if reader_index > 0: - time.sleep( # simplint: ignore[sleep-in-test] # needs a write-intent signal - 2 - ) # overlap with earlier readers so the writer contends mid-chain - - if next_reader_signal is not None: - next_reader_signal.set() - - if reader_index == 0: - time.sleep( # simplint: ignore[sleep-in-test] # needs a write-intent signal - 1 - ) # hold briefly before releasing the writer - - writer_or_prev_signal.set() - - release_signal.wait(timeout=_PROCESS_DEADLINE) - - with release_count.get_lock(): - release_count.value += 1 - - -def try_illegal_transition(lock_file: str, hold_mode: Literal["read", "write"], result: Synchronized[int]) -> None: - lock = ReadWriteLock(lock_file) - with getattr(lock, f"{hold_mode}_lock")(): - try: - if hold_mode == "read": - lock.acquire_write() - else: - lock.acquire_read() - except RuntimeError: - result.value = 0 - - -def recursive_lock(lock_file: str, mode: Literal["read", "write"], success_flag: Synchronized[int]) -> None: - lock = ReadWriteLock(lock_file) - acquire = getattr(lock, f"{mode}_lock") - with acquire(): - assert lock._lock_level == 1 - assert lock._current_mode == mode - - with acquire(): - assert lock._lock_level == 2 - assert lock._current_mode == mode - - with acquire(): - assert lock._lock_level == 3 - assert lock._current_mode == mode - - assert lock._lock_level == 2 - assert lock._current_mode == mode - - assert lock._lock_level == 1 - assert lock._current_mode == mode - - assert lock._lock_level == 0 - assert lock._current_mode is None - - success_flag.value = 1 - - -def acquire_lock_and_crash(lock_file: str, mode: Literal["read", "write"], acquired_event: EventType) -> None: - lock = ReadWriteLock(lock_file) # pragma: win32 no cover - with getattr(lock, f"{mode}_lock")(): # pragma: win32 no cover - acquired_event.set() - while True: # pragma: win32 no cover - time.sleep(0.1) # simplint: ignore[sleep-in-test] # a blocked lock wait can swallow SIGTERM - - @pytest.mark.parametrize("mode", [pytest.param("read", id="read"), pytest.param("write", id="write")]) @pytest.mark.parametrize( ("owner", "name", "real", "before"), @@ -1060,6 +1062,13 @@ def test_shared_directory_ignores_a_journal_beside_the_database( # pragma: need assert named.read_text() == "keep" +def _hot_journal(super_journal: Path) -> bytes: # pragma: needs posix-hard-link + # A journal whose tail names a super-journal makes SQLite unlink that file once it finishes the journal: + # https://www.sqlite.org/fileformat.html#the_rollback_journal + name = os.fsencode(super_journal) + return name + struct.pack(">II", len(name), sum(name)) + bytes.fromhex("d9d505f920a163d7") + + @pytest.mark.requires_hard_links @_NEEDS_POSIX_HARD_LINK def test_shared_directory_ignores_a_directory_at_the_wal_name( # pragma: needs posix-hard-link @@ -1376,13 +1385,6 @@ def replace_then_connect( assert_read_write_lock_state(str(moved / "lock.db"), "write", available=True) -def _hot_journal(super_journal: Path) -> bytes: # pragma: needs posix-hard-link - # A journal whose tail names a super-journal makes SQLite unlink that file once it finishes the journal: - # https://www.sqlite.org/fileformat.html#the_rollback_journal - name = os.fsencode(super_journal) - return name + struct.pack(">II", len(name), sum(name)) + bytes.fromhex("d9d505f920a163d7") - - def _entries(directory: Path) -> list[tuple[str, bytes | None]]: # pragma: needs posix-hard-link return sorted((path.name, path.read_bytes() if path.is_file() else None) for path in directory.iterdir()) diff --git a/tests/test_read_write_fork.py b/tests/test_read_write_fork.py index e1afe22f..55e7e921 100644 --- a/tests/test_read_write_fork.py +++ b/tests/test_read_write_fork.py @@ -23,7 +23,7 @@ NEEDS_FORK1, ) from tests.descriptor_helpers import descriptors_on -from tests.read_write_helpers import OPPOSITE_MODE +from tests.read_write_helpers import OPPOSITE_MODE, writer_gets_in if TYPE_CHECKING: from _typeshed import StrOrBytesPath # simplint: ignore[private-import-path] # tests internal behavior @@ -103,6 +103,35 @@ def test_read_write_lock_dropped_instances_leave_no_descriptors(tmp_path: Path) assert result == (0, "", "") +def _run_fork_script(script: str, arguments: list[str], *, timeout: float) -> _ForkResult: + process = subprocess.Popen( # executes this test's fixed interpreter and source + [sys.executable, "-c", script, *arguments], + start_new_session=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + try: + process_output, process_error = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as error: + with contextlib.suppress(ProcessLookupError): + if sys.platform == "win32": # pragma: win32 cover + process.kill() + else: # pragma: win32 no cover + os.killpg(process.pid, signal.SIGKILL) + process_output, process_error = process.communicate() + msg = f"fork script exceeded {timeout} seconds: stdout={process_output!r}, stderr={process_error!r}" + raise AssertionError(msg) from error + assert process.returncode is not None # pragma: needs fork + return _ForkResult(process.returncode, process_output, process_error) # pragma: needs fork + + +class _ForkResult(NamedTuple): + returncode: int + output: str + error: str + + @_NEEDS_FD_DIRECTORY def test_read_write_lock_keeps_its_probe_descriptor_while_held(tmp_path: Path) -> None: # pragma: needs fd-directory lock_path: Final[Path] = tmp_path / "held.db" @@ -268,8 +297,7 @@ def collect_then_build() -> filelock._read_write._InodeProbes: ReadWriteLock(tmp_path / "other.db", is_singleton=False).close() finally: gc.enable() - with ReadWriteLock(tmp_path / "dropped.db", is_singleton=False).write_lock(blocking=False): - pass + assert writer_gets_in(tmp_path / "dropped.db") @NEEDS_FORK # pragma: needs fork @@ -1191,35 +1219,6 @@ async def check_child() -> None: assert result == (0, "", "") -def _run_fork_script(script: str, arguments: list[str], *, timeout: float) -> _ForkResult: - process = subprocess.Popen( # executes this test's fixed interpreter and source - [sys.executable, "-c", script, *arguments], - start_new_session=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - text=True, - ) - try: - process_output, process_error = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired as error: - with contextlib.suppress(ProcessLookupError): - if sys.platform == "win32": # pragma: win32 cover - process.kill() - else: # pragma: win32 no cover - os.killpg(process.pid, signal.SIGKILL) - process_output, process_error = process.communicate() - msg = f"fork script exceeded {timeout} seconds: stdout={process_output!r}, stderr={process_error!r}" - raise AssertionError(msg) from error - assert process.returncode is not None # pragma: needs fork - return _ForkResult(process.returncode, process_output, process_error) # pragma: needs fork - - -class _ForkResult(NamedTuple): - returncode: int - output: str - error: str - - _TRY_WRITE_SCRIPT: Final[str] = textwrap.dedent( """ import sys diff --git a/tests/test_read_write_unit.py b/tests/test_read_write_unit.py index dd0dfbec..890d59ae 100644 --- a/tests/test_read_write_unit.py +++ b/tests/test_read_write_unit.py @@ -15,7 +15,7 @@ import pytest from tests.capability_marks import NEEDS_GENERATOR_EXCEPTION_CONTEXT, NEEDS_PROMPT_FINALIZATION -from tests.read_write_helpers import ACQUIRE_SETTINGS, INVALID_TIMEOUTS +from tests.read_write_helpers import ACQUIRE_SETTINGS, INVALID_TIMEOUTS, writer_gets_in pytest.importorskip("sqlite3") @@ -154,30 +154,30 @@ def test_init_sets_attributes(lock_file: str) -> None: lock = ReadWriteLock(lock_file, timeout=5.0, blocking=False, is_singleton=False) assert (lock.lock_file, lock.timeout) == (lock_file, pytest.approx(5.0)) assert lock.blocking is False - assert lock._lock_level == 0 - assert lock._current_mode is None - assert lock._write_thread_id is None + assert lock._hold.level == 0 + assert lock._hold.mode is None + assert lock._hold.write_thread_id is None assert Path(lock_file).is_file() def test_acquire_release_read(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) proxy = lock.acquire_read() - assert (lock._lock_level, lock._current_mode) == (1, "read") + assert (lock._hold.level, lock._hold.mode) == (1, "read") lock.release() - assert lock._lock_level == 0 - assert lock._current_mode is None + assert lock._hold.level == 0 + assert lock._hold.mode is None assert isinstance(proxy, object) def test_acquire_release_write(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) lock.acquire_write() - assert (lock._lock_level, lock._current_mode, lock._write_thread_id) == (1, "write", threading.get_ident()) + assert (lock._hold.level, lock._hold.mode, lock._hold.write_thread_id) == (1, "write", threading.get_ident()) lock.release() - assert lock._lock_level == 0 - assert lock._current_mode is None - assert lock._write_thread_id is None + assert lock._hold.level == 0 + assert lock._hold.mode is None + assert lock._hold.write_thread_id is None @pytest.mark.parametrize( @@ -192,11 +192,11 @@ def test_reentrant_lock(lock_file: str, mode: Literal["read", "write"]) -> None: acquire = getattr(lock, f"acquire_{mode}") acquire() acquire() - assert lock._lock_level == 2 + assert lock._hold.level == 2 lock.release() - assert (lock._lock_level, lock._current_mode) == (1, mode) + assert (lock._hold.level, lock._hold.mode) == (1, mode) lock.release() - assert lock._lock_level == 0 + assert lock._hold.level == 0 def test_upgrade_read_to_write_prohibited(lock_file: str) -> None: @@ -222,26 +222,29 @@ def test_release_unheld_lock_raises(lock_file: str) -> None: def test_release_force_unheld_is_noop(lock_file: str) -> None: - ReadWriteLock(lock_file, is_singleton=False).release(force=True) + lock = ReadWriteLock(lock_file, is_singleton=False) + lock.release(force=True) + assert writer_gets_in(lock_file) + lock.close() def test_release_force_resets_level(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) lock.acquire_write() lock.acquire_write() - assert lock._lock_level == 2 + assert lock._hold.level == 2 lock.release(force=True) - assert lock._lock_level == 0 - assert lock._current_mode is None + assert lock._hold.level == 0 + assert lock._hold.mode is None def test_release_force_closes_lingering_connection_at_zero_level(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) lock.acquire_write() - lock._lock_level = 0 + lock._hold.level = 0 lock.release(force=True) assert lock._con is None - assert lock._lock_level == 0 + assert lock._hold.level == 0 @pytest.mark.parametrize( @@ -254,9 +257,9 @@ def test_release_force_closes_lingering_connection_at_zero_level(lock_file: str) def test_lock_context_manager(lock_file: str, mode: Literal["read", "write"]) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with getattr(lock, f"{mode}_lock")(): - assert (lock._lock_level, lock._current_mode) == (1, mode) - assert lock._lock_level == 0 - assert lock._current_mode is None + assert (lock._hold.level, lock._hold.mode) == (1, mode) + assert lock._hold.level == 0 + assert lock._hold.mode is None @pytest.mark.parametrize( @@ -269,33 +272,33 @@ def test_lock_context_manager(lock_file: str, mode: Literal["read", "write"]) -> def test_lock_uses_instance_defaults(lock_file: str, mode: Literal["read", "write"]) -> None: lock = ReadWriteLock(lock_file, timeout=3.0, blocking=True, is_singleton=False) with getattr(lock, f"{mode}_lock")(): - assert lock._current_mode == mode + assert lock._hold.mode == mode def test_read_lock_custom_timeout(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with lock.read_lock(timeout=5.0): - assert lock._current_mode == "read" + assert lock._hold.mode == "read" def test_write_lock_custom_blocking(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with lock.write_lock(blocking=True): - assert lock._current_mode == "write" + assert lock._hold.mode == "write" def test_read_lock_context_manager_overrides_defaults(lock_file: str) -> None: lock = ReadWriteLock(lock_file, timeout=10.0, blocking=False, is_singleton=False) with lock.read_lock(timeout=5.0, blocking=True): - assert lock._current_mode == "read" + assert lock._hold.mode == "read" def test_close_releases_lock_and_closes_connection(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) lock.acquire_write() - assert lock._lock_level == 1 + assert lock._hold.level == 1 lock.close() - assert lock._lock_level == 0 + assert lock._hold.level == 0 with pytest.raises(sqlite3.ProgrammingError, match="Cannot operate on a closed database"): lock.acquire_read() @@ -368,13 +371,13 @@ def test_read_lock_release_past_a_threads_own_depth_prohibited( def test_read_lock_shared_by_threads_ends_with_the_last_reader( sync_locks: tuple[ReadWriteLock, ReadWriteLock], other_thread: ThreadPoolExecutor ) -> None: - lock, contender = sync_locks + lock, _ = sync_locks lock.acquire_read() other_thread.submit(lock.acquire_read).result() lock.release() + assert not writer_gets_in(lock.lock_file) other_thread.submit(lock.release).result() - with contender.write_lock(blocking=False): - pass + assert writer_gets_in(lock.lock_file) @pytest.mark.parametrize( @@ -422,13 +425,11 @@ def test_write_lock_drop_from_different_thread_needs_an_explicit_override( ) -> None: with ( closing(ReadWriteLock(lock_file, is_singleton=False)) as lock, - closing(ReadWriteLock(lock_file, is_singleton=False)) as contender, ThreadPoolExecutor(max_workers=1) as other_thread, ): lock.acquire_write() other_thread.submit(drop, lock).result() - with contender.write_lock(blocking=False): - pass + assert writer_gets_in(lock_file) @pytest.mark.parametrize( @@ -441,9 +442,9 @@ def test_write_lock_drop_from_different_thread_needs_an_explicit_override( def test_sequential_mode_switch(lock_file: str, first: str, second: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with getattr(lock, f"{first}_lock")(): - pass + assert not writer_gets_in(lock_file) with getattr(lock, f"{second}_lock")(): - pass + assert not writer_gets_in(lock_file) def test_non_blocking_read_fails_when_write_held(lock_file: str) -> None: @@ -509,24 +510,24 @@ def hold_write() -> None: def test_nested_read_context_managers(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with lock.read_lock(): - assert lock._lock_level == 1 + assert lock._hold.level == 1 with lock.read_lock(): - assert lock._lock_level == 2 + assert lock._hold.level == 2 with lock.read_lock(): - assert lock._lock_level == 3 - assert lock._lock_level == 2 - assert lock._lock_level == 1 - assert lock._lock_level == 0 + assert lock._hold.level == 3 + assert lock._hold.level == 2 + assert lock._hold.level == 1 + assert lock._hold.level == 0 def test_nested_write_context_managers(lock_file: str) -> None: lock = ReadWriteLock(lock_file, is_singleton=False) with lock.write_lock(): - assert lock._lock_level == 1 + assert lock._hold.level == 1 with lock.write_lock(): - assert lock._lock_level == 2 - assert lock._lock_level == 1 - assert lock._lock_level == 0 + assert lock._hold.level == 2 + assert lock._hold.level == 1 + assert lock._hold.level == 0 @pytest.mark.parametrize( @@ -586,8 +587,8 @@ def test_double_check_read_reentrance(lock_file: str, mocker: MockerFixture) -> mock_lock = mocker.create_autospec(real_lock) def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: - lock._lock_level = 1 - lock._current_mode = "read" + lock._hold.level = 1 + lock._hold.mode = "read" _fake_open_transaction(lock, mocker) return real_lock.acquire(blocking, timeout) @@ -595,18 +596,18 @@ def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: mock_lock.release = real_lock.release lock._transaction_lock = mock_lock proxy = lock.acquire_read() - assert lock._lock_level == 2 + assert lock._hold.level == 2 assert proxy is not None - lock._lock_level = 0 - lock._current_mode = None - lock._con, lock._connection_transaction_released = None, True + lock._hold.level = 0 + lock._hold.mode = None + lock._owned_connection.connection, lock._owned_connection.transaction_released = None, True lock.release(force=True) def _fake_open_transaction(lock: ReadWriteLock, mocker: MockerFixture) -> None: # A level stands only while its transaction does, so a faked level needs a transaction behind it too. - lock._con = mocker.create_autospec(sqlite3.Connection, instance=True, in_transaction=True) - lock._connection_transaction_released = False + lock._owned_connection.connection = mocker.create_autospec(sqlite3.Connection, instance=True, in_transaction=True) + lock._owned_connection.transaction_released = False def test_double_check_write_reentrance(lock_file: str, mocker: MockerFixture) -> None: @@ -615,9 +616,9 @@ def test_double_check_write_reentrance(lock_file: str, mocker: MockerFixture) -> mock_lock = mocker.create_autospec(real_lock) def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: - lock._lock_level = 1 - lock._current_mode = "write" - lock._write_thread_id = threading.get_ident() + lock._hold.level = 1 + lock._hold.mode = "write" + lock._hold.write_thread_id = threading.get_ident() _fake_open_transaction(lock, mocker) return real_lock.acquire(blocking, timeout) @@ -625,12 +626,12 @@ def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: mock_lock.release = real_lock.release lock._transaction_lock = mock_lock proxy = lock.acquire_write() - assert lock._lock_level == 2 + assert lock._hold.level == 2 assert proxy is not None - lock._lock_level = 0 - lock._current_mode = None - lock._con, lock._connection_transaction_released = None, True - lock._write_thread_id = None + lock._hold.level = 0 + lock._hold.mode = None + lock._owned_connection.connection, lock._owned_connection.transaction_released = None, True + lock._hold.write_thread_id = None lock.release(force=True) @@ -653,8 +654,8 @@ def test_double_check_wrong_mode_raises( mock_lock = mocker.create_autospec(real_lock) def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: - lock._lock_level = 1 - lock._current_mode = conflicting_mode + lock._hold.level = 1 + lock._hold.mode = conflicting_mode _fake_open_transaction(lock, mocker) return real_lock.acquire(blocking, timeout) @@ -663,15 +664,15 @@ def fake_acquire(blocking: bool = True, timeout: float = -1) -> bool: lock._transaction_lock = mock_lock with pytest.raises(RuntimeError, match=match): getattr(lock, f"acquire_{acquire_mode}")() - lock._lock_level = 0 - lock._current_mode = None - lock._con, lock._connection_transaction_released = None, True + lock._hold.level = 0 + lock._hold.mode = None + lock._owned_connection.connection, lock._owned_connection.transaction_released = None, True def test_write_lock_context_manager_overrides_defaults(lock_file: str) -> None: lock = ReadWriteLock(lock_file, timeout=10.0, blocking=False, is_singleton=False) with lock.write_lock(timeout=5.0, blocking=True): - assert lock._current_mode == "write" + assert lock._hold.mode == "write" @pytest.mark.parametrize( @@ -684,16 +685,13 @@ def test_write_lock_context_manager_overrides_defaults(lock_file: str) -> None: def test_busy_timeout_recomputed_after_journal_mode( lock_file: str, mocker: MockerFixture, mode: Literal["read", "write"] ) -> None: - counter = iter([0.0, 0.0, 0.0, 0.5]) - mocker.patch( - "filelock._read_write.time", - new=SimpleNamespace( - monotonic=time.monotonic, perf_counter=mocker.create_autospec(time.perf_counter, side_effect=counter) - ), - ) + perf_counter = mocker.create_autospec(time.perf_counter, side_effect=[0.0, 0.0, 0.0, 0.5]) + mocker.patch("filelock._read_write.time", new=SimpleNamespace(monotonic=time.monotonic, perf_counter=perf_counter)) lock = ReadWriteLock(lock_file, is_singleton=False) getattr(lock, f"acquire_{mode}")(timeout=2.0) lock.release() + # the fourth reading recomputes the remaining wait after the journal mode switch + assert perf_counter.call_count == 4 def test_acquire_converts_connect_lock_error_to_timeout(lock_file: str, mocker: MockerFixture) -> None: @@ -787,8 +785,8 @@ def test_finish_connection_chains_rollback_then_close_error(lock_file: str, mock ) con.rollback.side_effect = sqlite3.OperationalError("rollback boom") con.close.side_effect = sqlite3.OperationalError("close boom") - lock._con = con - lock._connection_transaction_released = False + lock._owned_connection.connection = con + lock._owned_connection.transaction_released = False with pytest.raises(sqlite3.OperationalError, match="close boom") as excinfo: lock._finish_connection() @@ -804,7 +802,7 @@ def test_finish_connection_skips_rollback_when_already_released(lock_file: str) lock.acquire_write() con = lock._con assert con is not None - lock._connection_transaction_released = True + lock._owned_connection.transaction_released = True lock._finish_connection() @@ -852,8 +850,7 @@ def test_dropping_a_lock_closes_its_connection(lock_file: str) -> None: del lock - with ReadWriteLock(lock_file, is_singleton=False).write_lock(blocking=False): - pass + assert writer_gets_in(lock_file) @NEEDS_PROMPT_FINALIZATION diff --git a/tests/test_soft_lease.py b/tests/test_soft_lease.py index 77f22aef..28213f48 100644 --- a/tests/test_soft_lease.py +++ b/tests/test_soft_lease.py @@ -62,6 +62,21 @@ def marker(tmp_path: Path) -> Path: return tmp_path / "a.lock" +def test_lease_publishes_its_claim(marker: Path) -> None: + lease = _lease(marker) + + with lease: + owner = lease.owner + assert owner is not None + assert (owner.pid, owner.hostname, owner.mode, owner.lease_duration) == ( + os.getpid(), + host_name(), + "lease", + _DURATION, + ) + assert owner.token == lease.token + + def _lease( marker: Path, *, @@ -78,32 +93,6 @@ def _lease( ) -class _Reports: - # Keeps the compromises a lease reports and wakes the test when the first one lands, instead of polling for it. - def __init__(self) -> None: - self.seen: list[LeaseCompromise] = [] - self.landed: Final = Event() - - def __call__(self, compromise: LeaseCompromise) -> None: - self.seen.append(compromise) - self.landed.set() - - -def test_lease_publishes_its_claim(marker: Path) -> None: - lease = _lease(marker) - - with lease: - owner = lease.owner - assert owner is not None - assert (owner.pid, owner.hostname, owner.mode, owner.lease_duration) == ( - os.getpid(), - host_name(), - "lease", - _DURATION, - ) - assert owner.token == lease.token - - def test_lease_token_names_the_claim_only_while_held(marker: Path) -> None: lease = _lease(marker) @@ -174,14 +163,16 @@ def test_lease_heartbeat_keeps_a_live_claim_past_its_duration(marker: Path) -> N _lease(marker).acquire() -def test_lease_heartbeat_uses_acquisition_directory(working_directories: tuple[Path, Path]) -> None: +def test_lease_heartbeat_uses_acquisition_directory( + working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: first, second = working_directories compromised = Event() lease = SoftFileLease( "resource.lock", lease_duration=0.9, heartbeat_interval=0.05, on_compromise=lambda _: compromised.set() ) with lease: - os.chdir(second) + monkeypatch.chdir(second) os.utime(first / "resource.lock", (0, 0)) assert not compromised.wait(timeout=0.3) @@ -265,6 +256,17 @@ def test_lease_reports_compromise_when_the_marker_vanishes(marker: Path) -> None assert [(c.reason, c.token, c.lock_file) for c in reports.seen] == [("marker-missing", token, str(marker))] +class _Reports: + # Keeps the compromises a lease reports and wakes the test when the first one lands, instead of polling for it. + def __init__(self) -> None: + self.seen: list[LeaseCompromise] = [] + self.landed: Final = Event() + + def __call__(self, compromise: LeaseCompromise) -> None: + self.seen.append(compromise) + self.landed.set() + + @NEEDS_UNLINK_OPEN_FILE def test_lease_reports_compromise_when_a_peer_takes_over(marker: Path) -> None: # pragma: needs unlink-open-file reports = _Reports() @@ -343,7 +345,8 @@ def test_lease_heartbeat_tolerates_one_refresh_under_a_peers_exchange( # pragma def spy(path: str | os.PathLike[str], *, dir_fd: int | None = None) -> os.stat_result: found: Final = lstat(path, dir_fd=dir_fd) - os.fspath(path) == str(marker) and refreshed.release() + if os.fspath(path) == str(marker): # pragma: no branch # guard: only the lease refresh calls lstat here + refreshed.release() return found with lease: diff --git a/tests/test_soft_stale.py b/tests/test_soft_stale.py index 25690923..32cf6bda 100644 --- a/tests/test_soft_stale.py +++ b/tests/test_soft_stale.py @@ -99,12 +99,22 @@ def test_stale_lock_broken_when_process_dead(lock_path: Path, mocker: MockerFixt _assert_self_heals(lock_path) +def _assert_self_heals(lock_path: Path) -> None: + with held_after_reclaim(SoftFileLock(lock_path)) as lock: + assert lock.is_locked + + def test_stale_lock_not_broken_when_process_alive(lock_path: Path, mocker: MockerFixture) -> None: lock_path.write_text(_holder(os.getpid()), encoding="utf-8") mocker.patch("filelock._identity.process_alive", return_value=True, autospec=True) _assert_times_out(lock_path) +def _assert_times_out(lock_path: Path, *, timeout: float = 0.1) -> None: + with pytest.raises(TimeoutError): + SoftFileLock(lock_path, timeout=timeout).acquire() + + def test_stale_lock_not_broken_different_hostname(lock_path: Path) -> None: lock_path.write_text(_holder(_DEAD_PID, host="other-host.example.com"), encoding="utf-8") _assert_times_out(lock_path) @@ -219,6 +229,10 @@ def test_stale_break_restores_a_stranded_empty_marker( # pragma: win32 no cover assert (lock.read_bytes(), list(tmp_path.glob("test.lock.break.*"))) == (b"", []) +def _break_name(lock: Path, pid: int = _DEAD_PID) -> Path: # pragma: win32 no cover + return lock.with_name(f"test.lock.break.{pid}.{'0' * 32}") + + def test_timeout_names_an_owner_that_recorded_no_pid(lock_path: Path) -> None: lock_path.write_bytes(b"") with pytest.raises(Timeout, match="held by an owner that recorded no PID"): @@ -358,10 +372,6 @@ def peer_exchanges_then_unlink(path: Path, *, missing_ok: bool = False) -> None: assert list(lock_path.parent.glob("test.lock*")) == [] -def _break_name(lock: Path, pid: int = _DEAD_PID) -> Path: # pragma: win32 no cover - return lock.with_name(f"test.lock.break.{pid}.{'0' * 32}") - - @pytest.mark.skipif(sys.platform == "win32", reason="Windows cannot exchange two names") @STALE_BREAKERS @NEEDS_STALE_RECLAIM @@ -1191,6 +1201,15 @@ def test_stale_detection_errors_suppressed(lock_path: Path, mocker: MockerFixtur mock_read.assert_called() +@_WINDOWS_ONLY +def test_windows_stale_lock_uses_captured_process_error( # pragma: win32 cover + lock_path: Path, mocker: MockerFixture +) -> None: + _fail_open_process(mocker, _WIN_ERROR_INVALID_PARAMETER, _WIN_ERROR_ACCESS_DENIED) + lock_path.write_text(_holder(_DEAD_PID), encoding="utf-8") + _assert_self_heals(lock_path) + + def _fail_open_process(mocker: MockerFixture, captured_error: int, ambient_error: int) -> None: # pragma: win32 cover if sys.platform != "win32": # pragma: win32 cover # narrows ctypes.windll for the type checker pytest.skip("windows-only") # pragma: win32 no cover @@ -1204,15 +1223,6 @@ def fail_open_process(_access: int, _inherit_handle: bool, _pid: int) -> None: mocker.patch("filelock._identity._KERNEL32.OpenProcess", side_effect=fail_open_process, autospec=True) -@_WINDOWS_ONLY -def test_windows_stale_lock_uses_captured_process_error( # pragma: win32 cover - lock_path: Path, mocker: MockerFixture -) -> None: - _fail_open_process(mocker, _WIN_ERROR_INVALID_PARAMETER, _WIN_ERROR_ACCESS_DENIED) - lock_path.write_text(_holder(_DEAD_PID), encoding="utf-8") - _assert_self_heals(lock_path) - - @_WINDOWS_ONLY def test_windows_stale_lock_held_on_access_denied( # pragma: win32 cover lock_path: Path, mocker: MockerFixture @@ -1262,6 +1272,24 @@ def test_windows_process_probe_closes_handles(lock_path: Path) -> None: # pragm lock.release() +def _current_process_handle_count() -> int: # pragma: win32 cover + if sys.platform != "win32": # pragma: win32 cover + pytest.skip("windows-only") # pragma: no cover # only ever called on win32, so this guard never runs + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.GetCurrentProcess.argtypes = [] + kernel32.GetCurrentProcess.restype = wintypes.HANDLE + kernel32.GetProcessHandleCount.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + kernel32.GetProcessHandleCount.restype = wintypes.BOOL + count = wintypes.DWORD() + assert kernel32.GetProcessHandleCount(kernel32.GetCurrentProcess(), ctypes.byref(count)), ctypes.WinError( + ctypes.get_last_error() + ) + return count.value + + def test_pid_without_a_lock_file(lock_path: Path) -> None: assert SoftFileLock(lock_path).pid is None @@ -1346,34 +1374,6 @@ def test_write_failure_rolls_back_acquire(lock_path: Path, mocker: MockerFixture assert not lock_path.exists() -def _assert_self_heals(lock_path: Path) -> None: - with held_after_reclaim(SoftFileLock(lock_path)) as lock: - assert lock.is_locked - - -def _assert_times_out(lock_path: Path, *, timeout: float = 0.1) -> None: - with pytest.raises(TimeoutError): - SoftFileLock(lock_path, timeout=timeout).acquire() - - -def _current_process_handle_count() -> int: # pragma: win32 cover - if sys.platform != "win32": # pragma: win32 cover - pytest.skip("windows-only") # pragma: no cover # only ever called on win32, so this guard never runs - import ctypes - from ctypes import wintypes - - kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) - kernel32.GetCurrentProcess.argtypes = [] - kernel32.GetCurrentProcess.restype = wintypes.HANDLE - kernel32.GetProcessHandleCount.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] - kernel32.GetProcessHandleCount.restype = wintypes.BOOL - count = wintypes.DWORD() - assert kernel32.GetProcessHandleCount(kernel32.GetCurrentProcess(), ctypes.byref(count)), ctypes.WinError( - ctypes.get_last_error() - ) - return count.value - - def _hold_reports_foreign_identity(mocker: MockerFixture) -> None: # Make the held descriptor report a different inode than the file on disk, as if a peer replaced the marker at the # path. fstat runs only on our own lock fd, so nothing else in acquire or release is disturbed. @@ -1597,14 +1597,6 @@ def test_close_error_policy_cleans_marker(lock_path: Path, mocker: MockerFixture assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) -def test_close_error_suppress_policy_cleans_marker(lock_path: Path, mocker: MockerFixture) -> None: - lock = SoftFileLock(lock_path, close_error_policy="suppress") - lock.acquire() - with _close_after_commit(mocker, lock) as (_, attempts): - lock.release() - assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) - - @contextmanager def _close_after_commit(mocker: MockerFixture, lock: SoftFileLock) -> Generator[tuple[OSError, list[int]]]: real_close = os.close @@ -1637,6 +1629,14 @@ def close(fd: int) -> None: mocker.stop(close_mock) +def test_close_error_suppress_policy_cleans_marker(lock_path: Path, mocker: MockerFixture) -> None: + lock = SoftFileLock(lock_path, close_error_policy="suppress") + lock.acquire() + with _close_after_commit(mocker, lock) as (_, attempts): + lock.release() + assert (len(attempts), lock.is_locked, lock.lock_counter, lock_path.exists()) == (1, False, 0, False) + + def test_close_error_suppression_releases_nested_acquisitions_once(lock_path: Path, mocker: MockerFixture) -> None: lock = SoftFileLock(lock_path, close_error_policy="suppress") lock.acquire() diff --git a/tests/test_strict_soft_failures.py b/tests/test_strict_soft_failures.py index 714dd90d..38509ed3 100644 --- a/tests/test_strict_soft_failures.py +++ b/tests/test_strict_soft_failures.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import gc import os import pickle # ruff:ignore[suspicious-pickle-import] # round-trip uses bytes produced in this test @@ -295,6 +296,36 @@ def test_strict_soft_reaper_removes_private_before_publication(tmp_path: Path, m assert (lock.claims, lock.is_locked, private_path.exists()) == ((), False, False) +def _reap_before_publication(lock_path: Path, token: str, mocker: MockerFixture) -> None: # pragma: needs dir-fd + real_link: Final = os.link + peer_scanned = False + + def reap_before_link( + source: _PathValue, + destination: _PathValue, + *, + src_dir_fd: int | None = None, + dst_dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> None: + nonlocal peer_scanned + # A peer's scan ages out the private record before this publisher links it; the peer's own links pass. + if Path(os.fsdecode(destination)).name.startswith("intent-") and not peer_scanned: + peer_scanned = True + os.utime(_private_claim_path(lock_path, token), (0, 0)) + with StrictSoftFileLock(lock_path, timeout=0): + pass + real_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=follow_symlinks) + + real_token_hex: Final = secrets.token_hex + mocker.patch( + "filelock._strict.secrets.token_hex", + autospec=True, + side_effect=lambda nbytes: real_token_hex(nbytes) if peer_scanned else token, + ) + mocker.patch("filelock._strict.os.link", autospec=True, side_effect=reap_before_link) + + def test_strict_soft_reaper_removes_private_after_publication(tmp_path: Path, mocker: MockerFixture) -> None: lock_path = tmp_path / "resource.lock" _initialize_protocol(lock_path) @@ -370,36 +401,6 @@ def replace_instead_of_unlink(path: _PathValue, *, dir_fd: int | None = None) -> ) -def _reap_before_publication(lock_path: Path, token: str, mocker: MockerFixture) -> None: # pragma: needs dir-fd - real_link: Final = os.link - peer_scanned = False - - def reap_before_link( - source: _PathValue, - destination: _PathValue, - *, - src_dir_fd: int | None = None, - dst_dir_fd: int | None = None, - follow_symlinks: bool = True, - ) -> None: - nonlocal peer_scanned - # A peer's scan ages out the private record before this publisher links it; the peer's own links pass. - if Path(os.fsdecode(destination)).name.startswith("intent-") and not peer_scanned: - peer_scanned = True - os.utime(_private_claim_path(lock_path, token), (0, 0)) - with StrictSoftFileLock(lock_path, timeout=0): - pass - real_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=follow_symlinks) - - real_token_hex: Final = secrets.token_hex - mocker.patch( - "filelock._strict.secrets.token_hex", - autospec=True, - side_effect=lambda nbytes: real_token_hex(nbytes) if peer_scanned else token, - ) - mocker.patch("filelock._strict.os.link", autospec=True, side_effect=reap_before_link) - - def test_strict_soft_private_close_failure_removes_public_claim(tmp_path: Path, mocker: MockerFixture) -> None: lock_path = tmp_path / "resource.lock" _initialize_protocol(lock_path) @@ -481,6 +482,34 @@ def test_strict_soft_held_directory_close_failure_removes_both_claims(tmp_path: assert (lock.claims, lock.is_locked) == ((), False) +def _fail_directory_close_after( # pragma: needs dir-fd + mocker: MockerFixture, function: str, prefix: str, message: str +) -> None: + # Every claim operation walks to its directory through descriptors it closes on the way, so fail the first + # directory close after os.link or os.unlink touched the name rather than count the directory closes. Where os.link + # takes no dir_fd it gets full paths, so match the final component. + real_close: Final = os.close + real_fstat: Final = os.fstat + real_function: Final = getattr(os, function) + armed = False + + def arm_after(*paths: _PathValue, **options: int | bool | None) -> None: + nonlocal armed + real_function(*paths, **options) + armed = armed or Path(os.fsdecode(paths[-1])).name.startswith(prefix) + + def fail_directory_close(fd: int) -> None: + nonlocal armed + is_directory = stat.S_ISDIR(real_fstat(fd).st_mode) + real_close(fd) + if armed and is_directory: + armed = False + raise OSError(EIO, message) + + mocker.patch(f"filelock._strict.os.{function}", side_effect=arm_after, autospec=True) + mocker.patch("filelock._strict.os.close", side_effect=fail_directory_close, autospec=True) + + def _stage_claim_record(lock_path: Path) -> Path: _initialize_protocol(lock_path) return _private_claim_path(lock_path, "0" * 32) @@ -551,6 +580,16 @@ def deny_abandoned_record(path: _PathValue, *, dir_fd: int | None = None) -> Non assert (reaper_unlinks, sleep.call_count, private_path.exists()) == (1, 0, True) +def _patch_strict_clock( + mocker: MockerFixture, + *, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + wall: Callable[[], float] = time.time, +) -> None: + mocker.patch("filelock._strict.time", new=SimpleNamespace(monotonic=monotonic, sleep=sleep, time=wall)) + + @pytest.mark.parametrize( "stage_record", [pytest.param(_stage_claim_record, id="claim"), pytest.param(_stage_sentinel_record, id="sentinel")], @@ -808,6 +847,12 @@ def fail_cleanup_directory_close(fd: int) -> None: assert len(lock.claims) == 2 +def _leaf_error_messages(error: BaseException) -> list[str]: # pragma: needs dir-fd + if isinstance(error, BaseExceptionGroup): + return [message for child in error.exceptions for message in _leaf_error_messages(child)] + return [str(error)] + + def test_strict_soft_sentinel_publication_race_acquires_over_strict_winner( tmp_path: Path, mocker: MockerFixture ) -> None: @@ -1228,13 +1273,6 @@ def replace_before_open( StrictSoftFileLock(lock_path).acquire() -def _write_held_claim(claims: Path) -> Path: - claims.mkdir(parents=True) - claim = claims / f"held-v1-{'0' * 32}.claim" - claim.write_bytes(b"filelock-strict-v1\n" + b"0" * 32 + b"\n1\n686f7374\n\n") - return claim - - def test_strict_soft_stale_claim_read_is_skipped(tmp_path: Path, mocker: MockerFixture) -> None: # An NFS peer that unlinks its own claim leaves this client's cached filehandle stale, so the open returns ESTALE # rather than ENOENT. A stale handle that outlives revalidation is a vanished claim, not an unreadable one. @@ -1246,6 +1284,13 @@ def test_strict_soft_stale_claim_read_is_skipped(tmp_path: Path, mocker: MockerF assert StrictSoftFileLock(lock_path).claims == () +def _write_held_claim(claims: Path) -> Path: + claims.mkdir(parents=True) + claim = claims / f"held-v1-{'0' * 32}.claim" + claim.write_bytes(b"filelock-strict-v1\n" + b"0" * 32 + b"\n1\n686f7374\n\n") + return claim + + def _open_raising_for(claim: Path, error: OSError, mocker: MockerFixture) -> MockType: real_open = os.open @@ -1511,40 +1556,6 @@ def _private_claim_path(lock_path: Path, token: str) -> Path: return Path(f"{lock_path}.filelock") / "claims" / f".intent-v1-{token}.claim.private-v1-{token}.tmp" -def _fail_directory_close_after( # pragma: needs dir-fd - mocker: MockerFixture, function: str, prefix: str, message: str -) -> None: - # Every claim operation walks to its directory through descriptors it closes on the way, so fail the first - # directory close after os.link or os.unlink touched the name rather than count the directory closes. Where os.link - # takes no dir_fd it gets full paths, so match the final component. - real_close: Final = os.close - real_fstat: Final = os.fstat - real_function: Final = getattr(os, function) - armed = False - - def arm_after(*paths: _PathValue, **options: int | bool | None) -> None: - nonlocal armed - real_function(*paths, **options) - armed = armed or Path(os.fsdecode(paths[-1])).name.startswith(prefix) - - def fail_directory_close(fd: int) -> None: - nonlocal armed - is_directory = stat.S_ISDIR(real_fstat(fd).st_mode) - real_close(fd) - if armed and is_directory: - armed = False - raise OSError(EIO, message) - - mocker.patch(f"filelock._strict.os.{function}", side_effect=arm_after, autospec=True) - mocker.patch("filelock._strict.os.close", side_effect=fail_directory_close, autospec=True) - - -def _leaf_error_messages(error: BaseException) -> list[str]: # pragma: needs dir-fd - if isinstance(error, BaseExceptionGroup): - return [message for child in error.exceptions for message in _leaf_error_messages(child)] - return [str(error)] - - @pytest.mark.parametrize( "error", [ @@ -1950,6 +1961,17 @@ def fail_held_link(source: _PathValue, destination: _PathValue, **options: int | assert (descriptors_on(lock_path.stat()), lock.is_locked) == (0, False) +def _deny_claim_unlinks(mocker: MockerFixture) -> None: # pragma: needs fd-directory + real_unlink: Final = os.unlink + + def deny(path: _PathValue, *, dir_fd: int | None = None) -> None: + if os.fsdecode(path).endswith(".claim"): + raise PermissionError(EACCES, "claim unlink denied") + real_unlink(path, dir_fd=dir_fd) + + mocker.patch("filelock._strict.os.unlink", autospec=True, side_effect=deny) + + @_NEEDS_FD_DIRECTORY # pragma: needs fd-directory @NEEDS_COLLECTED_FINALIZATION @pytest.mark.parametrize("thread_local", [pytest.param(True, id="thread-local"), pytest.param(False, id="shared")]) @@ -1966,17 +1988,6 @@ def test_strict_soft_dropped_hold_with_stuck_claims_closes_its_descriptor( # pr assert descriptors_on(lock_path.stat()) == 0 -def _deny_claim_unlinks(mocker: MockerFixture) -> None: # pragma: needs fd-directory - real_unlink: Final = os.unlink - - def deny(path: _PathValue, *, dir_fd: int | None = None) -> None: - if os.fsdecode(path).endswith(".claim"): - raise PermissionError(EACCES, "claim unlink denied") - real_unlink(path, dir_fd=dir_fd) - - mocker.patch("filelock._strict.os.unlink", autospec=True, side_effect=deny) - - def test_strict_soft_acquire_leaves_a_live_threads_pending_doorway_alone( lock_path: Path, mocker: MockerFixture ) -> None: @@ -2146,6 +2157,7 @@ def test_strict_soft_claim_removed_between_inspection_and_open_is_skipped( claim: Final = _write_held_claim(Path(f"{tmp_path / 'resource.lock'}.filelock") / "claims") def remove_after_inspection(inspect: _Inspect) -> _Inspect: + @functools.wraps(inspect) def inspect_then_remove(path: _PathValue, **options: int | bool | None) -> os.stat_result: found = inspect(path, **options) # Windows inspects nothing else in this scan; with dir_fd the claim directory is stat-ed too. @@ -2214,6 +2226,7 @@ def test_strict_soft_coordination_directory_swapped_for_a_junction_leaves_target swapped: list[bool] = [] def swap_after_the_inspection(inspect: _Inspect) -> _Inspect: + @functools.wraps(inspect) def inspect_then_swap(path: _PathValue, **options: int | bool | None) -> os.stat_result: # The name was a real directory at inspection; by the time the lock creates claims in it, it is a junction. found = inspect(path, **options) @@ -2435,13 +2448,3 @@ def deny_once(path: _PathValue, *, dir_fd: int | None = None) -> None: lock.release() assert (len(denied), sleep.call_args_list, lock.claims) == (1, [mocker.call(0.001)], ()) - - -def _patch_strict_clock( - mocker: MockerFixture, - *, - monotonic: Callable[[], float] = time.monotonic, - sleep: Callable[[float], None] = time.sleep, - wall: Callable[[], float] = time.time, -) -> None: - mocker.patch("filelock._strict.time", new=SimpleNamespace(monotonic=monotonic, sleep=sleep, time=wall)) diff --git a/tests/test_strict_soft_paths.py b/tests/test_strict_soft_paths.py index 7be21694..6db139f1 100644 --- a/tests/test_strict_soft_paths.py +++ b/tests/test_strict_soft_paths.py @@ -24,13 +24,14 @@ def test_strict_soft_relative_release_uses_acquisition_directory( working_directories: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, ) -> None: first, second = working_directories lock = StrictSoftFileLock("resource.lock") lock.acquire() assert tuple(claim.state for claim in StrictSoftFileLock(first / "resource.lock").claims) == ("held", "intent") - os.chdir(second) + monkeypatch.chdir(second) lock.release() assert ( @@ -42,13 +43,14 @@ def test_strict_soft_relative_release_uses_acquisition_directory( @pytest.mark.asyncio async def test_async_strict_soft_relative_release_uses_acquisition_directory( working_directories: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, ) -> None: first, second = working_directories lock = AsyncStrictSoftFileLock("resource.lock", run_in_executor=False) await lock.acquire() assert tuple(claim.state for claim in StrictSoftFileLock(first / "resource.lock").claims) == ("held", "intent") - os.chdir(second) + monkeypatch.chdir(second) await lock.release() assert ( diff --git a/tests/test_strict_soft_stress.py b/tests/test_strict_soft_stress.py index 3d0d2039..f72d7357 100644 --- a/tests/test_strict_soft_stress.py +++ b/tests/test_strict_soft_stress.py @@ -81,7 +81,14 @@ def test_strict_soft_eight_process_contention_has_no_overlap(tmp_path: Path) -> try: outputs.extend(process.communicate(timeout=max(0.1, deadline - time.monotonic())) for process in processes) except subprocess.TimeoutExpired: # pragma: no cover - test failure cleanup - _kill_processes(processes) + for process in processes: + if process.poll() is None: + if sys.platform == "win32": + process.kill() + else: + os.killpg(process.pid, signal.SIGKILL) + for process in processes: + process.communicate() pytest.fail(f"strict soft-lock stress workers exceeded {_WORKER_DEADLINE} seconds") assert counter_path.exists(), outputs @@ -90,14 +97,3 @@ def test_strict_soft_eight_process_contention_has_no_overlap(tmp_path: Path) -> [0] * process_count, process_count * acquisitions_per_process, ), outputs - - -def _kill_processes(processes: list[subprocess.Popen[str]]) -> None: # pragma: no cover - test failure cleanup - for process in processes: - if process.poll() is None: - if sys.platform == "win32": - process.kill() - else: - os.killpg(process.pid, signal.SIGKILL) - for process in processes: - process.communicate() diff --git a/tests/test_unix_fallback.py b/tests/test_unix_fallback.py index bef75347..02d47f3c 100644 --- a/tests/test_unix_fallback.py +++ b/tests/test_unix_fallback.py @@ -258,7 +258,7 @@ def test_lock_file_it_cannot_write_stays_denied_where_flock_works(tmp_path: Path @NEEDS_FCNTL def test_denied_lock_file_is_reopened_in_the_acquisition_directory( # pragma: needs fcntl - working_directories: tuple[Path, Path], mocker: MockerFixture + working_directories: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture ) -> None: # Resolved against the working directory of the moment, a chdir between the refused O_RDWR open and the read-only # reopen probes a file in another directory, or none. @@ -270,7 +270,7 @@ def test_denied_lock_file_is_reopened_in_the_acquisition_directory( # pragma: n def deny_write_then_chdir(path: str, flags: int, mode: int = 0o777, *, dir_fd: int | None = None) -> int: if not flags & os.O_CREAT and "test.lock" in path: if flags & os.O_RDWR: - os.chdir(second) + monkeypatch.chdir(second) raise PermissionError(EACCES, "Permission denied", path) read_only_opens.append(path) return real_open(path, flags, mode, dir_fd=dir_fd) diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 0a5f451d..0e128693 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -16,3 +16,4 @@ ) def test_virtualenv(tmp_path: Path) -> None: cli_run([str(tmp_path), "--no-pip", "--no-setuptools", "--no-periodic-update"]) + assert (tmp_path / "pyvenv.cfg").is_file()