diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2175d9..486e2f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: priority: 3 args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - repo: https://github.com/gaborbernat/simplint-pre-commit - rev: c8719dca99c13381570bc69f8955c9448fc271b7 # frozen: v0.7.1 + rev: 073bd9250c7a4d15b71eef71f2c95e4af5c76283 # frozen: v0.10.3 hooks: - id: simplint priority: 4 diff --git a/changelog.d/+startup-failure.bugfix.md b/changelog.d/+startup-failure.bugfix.md new file mode 100644 index 0000000..961a3b8 --- /dev/null +++ b/changelog.d/+startup-failure.bugfix.md @@ -0,0 +1 @@ +Report an error when devpi exits before it is ready, and clean up the child process if startup fails. diff --git a/pyproject.toml b/pyproject.toml index 3b6f151..0d872d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,29 +130,23 @@ lint.select = [ "ALL", ] lint.ignore = [ - "any-type", # allow Any as type annotation "CPY", # No copyright statements "DOC", # not supported - "hardcoded-bind-all-interfaces", # Possible binding to all interface - "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "incorrect-blank-line-before-class", # Conflicts with `blank-line-before-class` "missing-trailing-comma", # Conflict with formatter - "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible - "non-empty-init-module", # `__init__` module should only contain docstrings and re-exports + "multi-line-summary-first-line", # Conflicts with `multi-line-summary-second-line` "single-line-implicit-string-concatenation", # Conflict with formatter - "undocumented-magic-method", # undocumented-public-api (simplint SMP903) covers the public API instead - "undocumented-public-class", # undocumented-public-api (simplint SMP903) covers the public API instead - "undocumented-public-function", # undocumented-public-api (simplint SMP903) covers the public API instead - "undocumented-public-init", # undocumented-public-api (simplint SMP903) covers the public API instead - "undocumented-public-method", # undocumented-public-api (simplint SMP903) covers the public API instead - "undocumented-public-nested-class", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-magic-method", # function-docstring (simplint) owns internal docstrings + "undocumented-public-class", # undocumented-public-api (simplint) owns the public API + "undocumented-public-function", # undocumented-public-api (simplint) owns the public API + "undocumented-public-init", # undocumented-public-api (simplint) owns the public API + "undocumented-public-method", # undocumented-public-api (simplint) owns the public API + "undocumented-public-nested-class", # undocumented-public-api (simplint) owns the public API ] lint.per-file-ignores."tests/**/*.py" = [ - "assert", # asserts allowed in tests - "D", # don't care about documentation in tests - "FBT", # don't care about booleans as positional arguments in tests - "implicit-namespace-package", # no implicit namespace - "magic-value-comparison", # Magic value used in comparison, consider replacing with a constant variable - "subprocess-without-shell-equals-true", # `subprocess` call: check for execution of untrusted input + "assert", # asserts allowed in tests + "D", # test names carry the intent + "implicit-namespace-package", # test directories are not packages ] lint.isort = { known-first-party = [ "devpi_process", diff --git a/src/devpi_process/_command.py b/src/devpi_process/_command.py new file mode 100644 index 0000000..44dbeac --- /dev/null +++ b/src/devpi_process/_command.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from subprocess import run # ruff:ignore[suspicious-subprocess-import] command uses an argument list + + +def check_call(cmd: list[str]) -> None: + run(cmd, check=True, capture_output=True) # ruff:ignore[subprocess-without-shell-equals-true] argument list + + +__all__ = [ + "check_call", +] diff --git a/src/devpi_process/_index.py b/src/devpi_process/_index.py index b7580b1..3c08df0 100644 --- a/src/devpi_process/_index.py +++ b/src/devpi_process/_index.py @@ -1,10 +1,8 @@ from __future__ import annotations -from subprocess import run # ruff:ignore[suspicious-subprocess-import] # no shell, argument lists only -from typing import TYPE_CHECKING +from pathlib import Path # ruff:ignore[typing-only-standard-library-import] public annotation supports get_type_hints -if TYPE_CHECKING: - from pathlib import Path +from ._command import check_call class Index: @@ -35,7 +33,7 @@ def url(self) -> str: def use(self) -> None: """Use this index server.""" - _check_call([*self._client_cmd_base, "use", f"{self.user}/{self.name}"]) + check_call([*self._client_cmd_base, "use", f"{self.user}/{self.name}"]) def upload(self, *files: Path) -> None: """ @@ -43,11 +41,7 @@ def upload(self, *files: Path) -> None: :param files: the files to upload """ - _check_call(self._client_cmd_base + ["upload", "--index", self.name] + [str(i) for i in files]) - - -def _check_call(cmd: list[str]) -> None: - run(cmd, check=True, capture_output=True) # ruff:ignore[subprocess-without-shell-equals-true] # no shell, argument list only + check_call(self._client_cmd_base + ["upload", "--index", self.name] + [str(i) for i in files]) __all__ = [ diff --git a/src/devpi_process/_server.py b/src/devpi_process/_server.py index c796420..fbdd0ed 100644 --- a/src/devpi_process/_server.py +++ b/src/devpi_process/_server.py @@ -1,17 +1,19 @@ from __future__ import annotations -import random +import secrets import socket import string import sys import sysconfig from contextlib import closing +from logging import getLogger from pathlib import Path from subprocess import PIPE, Popen # ruff:ignore[suspicious-subprocess-import] # no shell, argument lists only from threading import Thread from typing import IO, TYPE_CHECKING, cast -from ._index import Index, _check_call +from ._command import check_call +from ._index import Index if TYPE_CHECKING: from collections.abc import Iterator, Sequence @@ -30,7 +32,8 @@ class IndexServer: def __init__( self, path: Path, - with_root_pypi: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public parameter, changing it breaks callers + # Preserve the public positional argument. + with_root_pypi: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] start_args: Sequence[str] | None = None, ) -> None: """ @@ -46,14 +49,14 @@ def __init__( self.host = "localhost" with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as socket_handler: - socket_handler.bind(("", 0)) + socket_handler.bind(("", 0)) # devpi can bind to any IPv4 interface via start_args self.port = cast("int", socket_handler.getsockname()[1]) - self._passwd = "".join(random.choices(string.ascii_letters, k=8)) # ruff:ignore[suspicious-non-cryptographic-random-usage] # throwaway password of a test server + self._passwd = "".join(secrets.choice(string.ascii_letters) for _ in range(8)) scripts_dir = sysconfig.get_path("scripts") if scripts_dir is None: - msg = "could not get scripts folder of host interpreter" # pragma: no cover # typing guard - raise RuntimeError(msg) # pragma: no cover # typing guard + msg = "could not get scripts folder of host interpreter" + raise RuntimeError(msg) def _exe(name: str) -> str: return str(Path(scripts_dir) / f"{name}{'.exe' if sys.platform == 'win32' else ''}") @@ -71,8 +74,12 @@ def _exe(name: str) -> str: def __enter__(self) -> Self: """Start the index server.""" - self._create_and_start_server() - self._setup_client() + try: + self._create_and_start_server() + self._setup_client() + except BaseException: + self.__exit__(None, None, None) + raise return self def _create_and_start_server(self) -> None: @@ -83,13 +90,13 @@ def _create_and_start_server(self) -> None: cmd.extend(("--role", "standalone", "--root-passwd", self._passwd)) if self._with_root_pypi is False: cmd.append("--no-root-pypi") - _check_call(cmd) + check_call(cmd) # 2. start the server cmd = [self._server, "--serverdir", server_at, "--port", str(self.port)] cmd.extend(self._start_args) self._process = Popen(cmd, stdout=PIPE, universal_newlines=True) # ruff:ignore[subprocess-without-shell-equals-true] # no shell, argument list only stdout = self._drain_stdout() - for line in stdout: # pragma: no branch # will always loop at least once + for line in stdout: if "serving at url" in line: def _keep_draining() -> None: @@ -100,21 +107,24 @@ def _keep_draining() -> None: self._stdout_drain = Thread(target=_keep_draining, name="tox-test-stdout-drain") self._stdout_drain.start() break + else: + msg = f"devpi server at {self._server_dir} exited before it was ready" + raise RuntimeError(msg) def _drain_stdout(self) -> Iterator[str]: process = cast("Popen[str]", self._process) stdout = cast("IO[str]", process.stdout) while True: - if process.poll() is not None: # pragma: no cover # a dead server is a failure path the tests do not drive - print(f"devpi server with pid {process.pid} at {self._server_dir} died") # ruff:ignore[print] # reports the dead server in the test output + if process.poll() is not None: + getLogger(__name__).error("devpi server with pid %s at %s died", process.pid, self._server_dir) break yield stdout.readline() def _setup_client(self) -> None: self._client_dir.mkdir(exist_ok=True) base = ["--clientdir", str(self._client_dir)] - _check_call([self._client, "use", *base, self.url]) - _check_call([self._client, "login", *base, self.user, "--password", self._passwd]) + check_call([self._client, "use", *base, self.url]) + check_call([self._client, "login", *base, self.user, "--password", self._passwd]) @property def url(self) -> str: @@ -139,9 +149,9 @@ def __exit__( :param exc_val: the exception leaving the block, or None :param exc_tb: traceback of that exception, or None """ - if self._process is not None: # pragma: no cover # defend against devpi startup fail + if self._process is not None: self._process.terminate() - if self._stdout_drain is not None and self._stdout_drain.is_alive(): # pragma: no cover # devpi startup fail + if self._stdout_drain is not None and self._stdout_drain.is_alive(): self._stdout_drain.join() def __repr__(self) -> str: @@ -157,11 +167,11 @@ def create_index(self, name: str, *args: str) -> Index: :returns: the created index :raises ValueError: when an index with this name already exists """ - if name in self._indexes: # pragma: no cover # a duplicate name is a caller error the tests do not drive + if name in self._indexes: msg = f"index {name} already exists" raise ValueError(msg) base = [self._client, "--clientdir", str(self._client_dir)] - _check_call([*base, "index", "-c", name, *args]) + check_call([*base, "index", "-c", name, *args]) index = Index(f"{self.url}/{self.user}", name, self.user, base) self._indexes[name] = index return index diff --git a/tests/demo_pkg_inline/build.py b/tests/demo_pkg_inline/build.py index 30a145e..6741161 100644 --- a/tests/demo_pkg_inline/build.py +++ b/tests/demo_pkg_inline/build.py @@ -51,25 +51,24 @@ def build_wheel( wheel_directory: str, - metadata_directory: str | None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument - config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument + metadata_directory: str | None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature + config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature ) -> str: base_name = f"{_NAME}-{_VERSION}-py{sys.version_info[0]}-none-any.whl" with ZipFile(str(Path(wheel_directory) / base_name), "w") as zip_file_handler: - for arc_name, data in _CONTENT.items(): # pragma: no branch # the content table always has entries + for arc_name, data in _CONTENT.items(): # pragma: no branch # _CONTENT is nonempty zip_file_handler.writestr(arc_name, dedent(data).strip()) return base_name -def get_requires_for_build_wheel( - config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument -) -> list[str]: - return [] # pragma: no cover # only executed in non-host pythons +# PEP 517 requires the unused argument. +def get_requires_for_build_wheel(config_settings: None = None) -> list[str]: # ruff:ignore[unused-function-argument] + return [] def build_sdist( sdist_directory: str, - config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument + config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature ) -> str: result = f"{_NAME}-{_VERSION}.tar.gz" with tarfile.open(str(Path(sdist_directory) / result), "w:gz") as tar: @@ -85,7 +84,6 @@ def build_sdist( return result -def get_requires_for_build_sdist( - config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument -) -> list[str]: - return [] # pragma: no cover # only executed in non-host pythons +# PEP 517 requires the unused argument. +def get_requires_for_build_sdist(config_settings: None = None) -> list[str]: # ruff:ignore[unused-function-argument] + return [] diff --git a/tests/test_devpi_process.py b/tests/test_devpi_process.py index 962f831..6b23853 100644 --- a/tests/test_devpi_process.py +++ b/tests/test_devpi_process.py @@ -1,34 +1,50 @@ from __future__ import annotations +import socket +import sysconfig +from contextlib import ExitStack +from http import HTTPStatus from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, get_type_hints import pytest from httpx import get -from devpi_process import IndexServer +from devpi_process import Index, IndexServer, __version__ if TYPE_CHECKING: - from _pytest.tmpdir import TempPathFactory + from types import ModuleType def test_version() -> None: - import devpi_process # ruff:ignore[import-outside-top-level] # the test checks the package import itself + assert __version__ is not None - assert devpi_process.__version__ is not None + +def test_index_upload_type_hints() -> None: + assert get_type_hints(Index.upload)["files"] is Path @pytest.fixture(scope="session") -def demo_artifacts(tmp_path_factory: TempPathFactory) -> tuple[Path, Path]: +def demo_backend() -> ModuleType: spec = spec_from_file_location("demo_pkg_inline.build", str(Path(__file__).parent / "demo_pkg_inline" / "build.py")) assert spec is not None build = module_from_spec(spec) loader = spec.loader assert loader is not None loader.exec_module(build) + return build + + +@pytest.fixture(scope="session") +def demo_artifacts(tmp_path_factory: pytest.TempPathFactory, demo_backend: ModuleType) -> tuple[Path, Path]: base = tmp_path_factory.mktemp("wheel") - return base / build.build_wheel(str(base)), base / build.build_sdist(str(base)) + return base / demo_backend.build_wheel(str(base)), base / demo_backend.build_sdist(str(base)) + + +def test_demo_backend_requires_no_extra_build_dependencies(demo_backend: ModuleType) -> None: + assert demo_backend.get_requires_for_build_wheel() == [] + assert demo_backend.get_requires_for_build_sdist() == [] def test_create_server(tmp_path: Path, demo_artifacts: tuple[Path, Path]) -> None: @@ -65,8 +81,48 @@ def test_create_server_with_pypi(tmp_path: Path) -> None: def test_create_server_start_args(tmp_path: Path) -> None: - with IndexServer(tmp_path, start_args=["--offline-mode"]) as server: - assert server._process is not None # ruff:ignore[private-member-access] # the test inspects the spawned command - args = server._process.args # ruff:ignore[private-member-access] # the test inspects the spawned command - assert isinstance(args, list) - assert args[-1] == "--offline-mode" + with IndexServer(tmp_path, start_args=["--host", "127.0.0.1"]) as server: + response = get(f"{server.url}/+status", headers={"Accept": "application/json"}) + assert response.status_code == HTTPStatus.OK + assert response.json()["result"]["host"] == "127.0.0.1" + + +def test_server_probes_all_ipv4_interfaces(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + bound_addresses: list[socket._Address] = [] + + class ProbeSocket(socket.socket): + def bind(self, address: socket._Address) -> None: + bound_addresses.append(address) + super().bind(address) + + monkeypatch.setattr(socket, "socket", ProbeSocket) + + IndexServer(tmp_path) + + assert bound_addresses == [("", 0)] + + +def test_create_server_rejects_duplicate_index(tmp_path: Path) -> None: + with IndexServer(tmp_path) as server: + server.create_index("base") + with pytest.raises(ValueError, match="index base already exists"): + server.create_index("base") + + +def test_create_server_rejects_missing_scripts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sysconfig, "get_path", lambda _name: None) + + with pytest.raises(RuntimeError, match="could not get scripts folder of host interpreter"): + IndexServer(tmp_path) + + +def test_create_server_reports_missing_executable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sysconfig, "get_path", lambda _name: str(tmp_path / "missing")) + + with ExitStack() as stack, pytest.raises(FileNotFoundError, match="devpi-init"): + stack.enter_context(IndexServer(tmp_path)) + + +def test_create_server_reports_startup_failure(tmp_path: Path) -> None: + with ExitStack() as stack, pytest.raises(RuntimeError, match="exited before it was ready"): + stack.enter_context(IndexServer(tmp_path, start_args=["--definitely-invalid"]))