Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ repos:
priority: 3
args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"]
- repo: https://github.com/gaborbernat/simplint-pre-commit
rev: c8719dca99c13381570bc69f8955c9448fc271b7 # frozen: v0.7.1
rev: 073bd9250c7a4d15b71eef71f2c95e4af5c76283 # frozen: v0.10.3
hooks:
- id: simplint
priority: 4
Expand Down
1 change: 1 addition & 0 deletions changelog.d/+startup-failure.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Report an error when devpi exits before it is ready, and clean up the child process if startup fails.
28 changes: 11 additions & 17 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -130,29 +130,23 @@ lint.select = [
"ALL",
]
lint.ignore = [
"any-type", # allow Any as type annotation
"CPY", # No copyright statements
"DOC", # not supported
"hardcoded-bind-all-interfaces", # Possible binding to all interface
"incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible
"incorrect-blank-line-before-class", # Conflicts with `blank-line-before-class`
"missing-trailing-comma", # Conflict with formatter
"multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible
"non-empty-init-module", # `__init__` module should only contain docstrings and re-exports
"multi-line-summary-first-line", # Conflicts with `multi-line-summary-second-line`
"single-line-implicit-string-concatenation", # Conflict with formatter
"undocumented-magic-method", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-public-class", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-public-function", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-public-init", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-public-method", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-public-nested-class", # undocumented-public-api (simplint SMP903) covers the public API instead
"undocumented-magic-method", # function-docstring (simplint) owns internal docstrings
"undocumented-public-class", # undocumented-public-api (simplint) owns the public API
"undocumented-public-function", # undocumented-public-api (simplint) owns the public API
"undocumented-public-init", # undocumented-public-api (simplint) owns the public API
"undocumented-public-method", # undocumented-public-api (simplint) owns the public API
"undocumented-public-nested-class", # undocumented-public-api (simplint) owns the public API
]
lint.per-file-ignores."tests/**/*.py" = [
"assert", # asserts allowed in tests
"D", # don't care about documentation in tests
"FBT", # don't care about booleans as positional arguments in tests
"implicit-namespace-package", # no implicit namespace
"magic-value-comparison", # Magic value used in comparison, consider replacing with a constant variable
"subprocess-without-shell-equals-true", # `subprocess` call: check for execution of untrusted input
"assert", # asserts allowed in tests
"D", # test names carry the intent
"implicit-namespace-package", # test directories are not packages
]
lint.isort = { known-first-party = [
"devpi_process",
Expand Down
12 changes: 12 additions & 0 deletions src/devpi_process/_command.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from __future__ import annotations

from subprocess import run # ruff:ignore[suspicious-subprocess-import] command uses an argument list


def check_call(cmd: list[str]) -> None:
run(cmd, check=True, capture_output=True) # ruff:ignore[subprocess-without-shell-equals-true] argument list


__all__ = [
"check_call",
]
14 changes: 4 additions & 10 deletions src/devpi_process/_index.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
from __future__ import annotations

from subprocess import run # ruff:ignore[suspicious-subprocess-import] # no shell, argument lists only
from typing import TYPE_CHECKING
from pathlib import Path # ruff:ignore[typing-only-standard-library-import] public annotation supports get_type_hints

if TYPE_CHECKING:
from pathlib import Path
from ._command import check_call


class Index:
Expand Down Expand Up @@ -35,19 +33,15 @@ def url(self) -> str:

def use(self) -> None:
"""Use this index server."""
_check_call([*self._client_cmd_base, "use", f"{self.user}/{self.name}"])
check_call([*self._client_cmd_base, "use", f"{self.user}/{self.name}"])

def upload(self, *files: Path) -> None:
"""
Upload packages to the index.

:param files: the files to upload
"""
_check_call(self._client_cmd_base + ["upload", "--index", self.name] + [str(i) for i in files])


def _check_call(cmd: list[str]) -> None:
run(cmd, check=True, capture_output=True) # ruff:ignore[subprocess-without-shell-equals-true] # no shell, argument list only
check_call(self._client_cmd_base + ["upload", "--index", self.name] + [str(i) for i in files])


__all__ = [
Expand Down
48 changes: 29 additions & 19 deletions src/devpi_process/_server.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
from __future__ import annotations

import random
import secrets
import socket
import string
import sys
import sysconfig
from contextlib import closing
from logging import getLogger
from pathlib import Path
from subprocess import PIPE, Popen # ruff:ignore[suspicious-subprocess-import] # no shell, argument lists only
from threading import Thread
from typing import IO, TYPE_CHECKING, cast

from ._index import Index, _check_call
from ._command import check_call
from ._index import Index

if TYPE_CHECKING:
from collections.abc import Iterator, Sequence
Expand All @@ -30,7 +32,8 @@ class IndexServer:
def __init__(
self,
path: Path,
with_root_pypi: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public parameter, changing it breaks callers
# Preserve the public positional argument.
with_root_pypi: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument]
start_args: Sequence[str] | None = None,
) -> None:
"""
Expand All @@ -46,14 +49,14 @@ def __init__(

self.host = "localhost"
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as socket_handler:
socket_handler.bind(("", 0))
socket_handler.bind(("", 0)) # devpi can bind to any IPv4 interface via start_args
self.port = cast("int", socket_handler.getsockname()[1])
self._passwd = "".join(random.choices(string.ascii_letters, k=8)) # ruff:ignore[suspicious-non-cryptographic-random-usage] # throwaway password of a test server
self._passwd = "".join(secrets.choice(string.ascii_letters) for _ in range(8))

scripts_dir = sysconfig.get_path("scripts")
if scripts_dir is None:
msg = "could not get scripts folder of host interpreter" # pragma: no cover # typing guard
raise RuntimeError(msg) # pragma: no cover # typing guard
msg = "could not get scripts folder of host interpreter"
raise RuntimeError(msg)

def _exe(name: str) -> str:
return str(Path(scripts_dir) / f"{name}{'.exe' if sys.platform == 'win32' else ''}")
Expand All @@ -71,8 +74,12 @@ def _exe(name: str) -> str:

def __enter__(self) -> Self:
"""Start the index server."""
self._create_and_start_server()
self._setup_client()
try:
self._create_and_start_server()
self._setup_client()
except BaseException:
self.__exit__(None, None, None)
raise
return self

def _create_and_start_server(self) -> None:
Expand All @@ -83,13 +90,13 @@ def _create_and_start_server(self) -> None:
cmd.extend(("--role", "standalone", "--root-passwd", self._passwd))
if self._with_root_pypi is False:
cmd.append("--no-root-pypi")
_check_call(cmd)
check_call(cmd)
# 2. start the server
cmd = [self._server, "--serverdir", server_at, "--port", str(self.port)]
cmd.extend(self._start_args)
self._process = Popen(cmd, stdout=PIPE, universal_newlines=True) # ruff:ignore[subprocess-without-shell-equals-true] # no shell, argument list only
stdout = self._drain_stdout()
for line in stdout: # pragma: no branch # will always loop at least once
for line in stdout:
if "serving at url" in line:

def _keep_draining() -> None:
Expand All @@ -100,21 +107,24 @@ def _keep_draining() -> None:
self._stdout_drain = Thread(target=_keep_draining, name="tox-test-stdout-drain")
self._stdout_drain.start()
break
else:
msg = f"devpi server at {self._server_dir} exited before it was ready"
raise RuntimeError(msg)

def _drain_stdout(self) -> Iterator[str]:
process = cast("Popen[str]", self._process)
stdout = cast("IO[str]", process.stdout)
while True:
if process.poll() is not None: # pragma: no cover # a dead server is a failure path the tests do not drive
print(f"devpi server with pid {process.pid} at {self._server_dir} died") # ruff:ignore[print] # reports the dead server in the test output
if process.poll() is not None:
getLogger(__name__).error("devpi server with pid %s at %s died", process.pid, self._server_dir)
break
yield stdout.readline()

def _setup_client(self) -> None:
self._client_dir.mkdir(exist_ok=True)
base = ["--clientdir", str(self._client_dir)]
_check_call([self._client, "use", *base, self.url])
_check_call([self._client, "login", *base, self.user, "--password", self._passwd])
check_call([self._client, "use", *base, self.url])
check_call([self._client, "login", *base, self.user, "--password", self._passwd])

@property
def url(self) -> str:
Expand All @@ -139,9 +149,9 @@ def __exit__(
:param exc_val: the exception leaving the block, or None
:param exc_tb: traceback of that exception, or None
"""
if self._process is not None: # pragma: no cover # defend against devpi startup fail
if self._process is not None:
self._process.terminate()
if self._stdout_drain is not None and self._stdout_drain.is_alive(): # pragma: no cover # devpi startup fail
if self._stdout_drain is not None and self._stdout_drain.is_alive():
self._stdout_drain.join()

def __repr__(self) -> str:
Expand All @@ -157,11 +167,11 @@ def create_index(self, name: str, *args: str) -> Index:
:returns: the created index
:raises ValueError: when an index with this name already exists
"""
if name in self._indexes: # pragma: no cover # a duplicate name is a caller error the tests do not drive
if name in self._indexes:
msg = f"index {name} already exists"
raise ValueError(msg)
base = [self._client, "--clientdir", str(self._client_dir)]
_check_call([*base, "index", "-c", name, *args])
check_call([*base, "index", "-c", name, *args])
index = Index(f"{self.url}/{self.user}", name, self.user, base)
self._indexes[name] = index
return index
Expand Down
22 changes: 10 additions & 12 deletions tests/demo_pkg_inline/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,25 +51,24 @@

def build_wheel(
wheel_directory: str,
metadata_directory: str | None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument
metadata_directory: str | None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature
) -> str:
base_name = f"{_NAME}-{_VERSION}-py{sys.version_info[0]}-none-any.whl"
with ZipFile(str(Path(wheel_directory) / base_name), "w") as zip_file_handler:
for arc_name, data in _CONTENT.items(): # pragma: no branch # the content table always has entries
for arc_name, data in _CONTENT.items(): # pragma: no branch # _CONTENT is nonempty
zip_file_handler.writestr(arc_name, dedent(data).strip())
return base_name


def get_requires_for_build_wheel(
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument
) -> list[str]:
return [] # pragma: no cover # only executed in non-host pythons
# PEP 517 requires the unused argument.
def get_requires_for_build_wheel(config_settings: None = None) -> list[str]: # ruff:ignore[unused-function-argument]
return []


def build_sdist(
sdist_directory: str,
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 signature
) -> str:
result = f"{_NAME}-{_VERSION}.tar.gz"
with tarfile.open(str(Path(sdist_directory) / result), "w:gz") as tar:
Expand All @@ -85,7 +84,6 @@ def build_sdist(
return result


def get_requires_for_build_sdist(
config_settings: None = None, # ruff:ignore[unused-function-argument] # PEP 517 hook signature, the backend must accept the argument
) -> list[str]:
return [] # pragma: no cover # only executed in non-host pythons
# PEP 517 requires the unused argument.
def get_requires_for_build_sdist(config_settings: None = None) -> list[str]: # ruff:ignore[unused-function-argument]
return []
80 changes: 68 additions & 12 deletions tests/test_devpi_process.py
Original file line number Diff line number Diff line change
@@ -1,34 +1,50 @@
from __future__ import annotations

import socket
import sysconfig
from contextlib import ExitStack
from http import HTTPStatus
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, get_type_hints

import pytest
from httpx import get

from devpi_process import IndexServer
from devpi_process import Index, IndexServer, __version__

if TYPE_CHECKING:
from _pytest.tmpdir import TempPathFactory
from types import ModuleType


def test_version() -> None:
import devpi_process # ruff:ignore[import-outside-top-level] # the test checks the package import itself
assert __version__ is not None

assert devpi_process.__version__ is not None

def test_index_upload_type_hints() -> None:
assert get_type_hints(Index.upload)["files"] is Path


@pytest.fixture(scope="session")
def demo_artifacts(tmp_path_factory: TempPathFactory) -> tuple[Path, Path]:
def demo_backend() -> ModuleType:
spec = spec_from_file_location("demo_pkg_inline.build", str(Path(__file__).parent / "demo_pkg_inline" / "build.py"))
assert spec is not None
build = module_from_spec(spec)
loader = spec.loader
assert loader is not None
loader.exec_module(build)
return build


@pytest.fixture(scope="session")
def demo_artifacts(tmp_path_factory: pytest.TempPathFactory, demo_backend: ModuleType) -> tuple[Path, Path]:
base = tmp_path_factory.mktemp("wheel")
return base / build.build_wheel(str(base)), base / build.build_sdist(str(base))
return base / demo_backend.build_wheel(str(base)), base / demo_backend.build_sdist(str(base))


def test_demo_backend_requires_no_extra_build_dependencies(demo_backend: ModuleType) -> None:
assert demo_backend.get_requires_for_build_wheel() == []
assert demo_backend.get_requires_for_build_sdist() == []


def test_create_server(tmp_path: Path, demo_artifacts: tuple[Path, Path]) -> None:
Expand Down Expand Up @@ -65,8 +81,48 @@ def test_create_server_with_pypi(tmp_path: Path) -> None:


def test_create_server_start_args(tmp_path: Path) -> None:
with IndexServer(tmp_path, start_args=["--offline-mode"]) as server:
assert server._process is not None # ruff:ignore[private-member-access] # the test inspects the spawned command
args = server._process.args # ruff:ignore[private-member-access] # the test inspects the spawned command
assert isinstance(args, list)
assert args[-1] == "--offline-mode"
with IndexServer(tmp_path, start_args=["--host", "127.0.0.1"]) as server:
response = get(f"{server.url}/+status", headers={"Accept": "application/json"})
assert response.status_code == HTTPStatus.OK
assert response.json()["result"]["host"] == "127.0.0.1"


def test_server_probes_all_ipv4_interfaces(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bound_addresses: list[socket._Address] = []

class ProbeSocket(socket.socket):
def bind(self, address: socket._Address) -> None:
bound_addresses.append(address)
super().bind(address)

monkeypatch.setattr(socket, "socket", ProbeSocket)

IndexServer(tmp_path)

assert bound_addresses == [("", 0)]


def test_create_server_rejects_duplicate_index(tmp_path: Path) -> None:
with IndexServer(tmp_path) as server:
server.create_index("base")
with pytest.raises(ValueError, match="index base already exists"):
server.create_index("base")


def test_create_server_rejects_missing_scripts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sysconfig, "get_path", lambda _name: None)

with pytest.raises(RuntimeError, match="could not get scripts folder of host interpreter"):
IndexServer(tmp_path)


def test_create_server_reports_missing_executable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sysconfig, "get_path", lambda _name: str(tmp_path / "missing"))

with ExitStack() as stack, pytest.raises(FileNotFoundError, match="devpi-init"):
stack.enter_context(IndexServer(tmp_path))


def test_create_server_reports_startup_failure(tmp_path: Path) -> None:
with ExitStack() as stack, pytest.raises(RuntimeError, match="exited before it was ready"):
stack.enter_context(IndexServer(tmp_path, start_args=["--definitely-invalid"]))