diff --git a/pyproject.toml b/pyproject.toml index fc7ced77cc..9615d44e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,8 +112,7 @@ include = [ "src/dstack/_internal/core/backends", "src/dstack/_internal/cli/services/configurators", "src/dstack/_internal/cli/commands", - "src/tests/_internal/core", - "src/tests/_internal/server/background/pipeline_tasks", + "src/tests", ] exclude = [ # OCI has pyright errors. Excluded rather than ignored because checking it makes pyright analyze diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index c6e31b6227..5c36acd506 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -13,7 +13,7 @@ from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError -from dstack._internal.utils.common import render_datetime_as_api +from dstack._internal.utils.common import get_or_error, render_datetime_as_api from tests._internal.cli.common import ( get_preset, get_session_run, @@ -543,7 +543,7 @@ def test_create_detaches_the_name_from_the_old_preset(self, tmp_path): assert exit_code == 0 create.assert_called_once() - assert store.get(preset.id).name is None + assert get_or_error(store.get(preset.id)).name is None def test_create_without_confirmation_exits_before_creating(self, tmp_path): preset = get_preset().model_copy(update={"name": "qwen"}) @@ -569,7 +569,7 @@ def test_create_without_confirmation_exits_before_creating(self, tmp_path): assert exit_code == 0 create.assert_not_called() - assert store.get(preset.id).name == "qwen" + assert get_or_error(store.get(preset.id)).name == "qwen" def test_get_and_delete_resolve_names(self, tmp_path, capsys): preset = get_preset().model_copy(update={"name": "qwen"}) diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index 2332746251..add3afc533 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from pathlib import Path from types import SimpleNamespace -from typing import IO, Any, List, Optional +from typing import IO, Any, List, Optional, cast from unittest.mock import patch from uuid import uuid4 @@ -19,6 +19,7 @@ ) from dstack._internal.cli.models.presets import VerifiedPreset from dstack._internal.cli.services.presets.agents.base import PresetAgentSpec +from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.models.configurations import ( DEFAULT_REPLICA_GROUP_NAME, @@ -32,6 +33,7 @@ ) from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import JobStatus, Run, RunStatus, ServiceSpec +from dstack.api import Client def plain_console(file: IO[str], *, width: int = 250) -> Console: @@ -50,20 +52,26 @@ def run_dstack_cli( repo_dir: Optional[Path] = None, ) -> int: exit_code = 0 + cwd = os.getcwd() + prev_home_dir = os.environ.get("HOME") + prev_userprofile = os.environ.get("USERPROFILE") if repo_dir is not None: - cwd = os.getcwd() os.chdir(repo_dir) if home_dir is not None: - prev_home_dir = os.environ.get("HOME") os.environ["HOME"] = str(home_dir) if IS_WINDOWS: - prev_userprofile = os.environ.get("USERPROFILE") os.environ["USERPROFILE"] = str(home_dir) with patch("sys.argv", ["dstack"] + cli_args): try: main() except SystemExit as e: - exit_code = e.code + # Same as the process exit status: no code is success, a non-int code is an error + if e.code is None: + exit_code = 0 + elif isinstance(e.code, int): + exit_code = e.code + else: + exit_code = 1 finally: if home_dir is not None: if prev_home_dir is None: @@ -95,28 +103,30 @@ def run_dstack_cli( def get_preset_benchmark() -> PresetBenchmark: - benchmark = PresetBenchmark( - tool="vllm bench serve", - tool_version="0.11.0", - command="vllm bench serve --base-url $SERVICE_URL", - workload={ - "api": "chat_completions", - "num_requests": 16, - "input_tokens": 1024, - "output_tokens": 128, - "concurrency": 1, - }, - metrics={ - "successful_requests": 16, - "failed_requests": 0, - "duration_seconds": 48.64, - "total_input_tokens": 16384, - "total_output_tokens": 2048, - "output_tok_per_s": 42.1, - "per_user_tok_per_s": 42.1, - "ttft_ms": {"mean": 110.9, "p50": 108.2, "p99": 121.6}, - "tpot_ms": {"mean": 7.5, "p50": 7.4, "p99": 8.1}, - }, + benchmark = PresetBenchmark.model_validate( + { + "tool": "vllm bench serve", + "tool_version": "0.11.0", + "command": "vllm bench serve --base-url $SERVICE_URL", + "workload": { + "api": "chat_completions", + "num_requests": 16, + "input_tokens": 1024, + "output_tokens": 128, + "concurrency": 1, + }, + "metrics": { + "successful_requests": 16, + "failed_requests": 0, + "duration_seconds": 48.64, + "total_input_tokens": 16384, + "total_output_tokens": 2048, + "output_tok_per_s": 42.1, + "per_user_tok_per_s": 42.1, + "ttft_ms": {"mean": 110.9, "p50": 108.2, "p99": 121.6}, + "tpot_ms": {"mean": 7.5, "p50": 7.4, "p99": 8.1}, + }, + } ) return benchmark @@ -258,14 +268,27 @@ def get_session_run(**overrides: Any) -> PresetSessionRun: def get_successful_preset_report(run: Run) -> PresetAgentSuccess: - return PresetAgentSuccess( - success=True, - run_id=run.id, - run_name=run.run_spec.run_name, - service_yaml="type: service", - trial=1, - base="Qwen/Qwen3.5-27B", - model="community/Qwen3.5-27B-GPTQ-Int4", - context_length=32768, - benchmark=get_preset_benchmark(), + return PresetAgentSuccess.model_validate( + { + "success": True, + "run_id": run.id, + "run_name": run.run_spec.run_name, + "service_yaml": "type: service", + "trial": 1, + "base": "Qwen/Qwen3.5-27B", + "model": "community/Qwen3.5-27B-GPTQ-Int4", + "context_length": 32768, + "benchmark": get_preset_benchmark(), + } ) + + +def get_verified_preset(store: PresetStore, preset_id: str) -> VerifiedPreset: + preset = store.get(preset_id) + assert isinstance(preset, VerifiedPreset) + return preset + + +def get_fake_api(**attrs: Any) -> Client: + # Only the attributes that the code under test reads + return cast(Client, SimpleNamespace(**attrs)) diff --git a/src/tests/_internal/cli/services/configurators/test_profile.py b/src/tests/_internal/cli/services/configurators/test_profile.py index 796b459b46..d8d3de1e43 100644 --- a/src/tests/_internal/cli/services/configurators/test_profile.py +++ b/src/tests/_internal/cli/services/configurators/test_profile.py @@ -75,7 +75,7 @@ def test_no_retry(self): def test_retry_duration(self): profile = Profile(name="test") modified, _ = apply_args(profile, ["--retry-duration", "1h"]) - profile.retry = ProfileRetry(on_events=None, duration="1h") + profile.retry = ProfileRetry.model_validate({"on_events": None, "duration": "1h"}) assert profile.model_dump() == modified.model_dump() diff --git a/src/tests/_internal/cli/services/configurators/test_run.py b/src/tests/_internal/cli/services/configurators/test_run.py index fbe80fddfb..7c463d553d 100644 --- a/src/tests/_internal/cli/services/configurators/test_run.py +++ b/src/tests/_internal/cli/services/configurators/test_run.py @@ -76,7 +76,7 @@ def test_env_override(self): assert modified.model_dump() == conf.model_dump() def test_ports_override(self): - conf = TaskConfiguration(commands=["whoami"], ports=["80"]) + conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["80"]}) modified, args = self.apply_args(conf, ["-p", "8000:80", "--port", "8001:8000"]) conf.ports = [ PortMapping(local_port=8000, container_port=80), @@ -85,12 +85,12 @@ def test_ports_override(self): assert modified.model_dump() == conf.model_dump() def test_local_ports_conflict(self): - conf = TaskConfiguration(commands=["whoami"], ports=["3000"]) + conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["3000"]}) with pytest.raises(ConfigurationError): self.apply_args(conf, ["-p", "3000:4000"]) def test_any_port(self): - conf = TaskConfiguration(commands=["whoami"], ports=["8000"]) + conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["8000"]}) modified, args = self.apply_args(conf, ["-p", "*:8000"]) conf.ports = [PortMapping(local_port=None, container_port=8000)] assert modified.model_dump() == conf.model_dump() @@ -132,6 +132,7 @@ def test_interpolates_service_group_commands(self): } ) modified, _ = self.apply_args(conf, ["hello"]) + assert isinstance(modified, ServiceConfiguration) assert modified.groups is not None assert modified.groups[0].commands == [ "echo hello ${{ groups[0].replicas[0].IP_ADDRESS }}" @@ -164,6 +165,7 @@ def test_homogeneous_service_interpolates_commands_once(self): } ) modified, _ = self.apply_args(conf, []) + assert isinstance(modified, ServiceConfiguration) assert modified.commands == ["echo ${{ env.FOO }}"] diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index d5277a6bf0..e55c3fd013 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -8,6 +8,7 @@ from contextlib import suppress from pathlib import Path from types import SimpleNamespace +from typing import cast import psutil import pytest @@ -60,7 +61,14 @@ from dstack._internal.core.errors import CLIError from dstack._internal.core.models.configurations import PresetAgentConfig, PresetConfiguration from dstack._internal.core.services.configs import ConfigManager -from tests._internal.cli.common import get_agent_spec, get_session_run, get_session_state +from dstack._internal.utils.common import get_or_error +from dstack.api import Client +from tests._internal.cli.common import ( + get_agent_spec, + get_fake_api, + get_session_run, + get_session_state, +) def _record_run(session, workspace_record): @@ -477,7 +485,7 @@ def test_inherits_only_required_environment(self, tmp_path, monkeypatch): monkeypatch.setenv("PATH", "/usr/bin") monkeypatch.setenv("HOME", "/home/test") monkeypatch.setenv("UNRELATED_SECRET", "must-not-be-inherited") - api = SimpleNamespace( + api = get_fake_api( project="main", client=SimpleNamespace(base_url="http://127.0.0.1:3000"), ) @@ -547,11 +555,13 @@ def _session_workspace(tmp_path): class TestAgentSession: def _configuration(self) -> PresetConfiguration: - return PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", - max_price=0.5, - env=["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"], + return PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3.5-27B", + "max_price": 0.5, + "env": ["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"], + } ) def _home(self, tmp_path, monkeypatch) -> None: @@ -630,7 +640,8 @@ def test_reports_invalid_existing_parent(self, tmp_path, monkeypatch): with pytest.raises(CLIError, match="Could not create agent output"): create_preset_session( - PresetConfiguration(name="qwen", base="Qwen/Qwen3.5-27B"), previous=() + PresetConfiguration.model_validate({"name": "qwen", "base": "Qwen/Qwen3.5-27B"}), + previous=(), ) def test_log_write_failure_warns_once(self, tmp_path, capsys): @@ -1325,7 +1336,7 @@ async def test_does_not_resurrect_an_externally_stopped_agent(self, tmp_path, mo workspace, session = _agent_setup(tmp_path) # Another CLI recorded a stop: the death must read as a decision, not an # outage to retry. - state = session.read_state() + state = get_or_error(session.read_state()) state.status = "interrupted" session.write_state(state) @@ -1622,6 +1633,7 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path): trials_dir = _write_trials(tmp_path, [record, rerun, record, nameless]) summary = _summarize_session_trials(trials_dir) + assert summary is not None # 4 records = 4 trials: one long-lived task commonly hosts several # trials, so identical records must not collapse the count. @@ -1649,6 +1661,7 @@ def test_trials_are_ordered_numerically_beyond_nine(self, tmp_path): trials_dir = _write_trials(tmp_path, records) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["series"] == [float(i * 100) for i in range(1, 13)] @@ -1666,6 +1679,7 @@ def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): (trials_dir / "2" / "task.dstack.yml").write_text("type: task\n") summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["count"] == 1 assert summary["series"] == [2000.0] @@ -1676,6 +1690,7 @@ def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): (trials_dir / "1" / "trial.json").write_text('{"benchmark": {"met') summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["count"] == 0 @@ -1780,18 +1795,18 @@ def test_detach_keeps_the_agent_and_stop_terminates_it(self, tmp_path, monkeypat session.write_state(state) monkeypatch.setattr(create_module, "confirm_ask", lambda *_: False) - _stop_or_detach_agent_session(session, None) + _stop_or_detach_agent_session(session, cast(Client, None)) assert agent.poll() is None - assert session.read_state().status == "running" + assert get_or_error(session.read_state()).status == "running" assert "Detached" in capsys.readouterr().out monkeypatch.setattr(create_module, "confirm_ask", lambda *_: True) - _stop_or_detach_agent_session(session, None) + _stop_or_detach_agent_session(session, cast(Client, None)) psutil.Process(agent.pid).wait(timeout=10) - assert session.read_state().status == "interrupted" + assert get_or_error(session.read_state()).status == "interrupted" finally: with suppress(OSError): - os.killpg(agent.pid, signal.SIGKILL) + os.killpg(agent.pid, signal.SIGKILL) # pyright: ignore[reportAttributeAccessIssue] class TestOffsetStoreSharing: diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index 959aeedd6d..c6203023f4 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -59,8 +59,11 @@ from dstack._internal.core.models.configurations import PresetAgentConfig, PresetConfiguration from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.runs import Run, RunStatus +from dstack._internal.utils.common import get_or_error +from dstack.api import Client from tests._internal.cli.common import ( get_agent_spec, + get_fake_api, get_preset, get_running_service_run, get_session_run, @@ -106,14 +109,16 @@ def get_preset_agent(provider=None): monkeypatch.setattr( "dstack._internal.cli.services.presets.create.get_preset_agent", get_preset_agent ) - configuration = PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - agent={"provider": "codex", "model": "gpt-6-astra", "effort": "xhigh"}, + configuration = PresetConfiguration.model_validate( + { + "name": "qwen-build", + "base": "Qwen/Qwen3.5-27B", + "agent": {"provider": "codex", "model": "gpt-6-astra", "effort": "xhigh"}, + } ) setup = _fresh_setup( - api=SimpleNamespace(project="main"), + api=get_fake_api(project="main"), configuration=configuration, session=_agent_session(tmp_path), build_name="qwen-build", @@ -140,8 +145,10 @@ def get_preset_agent(provider=None): ) _fresh_setup( - api=SimpleNamespace(project="main"), - configuration=PresetConfiguration(name="qwen-build", base="Qwen/Qwen3.5-27B"), + api=get_fake_api(project="main"), + configuration=PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B"} + ), session=_agent_session(tmp_path), build_name="qwen-build", allowed_fleets=("gpu-fleet",), @@ -173,26 +180,30 @@ def creation_context(tmp_path, monkeypatch): runs=run_apis, ), ) - configuration = PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - max_ttft=5000, - # Matches the fixture report's benchmark concurrency, which verification compares. - concurrency=1, - trials=1, - fleets=["gpu-fleet"], - env={"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"}, + configuration = PresetConfiguration.model_validate( + { + "name": "qwen-build", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "max_ttft": 5000, + # Matches the fixture report's benchmark concurrency, which verification compares. + "concurrency": 1, + "trials": 1, + "fleets": ["gpu-fleet"], + "env": {"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"}, + } ) - source_configuration = PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - max_ttft=5000, - concurrency=1, - trials=1, - fleets=["gpu-fleet"], - env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], + source_configuration = PresetConfiguration.model_validate( + { + "name": "qwen-build", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "max_ttft": 5000, + "concurrency": 1, + "trials": 1, + "fleets": ["gpu-fleet"], + "env": ["LICENSE", "TOKENIZERS_PARALLELISM=false"], + } ) monkeypatch.setattr( "dstack._internal.cli.services.presets.create.get_preset_agent", @@ -233,10 +244,9 @@ async def create(**kwargs): ) create_preset( - api=SimpleNamespace(), - configuration=PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", + api=get_fake_api(), + configuration=PresetConfiguration.model_validate( + {"name": "qwen", "base": "Qwen/Qwen3.5-27B"} ), store=PresetStore(tmp_path / "presets"), ) @@ -285,11 +295,13 @@ def fail_finish(self, preset_id=None): monkeypatch.setattr(PresetSession, "finish", fail_finish) result = create_preset( - api=SimpleNamespace(), - configuration=PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", - env=["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"], + api=get_fake_api(), + configuration=PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3.5-27B", + "env": ["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"], + } ), store=PresetStore(tmp_path / "presets"), ) @@ -326,17 +338,16 @@ def fail_finish(self, preset_id=None): with pytest.raises(RuntimeError, match="creation failed"): create_preset( - api=SimpleNamespace(), - configuration=PresetConfiguration( - name="qwen", - base="Qwen/Qwen3.5-27B", + api=get_fake_api(), + configuration=PresetConfiguration.model_validate( + {"name": "qwen", "base": "Qwen/Qwen3.5-27B"} ), store=PresetStore(tmp_path / "presets"), ) @pytest.mark.asyncio async def test_checks_active_fleets_before_claude_launch(self, tmp_path, monkeypatch): - api = SimpleNamespace( + api = get_fake_api( project="main", client=SimpleNamespace(fleets=SimpleNamespace(list=lambda *args, **kwargs: [])), ) @@ -348,9 +359,8 @@ async def test_checks_active_fleets_before_claude_launch(self, tmp_path, monkeyp ) with pytest.raises(CLIError, match="no fleets"): - configuration = PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", + configuration = PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B"} ) await _create_preset( api=api, @@ -569,8 +579,8 @@ def test_flag_overrides_and_property_stands_without_it(self): def configuration(): # A fresh object per call: the merger mutates its input. - return PresetConfiguration( - name="qwen", base="Qwen/Qwen3.5-27B", previous=["from-config"] + return PresetConfiguration.model_validate( + {"name": "qwen", "base": "Qwen/Qwen3.5-27B", "previous": ["from-config"]} ) configurator = PresetConfigurator(api_client=MagicMock()) @@ -629,7 +639,7 @@ async def run_agent(**kwargs): assert seen["record"] is True assert "## Previous Sessions" in seen["prompt"] assert "8d3b01aa" in seen["prompt"] - assert session.read_state().previous == ["8d3b01aa"] + assert get_or_error(session.read_state()).previous == ["8d3b01aa"] # constraints.json is a persistent session record. assert (session_path / "constraints.json").is_file() @@ -653,7 +663,7 @@ async def test_stops_only_recorded_build_runs(self, tmp_path, monkeypatch): '{"name":"qwen-build-1"}\n{"name":"unrelated-run"}\n{"name":"qwen-build-2"}\n' ) runs = _FakeRuns() - api = SimpleNamespace( + api = get_fake_api( runs=runs, project="main", client=SimpleNamespace(runs=runs), @@ -738,15 +748,17 @@ def test_a_live_session_prints_the_log_alone(self, tmp_path, monkeypatch, capsys class TestFindings: def test_passes_through_to_constraints(self): - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - max_ttft=5000, - min_context_length=8192, - concurrency=8, - trials=1, - input_tokens=8192, - shared_prefix_tokens=7424, + configuration = PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "max_ttft": 5000, + "min_context_length": 8192, + "concurrency": 8, + "trials": 1, + "input_tokens": 8192, + "shared_prefix_tokens": 7424, + } ) data = json.loads( @@ -761,40 +773,46 @@ def test_rejects_a_prefix_that_leaves_nothing_unique(self): # Every request would be identical, which measures the cache rather than # the configuration. with pytest.raises(ValidationError, match="less than input_tokens"): - PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - max_ttft=5000, - min_context_length=8192, - concurrency=8, - trials=1, - input_tokens=8192, - shared_prefix_tokens=8192, + PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "max_ttft": 5000, + "min_context_length": 8192, + "concurrency": 8, + "trials": 1, + "input_tokens": 8192, + "shared_prefix_tokens": 8192, + } ) def test_checks_against_the_default_input_tokens(self): # `input_tokens` unset means 1024, so a larger prefix is still rejected. with pytest.raises(ValidationError, match="less than input_tokens"): - PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - max_ttft=5000, - min_context_length=8192, - concurrency=8, - trials=1, - shared_prefix_tokens=2048, + PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "max_ttft": 5000, + "min_context_length": 8192, + "concurrency": 8, + "trials": 1, + "shared_prefix_tokens": 2048, + } ) class TestPerformanceConstraints: def test_max_ttft_reaches_the_constraints(self): - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - min_context_length=8192, - concurrency=8, - trials=1, - max_ttft=10000, + configuration = PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "min_context_length": 8192, + "concurrency": 8, + "trials": 1, + "max_ttft": 10000, + } ) data = json.loads( @@ -820,14 +838,16 @@ def test_throughput_is_derived_not_read(self): class TestBuildConstraints: def test_renders_defaults_for_the_optional_fields(self): - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - concurrency=8, - trials=3, - max_ttft=5000, - min_context_length=32768, - env=["HF_TOKEN"], + configuration = PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "concurrency": 8, + "trials": 3, + "max_ttft": 5000, + "min_context_length": 32768, + "env": ["HF_TOKEN"], + } ) text = _build_constraints( @@ -853,14 +873,16 @@ def test_renders_defaults_for_the_optional_fields(self): } def test_renders_custom_dataset_without_request_shape(self): - configuration = PresetConfiguration( - name="qwen", - base="Qwen/Qwen3-32B", - min_context_length=32768, - max_ttft=5000, - trials=3, - concurrency=8, - dataset="spec_bench", + configuration = PresetConfiguration.model_validate( + { + "name": "qwen", + "base": "Qwen/Qwen3-32B", + "min_context_length": 32768, + "max_ttft": 5000, + "trials": 3, + "concurrency": 8, + "dataset": "spec_bench", + } ) text = _build_constraints( @@ -883,13 +905,15 @@ def test_renders_custom_dataset_without_request_shape(self): } def test_renders_configured_values(self): - configuration = PresetConfiguration( - name="qwen", - model={"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"}, - min_context_length=32768, - max_ttft=5000, - trials=10, - concurrency=16, + configuration = PresetConfiguration.model_validate( + { + "name": "qwen", + "model": {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"}, + "min_context_length": 32768, + "max_ttft": 5000, + "trials": 10, + "concurrency": 16, + } ) data = json.loads( @@ -955,8 +979,10 @@ async def create(**kwargs): with pytest.raises(KeyboardInterrupt): create_preset( - api=SimpleNamespace(), - configuration=PresetConfiguration(name="qwen", base="Qwen/Qwen3.5-27B"), + api=get_fake_api(), + configuration=PresetConfiguration.model_validate( + {"name": "qwen", "base": "Qwen/Qwen3.5-27B"} + ), store=PresetStore(tmp_path / "presets"), ) @@ -991,7 +1017,7 @@ def test_interrupt_at_stop_prompt_stops_agent(self, tmp_path, monkeypatch): ) monkeypatch.setattr(Confirm, "ask", Mock(side_effect=KeyboardInterrupt)) - _stop_or_detach_agent_session(session, SimpleNamespace()) + _stop_or_detach_agent_session(session, get_fake_api()) assert stopped == ["terminate", "stop_runs"] state = json.loads((session_dir / "session.json").read_text()) @@ -1021,7 +1047,7 @@ def test_suspend_scrubs_workspace_token(self, tmp_path, capsys): # The live credential is gone; the rest of the workspace stays resumable. assert not (config_dir / "config.yml").exists() assert (workspace_root / "w" / "constraints.json").exists() - assert session.read_state().status == "interrupted" + assert get_or_error(session.read_state()).status == "interrupted" @pytest.mark.asyncio async def test_resume_uses_saved_claude_session(self, creation_context, monkeypatch, tmp_path): @@ -1165,7 +1191,7 @@ def test_no_offers_shows_the_shared_warning_without_failing(self, capsys): user="admin", job_plans=[SimpleNamespace(offers=[], total_offers=0, max_price=None)], ) - api = SimpleNamespace( + api = get_fake_api( project="main", client=SimpleNamespace(runs=SimpleNamespace(get_plan=lambda *a, **k: plan)), ) @@ -1279,7 +1305,7 @@ async def fake_attach(**kwargs): assert result.preset.id == "ab12cd34" assert creation_context.store.get("ab12cd34") is not None - assert session.read_state().status == "success" + assert get_or_error(session.read_state()).status == "success" def test_agent_death_without_report_suspends_instead_of_failing( self, creation_context, monkeypatch, tmp_path @@ -1307,7 +1333,7 @@ async def fake_attach(**kwargs): preset_id="ab12cd34", ) - assert session.read_state().status == "interrupted" + assert get_or_error(session.read_state()).status == "interrupted" def test_backs_off_when_claim_is_held(self, creation_context, monkeypatch, tmp_path): session = self._detached_session( @@ -1327,7 +1353,7 @@ def test_backs_off_when_claim_is_held(self, creation_context, monkeypatch, tmp_p store=creation_context.store, preset_id="ab12cd34", ) - assert session.read_state().status == "running" + assert get_or_error(session.read_state()).status == "running" release_session_claim(held) @@ -1359,7 +1385,7 @@ def test_reports_terminal_states_without_stopping( self._patch_root(monkeypatch, tmp_path) self._session_dir(tmp_path, get_session_state(**{"id": "ab12cd34", "status": status})) - stop_preset_session(SimpleNamespace(), "ab12cd34") + stop_preset_session(get_fake_api(), "ab12cd34") assert message in capsys.readouterr().out @@ -1383,7 +1409,7 @@ def test_finalizes_completed_session_and_reports_created(self, tmp_path, monkeyp lambda **kwargs: calls.append(kwargs), ) - stop_preset_session(SimpleNamespace(), "ab12cd34") + stop_preset_session(get_fake_api(), "ab12cd34") # Finalized silently like reconcile, honoring the persisted keep-service. assert len(calls) == 1 @@ -1426,7 +1452,7 @@ def record_finish(session, status): record_finish, ) - stop_preset_session(SimpleNamespace(), "ab12cd34") + stop_preset_session(get_fake_api(), "ab12cd34") # The intent is recorded before the agent dies, so a live owner's retry # loop can never resurrect it. @@ -1447,14 +1473,14 @@ def _session(self, tmp_path) -> PresetSession: ) return PresetSession(path=session_dir, preset_id="ab12cd34") - def _api(self, statuses: dict, stopped: list) -> SimpleNamespace: + def _runs_api(self, statuses: dict, stopped: list) -> Client: def get(project, name): return SimpleNamespace(status=statuses[name]) def stop(project, names, abort): stopped.extend(names) - return SimpleNamespace( + return get_fake_api( project="main", client=SimpleNamespace(runs=SimpleNamespace(get=get, stop=stop)), ) @@ -1476,7 +1502,7 @@ def stop(project, names, abort): ) def test_stops_only_non_terminal_runs(self, tmp_path, statuses, expected_stopped): stopped: list = [] - api = self._api(statuses, stopped) + api = self._runs_api(statuses, stopped) # No per-run prompt: active runs are stopped automatically (like dstack stop). _stop_active_session_runs(api, self._session(tmp_path)) diff --git a/src/tests/_internal/cli/services/presets/test_export.py b/src/tests/_internal/cli/services/presets/test_export.py index d1244252ae..28c75a8091 100644 --- a/src/tests/_internal/cli/services/presets/test_export.py +++ b/src/tests/_internal/cli/services/presets/test_export.py @@ -8,6 +8,7 @@ from dstack._internal.core.errors import CLIError from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.files import FilePathMapping +from dstack._internal.utils.common import get_or_error from tests._internal.cli.common import get_preset pytestmark = pytest.mark.windows @@ -27,7 +28,10 @@ def test_exports_a_directory_the_service_mounts(self, tmp_path: Path): destination = tmp_path / "deploy" / "qwen.dstack.yml" export_preset( - store.get(preset.id), preset_dir=preset_dir, destination=destination, force=False + get_or_error(store.get(preset.id)), + preset_dir=preset_dir, + destination=destination, + force=False, ) exported = destination.parent / "service" / "1" / "patches" @@ -46,7 +50,7 @@ def test_exports_a_deployable_service_configuration_with_its_files(self, tmp_pat destination = tmp_path / "deploy" / "qwen.dstack.yml" written = export_preset( - store.get(preset.id), + get_or_error(store.get(preset.id)), preset_dir=preset_dir, destination=destination, force=False, @@ -157,7 +161,7 @@ def test_refuses_to_overwrite_without_force(self, tmp_path: Path): with pytest.raises(CLIError, match="already exists"): export_preset( - store.get(get_preset().id), + get_or_error(store.get(get_preset().id)), preset_dir=preset_dir, destination=destination, force=False, @@ -165,7 +169,7 @@ def test_refuses_to_overwrite_without_force(self, tmp_path: Path): assert destination.read_text() == "existing" export_preset( - store.get(get_preset().id), + get_or_error(store.get(get_preset().id)), preset_dir=preset_dir, destination=destination, force=True, diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 12650c0e9f..e5e963388b 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -393,6 +393,7 @@ def test_a_failed_trial_keeps_its_benchmark_but_never_becomes_best(self, tmp_pat ) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["count"] == 3 # Still charted: the trial happened and its number is real. @@ -411,6 +412,7 @@ def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): ) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["best"] is None assert summary["best_failed"] is None @@ -425,6 +427,7 @@ def test_reports_the_gpu_of_a_single_node_group(self, tmp_path): ) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["gpu"] == "MI300X:192GB:1" @@ -439,6 +442,7 @@ def test_counts_the_worker_nodes_of_a_disaggregated_trial(self, tmp_path): ) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["gpu"] == "H200:141GB:8 x3" @@ -459,6 +463,7 @@ def test_shows_the_split_when_roles_ran_different_gpus(self, tmp_path): ) summary = _summarize_session_trials(trials_dir) + assert summary is not None # Group order, not sorted: the roles read in the order they ran. assert summary["gpu"] == "H200:141GB:8 + H100:80GB:8 x2" @@ -486,6 +491,7 @@ def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): ) summary = _summarize_session_trials(trials_dir) + assert summary is not None assert summary["best"] is None assert summary["best_failed"]["tok_s"] == 300.0 diff --git a/src/tests/_internal/cli/services/presets/test_prompt.py b/src/tests/_internal/cli/services/presets/test_prompt.py index a66092f02d..7fa783328a 100644 --- a/src/tests/_internal/cli/services/presets/test_prompt.py +++ b/src/tests/_internal/cli/services/presets/test_prompt.py @@ -188,7 +188,7 @@ def test_dedents_only_an_exactly_indented_body(self, tmp_path, monkeypatch): ] for content, expected in cases: doc.write_text(content) - previous = "x" if "previous" in content else None + previous = ["x"] if "previous" in content else [] rendered = get_preset_agent_system_prompt( user_prompt=None, baseline=True, previous=previous, custom_dataset=False ) diff --git a/src/tests/_internal/cli/services/presets/test_registry.py b/src/tests/_internal/cli/services/presets/test_registry.py index 6441a6fcd9..8358a51ec4 100644 --- a/src/tests/_internal/cli/services/presets/test_registry.py +++ b/src/tests/_internal/cli/services/presets/test_registry.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path from types import SimpleNamespace -from typing import Optional +from typing import Optional, cast from uuid import uuid4 import pytest @@ -42,7 +42,9 @@ PushPresetRequest, PushPresetResponse, ) +from dstack._internal.utils.common import get_or_error from dstack._internal.utils.files import create_file_archive +from dstack.api.server._group import APIRequest from dstack.api.server._presets import PresetsAPIClient from tests._internal.cli.common import get_preset @@ -176,7 +178,7 @@ def _symlink_member(name: str, linkname: str) -> tarfile.TarInfo: def _archive_member_texts(blob: bytes) -> dict[str, str]: with tarfile.open(fileobj=io.BytesIO(blob)) as archive: return { - member.name: archive.extractfile(member).read().decode("utf-8") + member.name: get_or_error(archive.extractfile(member)).read().decode("utf-8") for member in archive.getmembers() if member.isfile() } @@ -278,7 +280,7 @@ def request(path, body=None, **kwargs): calls.append(SimpleNamespace(path=path, body=body, kwargs=kwargs)) return response - return PresetsAPIClient(request, logging.getLogger(__name__)), calls + return PresetsAPIClient(cast(APIRequest, request), logging.getLogger(__name__)), calls def test_push_sends_the_name_and_the_typed_spec(self): info = _registry_preset_info(name="qwen") @@ -515,7 +517,7 @@ def send(session, request, **kwargs): pull_preset_from_registry(store, "dstack/qwen38") assert len(requests_sent) == 2 - pulled = store.get(str(remote.id)) + pulled = get_or_error(store.get(str(remote.id))) assert pulled.name == "dstack/qwen38" assert Path(pulled.service.files[0].local_path).read_text() == "public preset file" @@ -557,8 +559,8 @@ def test_does_not_touch_a_local_preset_with_the_unqualified_name(self, tmp_path, pull_preset_from_registry(store, "main/qwen38") - assert store.get(local.id).name == "qwen38" - assert store.get(str(stub_client.remote.id)).name == "main/qwen38" + assert get_or_error(store.get(local.id)).name == "qwen38" + assert get_or_error(store.get(str(stub_client.remote.id))).name == "main/qwen38" def test_repointing_a_name_releases_it_from_the_earlier_pull(self, tmp_path, stub_client): store = PresetStore(tmp_path / "presets") @@ -572,9 +574,9 @@ def test_repointing_a_name_releases_it_from_the_earlier_pull(self, tmp_path, stu second_id = str(stub_client.remote.id) pull_preset_from_registry(store, "main/qwen38") - assert store.get(first_id).name is None - assert store.get(second_id).name == "main/qwen38" - assert store.find_by_name("main/qwen38").id == second_id + assert get_or_error(store.get(first_id)).name is None + assert get_or_error(store.get(second_id)).name == "main/qwen38" + assert get_or_error(store.find_by_name("main/qwen38")).id == second_id def test_a_non_current_version_pulled_by_id_does_not_take_the_name( self, tmp_path, stub_client @@ -590,9 +592,9 @@ def test_a_non_current_version_pulled_by_id_does_not_take_the_name( old_id = str(stub_client.remote.id) pull_preset_from_registry(store, f"main/{old_id}") - assert store.get(old_id).name is None - assert store.get(current_id).name == "main/qwen38" - assert store.find_by_name("main/qwen38").id == current_id + assert get_or_error(store.get(old_id)).name is None + assert get_or_error(store.get(current_id)).name == "main/qwen38" + assert get_or_error(store.find_by_name("main/qwen38")).id == current_id def test_repulling_a_copy_by_id_keeps_the_name_it_already_holds(self, tmp_path, stub_client): # This copy took the qualified name when it was current. Re-pulling it by @@ -608,8 +610,8 @@ def test_repulling_a_copy_by_id_keeps_the_name_it_already_holds(self, tmp_path, ) pull_preset_from_registry(store, f"main/{preset_id}") - assert store.get(preset_id).name == "main/qwen38" - assert store.find_by_name("main/qwen38").id == preset_id + assert get_or_error(store.get(preset_id)).name == "main/qwen38" + assert get_or_error(store.find_by_name("main/qwen38")).id == preset_id def test_repulling_the_same_version_overwrites_in_place(self, tmp_path, stub_client): store = PresetStore(tmp_path / "presets") @@ -624,7 +626,7 @@ def test_repulling_the_same_version_overwrites_in_place(self, tmp_path, stub_cli stub_client.file_blobs["/app/a.txt"] = _file_archive_blob(tmp_path, "a.txt", "fresh") pull_preset_from_registry(store, "main/qwen38") - assert store.get(preset_id).name == "main/qwen38" + assert get_or_error(store.get(preset_id)).name == "main/qwen38" assert [path.parent.name for path in store.root.glob("*/preset.yml")] == [preset_id] assert (store.root / preset_id / "patch" / "a.txt").read_text() == "fresh" diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index 523032eab7..fb774a68ff 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -11,11 +11,16 @@ from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError, ConfigurationError -from dstack._internal.core.models.configurations import PresetConfiguration, ServiceConfiguration +from dstack._internal.core.models.configurations import ( + PresetConfiguration, + PresetModelBase, + ServiceConfiguration, +) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping from dstack._internal.core.models.presets import PortablePreset -from tests._internal.cli.common import get_preset +from dstack._internal.utils.common import get_or_error +from tests._internal.cli.common import get_preset, get_verified_preset pytestmark = pytest.mark.windows @@ -113,9 +118,8 @@ def test_loads_a_preset_that_requested_the_random_dataset(self, tmp_path: Path): data["benchmark"]["workload"]["shared_prefix_tokens"] = 0 path.write_text(yaml.safe_dump(data, sort_keys=False)) - loaded = store.get(preset.id) + loaded = get_verified_preset(store, preset.id) - assert loaded is not None assert loaded.configuration.dataset is None assert loaded.benchmark.workload.dataset is None @@ -366,11 +370,11 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): """ ) - preset = store.get("523096d6") + preset = get_verified_preset(store, "523096d6") - assert preset is not None assert preset.name == "dsv4-flash" assert preset.best_trial == 2 + assert isinstance(preset.configuration.model, PresetModelBase) assert preset.configuration.model.base == "deepseek-ai/DeepSeek-V4-Flash" assert preset.configuration.min_context_length == 1048576 assert preset.configuration.max_ttft == 675 @@ -385,7 +389,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert [group.name for group in preset.verified_on] == [ group.name for group in preset.service.replica_groups ] - assert preset.verified_on[0].replicas[0].gpu.name == ["MI300X"] + assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. assert preset.benchmark.workload.dataset is None assert store.list() == [preset] @@ -456,15 +460,14 @@ def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path """ ) - preset = store.get("aa11bb22") + preset = get_verified_preset(store, "aa11bb22") - assert preset is not None assert [(group.name, len(group.replicas)) for group in preset.verified_on] == [ ("small", 1), ("large", 2), ] - assert preset.verified_on[0].replicas[0].gpu.name == ["H100"] - assert preset.verified_on[1].replicas[0].gpu.name == ["H200"] + assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["H100"] + assert get_or_error(preset.verified_on[1].replicas[0].gpu).name == ["H200"] def test_reports_pre_0_21_presets_as_one_summary_line(self, tmp_path: Path, capsys): store = PresetStore(tmp_path / "presets") @@ -494,7 +497,7 @@ def test_resolves_relative_file_paths_against_the_preset_directory(self, tmp_pat preset.service.files = [FilePathMapping(local_path="service/1/patches", path="/patches")] store.save(preset) - loaded = store.get(preset.id) + loaded = get_or_error(store.get(preset.id)) assert loaded.service.files[0].local_path == str( tmp_path / "presets" / preset.id / "service" / "1" / "patches" @@ -507,7 +510,7 @@ def test_keeps_absolute_file_paths_as_saved(self, tmp_path: Path): preset.service.files = [FilePathMapping(local_path=absolute, path="/patches")] store.save(preset) - assert store.get(preset.id).service.files[0].local_path == absolute + assert get_or_error(store.get(preset.id)).service.files[0].local_path == absolute def test_release_name_keeps_file_paths_relative(self, tmp_path: Path): store = PresetStore(tmp_path / "presets") @@ -535,7 +538,7 @@ def test_preserves_literal_env_values(self, tmp_path: Path): ) store.save(preset) - env = store.get(preset.id).service.env + env = get_or_error(store.get(preset.id)).service.env assert env["TOKENIZERS_PARALLELISM"] == "false" assert env["MODEL_LABEL"] == "monkey" @@ -584,13 +587,19 @@ class TestResolvePresetPrompt: def test_resolves_inline_and_file_relative_to_configuration(self, tmp_path: Path): (tmp_path / "notes.md").write_text("From a file.\n") base = tmp_path - inline = PresetConfiguration(name="q", base="Q/M", prompt="Inline text.") - from_file = PresetConfiguration(name="q", base="Q/M", prompt={"path": "notes.md"}) + inline = PresetConfiguration.model_validate( + {"name": "q", "base": "Q/M", "prompt": "Inline text."} + ) + from_file = PresetConfiguration.model_validate( + {"name": "q", "base": "Q/M", "prompt": {"path": "notes.md"}} + ) assert store_module.resolve_preset_prompt(inline, base) == "Inline text." assert store_module.resolve_preset_prompt(from_file, base) == "From a file." assert ( - store_module.resolve_preset_prompt(PresetConfiguration(name="q", base="Q/M"), base) + store_module.resolve_preset_prompt( + PresetConfiguration.model_validate({"name": "q", "base": "Q/M"}), base + ) is None ) @@ -600,12 +609,16 @@ def test_rejects_missing_and_empty_prompt_files(self, tmp_path: Path): with pytest.raises(ConfigurationError, match="Failed to read"): store_module.resolve_preset_prompt( - PresetConfiguration(name="q", base="Q/M", prompt={"path": "missing.md"}), + PresetConfiguration.model_validate( + {"name": "q", "base": "Q/M", "prompt": {"path": "missing.md"}} + ), base, ) with pytest.raises(ConfigurationError, match="is empty"): store_module.resolve_preset_prompt( - PresetConfiguration(name="q", base="Q/M", prompt={"path": "empty.md"}), + PresetConfiguration.model_validate( + {"name": "q", "base": "Q/M", "prompt": {"path": "empty.md"}} + ), base, ) @@ -617,14 +630,14 @@ def test_finds_and_detaches_names(self, tmp_path: Path): store.save(named) store.save(get_preset().model_copy(update={"id": "01234567"})) - assert store.find_by_name("qwen").id == named.id + assert get_or_error(store.find_by_name("qwen")).id == named.id assert store.find_by_name("other") is None - released = store.release_name("qwen") + released = get_or_error(store.release_name("qwen")) assert released.id == named.id assert store.find_by_name("qwen") is None - assert store.get(named.id).name is None + assert get_or_error(store.get(named.id)).name is None def test_find_by_id_or_name_resolves_a_pulled_qualified_name(self, tmp_path: Path): store = PresetStore(tmp_path / "presets") @@ -636,7 +649,7 @@ def test_find_by_id_or_name_resolves_a_pulled_qualified_name(self, tmp_path: Pat # A qualified name is never id-shaped, so the id lookup must not # reject it before the name lookup runs. - assert store.find_by_id_or_name("main/qwen").id == pulled.id + assert get_or_error(store.find_by_id_or_name("main/qwen")).id == pulled.id assert store.find_by_id_or_name(pulled.id) == store.get(pulled.id) def test_find_by_id_or_name_returns_none_for_an_unknown_qualified_ref(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 26750c7b78..c7a4d9ea80 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -24,6 +24,7 @@ from dstack._internal.core.models.presets import PresetWorkload from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.runs import Run +from dstack._internal.utils.common import get_or_error from tests._internal.cli.common import ( SHARED_PREFIX_WORKLOAD, get_preset, @@ -89,12 +90,14 @@ def test_builds_portable_self_contained_preset(self, tmp_path): preset = build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - gateway="benchmark-gateway", - env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], + preset_configuration=PresetConfiguration.model_validate( + { + "name": "qwen-build", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "gateway": "benchmark-gateway", + "env": ["LICENSE", "TOKENIZERS_PARALLELISM=false"], + } ), report=get_successful_preset_report(run), workspace_path=tmp_path, @@ -113,8 +116,10 @@ def test_builds_portable_self_contained_preset(self, tmp_path): assert all(getattr(preset.service, field) is None for field in ProfileParams.model_fields) assert isinstance(preset.service.env["LICENSE"], EnvSentinel) assert preset.service.env["TOKENIZERS_PARALLELISM"] == "false" - assert preset.service.resources.gpu.vendor.value == "nvidia" - assert preset.verified_on[0].replicas[0].gpu.name == ["A6000"] + gpu = get_or_error(preset.service.resources.gpu) + assert gpu.vendor is not None + assert gpu.vendor.value == "nvidia" + assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["A6000"] def test_rewrites_file_paths_onto_the_mirrored_session_copies(self, tmp_path): # `files` local paths resolve into the agent workspace at submission, and @@ -136,7 +141,9 @@ def test_rewrites_file_paths_onto_the_mirrored_session_copies(self, tmp_path): preset = build_verified_preset( run=run, - preset_configuration=PresetConfiguration(name="qwen-build", base="Qwen/Qwen3.5-27B"), + preset_configuration=PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B"} + ), report=get_successful_preset_report(run), workspace_path=workspace, session_path=session, @@ -163,8 +170,8 @@ def test_rejects_a_file_without_a_mirrored_copy(self, tmp_path): with pytest.raises(CLIError, match="no mirrored copy"): build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", base="Qwen/Qwen3.5-27B" + preset_configuration=PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B"} ), report=get_successful_preset_report(run), workspace_path=workspace, @@ -238,10 +245,8 @@ def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): preset = build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - dataset="spec_bench", + preset_configuration=PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B", "dataset": "spec_bench"} ), report=report, workspace_path=tmp_path, @@ -260,12 +265,14 @@ def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path preset = build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - min_context_length=8192, - gateway="benchmark-gateway", - dataset="spec_bench", + preset_configuration=PresetConfiguration.model_validate( + { + "name": "qwen-build", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "gateway": "benchmark-gateway", + "dataset": "spec_bench", + } ), report=_dataset_report(run, dataset="spec_bench", shared_prefix_tokens=768), workspace_path=tmp_path, @@ -291,10 +298,8 @@ def test_rejects_benchmark_on_a_different_dataset(self, tmp_path, reported): ): build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", - base="Qwen/Qwen3.5-27B", - dataset="spec_bench", + preset_configuration=PresetConfiguration.model_validate( + {"name": "qwen-build", "base": "Qwen/Qwen3.5-27B", "dataset": "spec_bench"} ), report=_dataset_report(run, dataset=reported), workspace_path=tmp_path, @@ -311,12 +316,14 @@ def test_rejects_variant_for_exact_model_request(self, tmp_path): with pytest.raises(CLIError, match="changed an exact model request"): build_verified_preset( run=run, - preset_configuration=PresetConfiguration( - name="qwen-build", - model={ - "repo": "community/Qwen3.5-27B-GPTQ-Int4", - "name": "Qwen/Qwen3.5-27B", - }, + preset_configuration=PresetConfiguration.model_validate( + { + "name": "qwen-build", + "model": { + "repo": "community/Qwen3.5-27B-GPTQ-Int4", + "name": "Qwen/Qwen3.5-27B", + }, + } ), report=report, workspace_path=tmp_path, @@ -345,6 +352,7 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) + assert isinstance(report, PresetAgentSuccess) assert report.benchmark is not None assert report.benchmark.command.endswith("Bearer [redacted]'") assert "sk-live" not in report.benchmark.command @@ -373,6 +381,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) + assert isinstance(report, PresetAgentSuccess) assert report.benchmark is not None assert "bearer header" in report.benchmark.command diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index 7fdb6900ec..099b86238f 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -17,6 +17,7 @@ ) from dstack._internal.cli.utils.sparkline import SPARKS from dstack._internal.core.models.metrics import JobMetrics, Metric +from dstack._internal.utils.common import get_or_error GIB = 1024**3 CAPACITY_GB = 80 @@ -197,7 +198,7 @@ def test_a_sample_may_lag_a_few_intervals_and_still_read_as_live(self, age_secon @pytest.mark.parametrize("state,emphasised", [("running", True), ("terminated", False)]) def test_only_a_live_edge_is_emphasised(self, state: str, emphasised: bool): job, metrics = make_run("saturated", state=state) - axis = _time_axis(60, *_job_window(metrics)) + axis = _time_axis(60, *get_or_error(_job_window(metrics))) styles = {str(span.style) for span in axis.spans} assert ("bold grey58" in styles) == emphasised diff --git a/src/tests/_internal/proxy/gateway/repo/test_repo.py b/src/tests/_internal/proxy/gateway/repo/test_repo.py index 55cc4dbe94..bf95c7060f 100644 --- a/src/tests/_internal/proxy/gateway/repo/test_repo.py +++ b/src/tests/_internal/proxy/gateway/repo/test_repo.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from pydantic import AnyHttpUrl from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig, ModelEntrypoint from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo @@ -16,7 +17,7 @@ async def test_persist_repo(tmp_path: Path) -> None: srv_2 = make_service("proj-2", "run-2", domain="run-2.gtw.test") model_1 = make_model("proj-1", "model-1", run_name="run-1") entrypoint_1 = ModelEntrypoint(project_name="proj-1", domain="gateway.gtw.test", https=True) - config = GlobalProxyConfig(acme_settings=ACMESettings(server="https://acme.test")) + config = GlobalProxyConfig(acme_settings=ACMESettings(server=AnyHttpUrl("https://acme.test"))) file = tmp_path / "state-v2.json" repo = GatewayProxyRepo.load(file) @@ -33,5 +34,8 @@ async def test_persist_repo(tmp_path: Path) -> None: assert await repo.get_project("proj-1") == proj_1 assert await repo.get_project("proj-2") == proj_2 assert await repo.list_entrypoints() == [entrypoint_1] - assert set(await repo.list_services()) == {srv_1, srv_2} + assert sorted(await repo.list_services(), key=lambda s: (s.project_name, s.run_name)) == [ + srv_1, + srv_2, + ] assert await repo.list_models("proj-1") == [model_1] diff --git a/src/tests/_internal/proxy/gateway/repo/test_state_v1.py b/src/tests/_internal/proxy/gateway/repo/test_state_v1.py index 7856bea2d5..df118ec7ca 100644 --- a/src/tests/_internal/proxy/gateway/repo/test_state_v1.py +++ b/src/tests/_internal/proxy/gateway/repo/test_state_v1.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from pydantic import AnyHttpUrl from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig, ModelEntrypoint @@ -280,18 +281,18 @@ async def test_migrate_from_state_v1(tmp_path: Path) -> None: repo = GatewayProxyRepo.load(v2_file) assert await repo.get_config() == GlobalProxyConfig( acme_settings=ACMESettings( - server="https://acme.test/", + server=AnyHttpUrl("https://acme.test/"), eab_kid="test_eab_kid", eab_hmac_key="test_eab_hmac_key", ) ) assert await repo.get_project("proj-1") == Project(name="proj-1", ssh_private_key="test key 1") assert await repo.get_project("proj-2") == Project(name="proj-2", ssh_private_key="test key 2") - assert set(await repo.list_entrypoints()) == { + assert sorted(await repo.list_entrypoints(), key=lambda e: (e.project_name, e.domain)) == [ ModelEntrypoint(project_name="proj-1", domain="gateway.proj-1.gateway.test", https=True), ModelEntrypoint(project_name="proj-2", domain="gateway.proj-2.gateway.test", https=True), - } - assert set(await repo.list_services()) == { + ] + assert sorted(await repo.list_services(), key=lambda s: (s.project_name, s.run_name)) == [ Service( project_name="proj-1", run_name="run-1", @@ -362,7 +363,7 @@ async def test_migrate_from_state_v1(tmp_path: Path) -> None: ), ), ), - } + ] assert await repo.list_models("proj-1") == [ ChatModel( project_name="proj-1", diff --git a/src/tests/_internal/proxy/gateway/services/test_server_client.py b/src/tests/_internal/proxy/gateway/services/test_server_client.py index f387270c75..78aa71ff0f 100644 --- a/src/tests/_internal/proxy/gateway/services/test_server_client.py +++ b/src/tests/_internal/proxy/gateway/services/test_server_client.py @@ -1,6 +1,7 @@ import asyncio from pathlib import Path from tempfile import TemporaryDirectory +from typing import cast import httpcore2 as httpcore import httpx2 as httpx @@ -19,7 +20,7 @@ async def test_recovers_after_cancelled_pool_waiter(self, tmp_path, monkeypatch, (tmp_path / "server.sock").touch() client = HTTPMultiClient(tmp_path) cached = next(client._iter_clients_rand()) - pool = cached.client._transport._pool + pool = cast(httpx.AsyncHTTPTransport, cached.client._transport)._pool assert isinstance(pool, httpcore.AsyncConnectionPool) pool._max_connections = 1 # Closing the first response assigns a fresh connection to the queued request. @@ -117,7 +118,7 @@ async def handle(reader, writer): # A short path is needed for macOS's Unix-socket pathname limit. with TemporaryDirectory() as directory: socket = Path(directory) / "server.sock" - server = await asyncio.start_unix_server(handle, path=socket) + server = await asyncio.start_unix_server(handle, path=socket) # pyright: ignore[reportAttributeAccessIssue] client = HTTPMultiClient(Path(directory)) try: response = await client.post( diff --git a/src/tests/_internal/proxy/gateway/test_auth.py b/src/tests/_internal/proxy/gateway/test_auth.py index 1c5e2a5ec2..3b96306aa4 100644 --- a/src/tests/_internal/proxy/gateway/test_auth.py +++ b/src/tests/_internal/proxy/gateway/test_auth.py @@ -3,6 +3,7 @@ import httpx2 as httpx import pytest import pytest_asyncio +from aiocache.base import BaseCache from dstack._internal.proxy.gateway.auth import GatewayProxyAuthProvider from dstack._internal.proxy.gateway.services.server_client import HTTPMultiClient @@ -11,7 +12,7 @@ @pytest_asyncio.fixture(autouse=True) async def clear_auth_cache(): - cache = GatewayProxyAuthProvider.is_project_member.cache + cache = _get_auth_cache() await cache.clear() yield await cache.clear() @@ -27,7 +28,7 @@ def handle(request): requests.append(request) return httpx.Response(status) - cache = GatewayProxyAuthProvider.is_project_member.cache + cache = _get_auth_cache() set_value = AsyncMock(wraps=cache.set) monkeypatch.setattr(cache, "set", set_value) async with httpx.AsyncClient( @@ -77,3 +78,8 @@ async def test_no_servers_reaches_auth_error_boundary(self, tmp_path): await provider.is_project_member("test", "token") assert isinstance(exc.value.__cause__, httpx.RequestError) assert exc.value.__cause__.request.url.path == "/api/projects/test/get" + + +def _get_auth_cache() -> BaseCache: + # `aiocache.cached` sets `cache` on the wrapper at runtime, but its type doesn't declare it + return GatewayProxyAuthProvider.is_project_member.cache # pyright: ignore[reportAttributeAccessIssue] diff --git a/src/tests/_internal/proxy/lib/routers/test_model_proxy.py b/src/tests/_internal/proxy/lib/routers/test_model_proxy.py index d044f87e02..a2dfa3257c 100644 --- a/src/tests/_internal/proxy/lib/routers/test_model_proxy.py +++ b/src/tests/_internal/proxy/lib/routers/test_model_proxy.py @@ -2,7 +2,7 @@ from typing import AsyncIterator, Generator from unittest.mock import patch -import httpx +import httpx2 as httpx import openai import pytest from fastapi import FastAPI @@ -194,7 +194,9 @@ async def test_chat_completions_stream(mock_chat_client) -> None: ) completion = "" async for chunk in response: - completion += chunk.choices[0].delta.content + content = chunk.choices[0].delta.content + assert content is not None + completion += content assert completion == SAMPLE_RESPONSE diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py index e2db9f4254..3b6659729a 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py @@ -34,11 +34,11 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): config1 = get_volume_configuration( name="test-volume", - auto_cleanup_duration="1h", + auto_cleanup_duration=3600, ) config2 = get_volume_configuration( name="test-volume", - auto_cleanup_duration="3h", + auto_cleanup_duration=10800, ) volume1 = await create_volume( session=session, @@ -101,7 +101,7 @@ async def test_deletes_idle_volume_with_null_auto_cleanup_enabled( backend=BackendType.AWS, configuration=get_volume_configuration( name="test-volume", - auto_cleanup_duration="1h", + auto_cleanup_duration=3600, ), volume_provisioning_data=get_volume_provisioning_data(), last_job_processed_at=datetime.datetime.now(datetime.timezone.utc) diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_probes.py b/src/tests/_internal/server/background/scheduled_tasks/test_probes.py index f441046476..b0259ee33c 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_probes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_probes.py @@ -112,8 +112,8 @@ async def test_schedules_probe_execution(self, test_db, session: AsyncSession) - port=80, image="nginx", probes=[ - ProbeConfig(type="http", url="/1", timeout="1m"), - ProbeConfig(type="http", url="/2", timeout="2m"), + ProbeConfig.model_validate({"type": "http", "url": "/1", "timeout": "1m"}), + ProbeConfig.model_validate({"type": "http", "url": "/2", "timeout": "2m"}), ], ), ), diff --git a/src/tests/_internal/server/compatibility/test_runs.py b/src/tests/_internal/server/compatibility/test_runs.py index 2a02c66dbe..f670e1861d 100644 --- a/src/tests/_internal/server/compatibility/test_runs.py +++ b/src/tests/_internal/server/compatibility/test_runs.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import Any, Optional, Union, cast import pytest from packaging.version import Version @@ -61,8 +61,11 @@ def test_downgrades_groups_for_clients_without_them(self, client_version): patch_run_spec(run_spec, client_version) configuration = run_spec.configuration + assert isinstance(configuration, ServiceConfiguration) assert configuration.groups is None - assert [group["count"] for group in configuration.replicas] == [ + # Old clients get `replicas` as a list of groups, which the model doesn't type + legacy_groups = cast(list[dict[str, Any]], configuration.replicas) + assert [group["count"] for group in legacy_groups] == [ {"min": 1, "max": 1}, {"min": 2, "max": 2}, ] @@ -76,7 +79,9 @@ def test_keeps_groups_for_clients_that_support_them(self, client_version): patch_run_spec(run_spec, client_version) + assert isinstance(run_spec.configuration, ServiceConfiguration) assert run_spec.configuration.replicas is None + assert run_spec.configuration.groups is not None assert [group.replicas for group in run_spec.configuration.groups] == [ Range[int](min=1, max=1), Range[int](min=2, max=2), @@ -90,6 +95,7 @@ def test_leaves_a_service_without_groups_alone(self): patch_run_spec(run_spec, Version("0.21.0")) + assert isinstance(run_spec.configuration, ServiceConfiguration) assert run_spec.configuration.groups is None assert run_spec.configuration.replicas == Range[int](min=2, max=2) diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index 4182126199..90c1c9bd3f 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -1245,6 +1245,7 @@ async def test_updates_ssh_fleet(self, test_db, session: AsyncSession, client: A current_spec = get_fleet_spec(conf=current_conf) spec = current_spec.model_copy(deep=True) # 10.0.0.100 removed, 10.0.0.101 added + assert spec.configuration.ssh_config is not None spec.configuration.ssh_config.hosts = ["10.0.0.101"] fleet = await create_fleet(session=session, project=project, spec=current_spec) @@ -2357,9 +2358,11 @@ async def test_returns_update_plan_for_existing_fleet( conf = get_ssh_fleet_configuration(hosts=["10.0.0.100"]) spec = get_fleet_spec(conf=conf) effective_spec = spec.model_copy(deep=True) + assert effective_spec.configuration.ssh_config is not None effective_spec.configuration.ssh_config.ssh_key = None current_spec = spec.model_copy(deep=True) # `hosts` can be updated in-place + assert current_spec.configuration.ssh_config is not None current_spec.configuration.ssh_config.hosts = ["10.0.0.100", "10.0.0.101"] fleet = await create_fleet(session=session, project=project, spec=current_spec) @@ -2480,6 +2483,7 @@ async def test_returns_create_plan_for_existing_fleet( conf = get_ssh_fleet_configuration(placement=InstanceGroupPlacement.ANY) spec = get_fleet_spec(conf=conf) effective_spec = spec.model_copy(deep=True) + assert effective_spec.configuration.ssh_config is not None effective_spec.configuration.ssh_config.ssh_key = None current_spec = spec.model_copy(deep=True) # `placement` cannot be updated in-place diff --git a/src/tests/_internal/server/routers/test_gpus.py b/src/tests/_internal/server/routers/test_gpus.py index 09fee1c0e7..82729f171b 100644 --- a/src/tests/_internal/server/routers/test_gpus.py +++ b/src/tests/_internal/server/routers/test_gpus.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from unittest.mock import Mock, patch import gpuhunt @@ -19,7 +19,7 @@ from dstack._internal.core.models.resources import GPUSpec, Range, ResourcesSpec from dstack._internal.core.models.runs import RunSpec from dstack._internal.core.models.users import GlobalRole, ProjectRole -from dstack._internal.server.models import FleetModel, ProjectModel +from dstack._internal.server.models import DecryptedString, FleetModel, ProjectModel from dstack._internal.server.services import gpus as gpus_services from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( @@ -136,7 +136,7 @@ def create_mock_backends_with_offers( async def call_gpus_api( client: AsyncClient, project_name: str, - user_token: str, + user_token: Union[DecryptedString, str], run_spec: RunSpec, group_by: Optional[List[str]] = None, full_offers: Optional[bool] = None, @@ -144,7 +144,7 @@ async def call_gpus_api( reverse: Optional[bool] = None, ): """Helper to call the GPUs API with standard parameters.""" - json_data = {"run_spec": run_spec.model_dump()} + json_data: Dict[str, Any] = {"run_spec": run_spec.model_dump()} if group_by is not None: json_data["group_by"] = group_by if full_offers is not None: @@ -553,7 +553,9 @@ async def test_uses_targeted_instance_offers_when_instances_specified( run_spec = get_run_spec( run_name="test-run", repo_id=repo.name, - profile=Profile(name="default", instances=["targeted-instance"]), + profile=Profile.model_validate( + {"name": "default", "instances": ["targeted-instance"]} + ), ) offers_by_backend = { diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 8d64936c19..70cd675155 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -32,7 +32,7 @@ InstanceType, Resources, ) -from dstack._internal.core.models.profiles import Profile, Schedule +from dstack._internal.core.models.profiles import FleetReferenceOrShorthand, Profile, Schedule from dstack._internal.core.models.resources import GPUSpec, Range, ResourcesSpec from dstack._internal.core.models.runs import ( ApplyRunPlanInput, @@ -1308,7 +1308,7 @@ async def test_returns_run_with_job_connection_info_dev_environment( run_spec = get_run_spec( repo_id=repo.name, run_name="dev-env", - configuration=DevEnvironmentConfiguration(ide=ide), + configuration=DevEnvironmentConfiguration.model_validate({"ide": ide}), ) run = await create_run( session=session, @@ -1936,16 +1936,16 @@ async def test_service_with_two_replica_groups_returns_two_job_plans( configuration=ServiceConfiguration( port=8080, gateway=False, - replicas=[ + groups=[ ReplicaGroup( name="gpu-group", - count=Range[int](min=2, max=2), + replicas=Range[int](min=2, max=2), resources=ResourcesSpec(gpu=GPUSpec()), commands=["python server.py"], ), ReplicaGroup( name="cpu-group", - count=Range[int](min=1, max=1), + replicas=Range[int](min=1, max=1), resources=ResourcesSpec(gpu=None), commands=["python router.py"], ), @@ -2000,14 +2000,14 @@ async def test_service_reservation_group_filters_backends_by_reservation_support port=8080, gateway=False, image="nginx", - replicas=[ + groups=[ ReplicaGroup( name="reserved-group", - count=Range[int](min=1, max=1), + replicas=Range[int](min=1, max=1), reservation="my-reservation-id", ), ReplicaGroup( - count=Range[int](min=1, max=1), + replicas=Range[int](min=1, max=1), name="unreserved-group", ), ], @@ -2927,7 +2927,7 @@ async def test_offer_with_fleets_uses_selected_fleet_offer_collection( project_role=ProjectRole.USER, ) repo = await create_repo(session=session, project_id=project.id) - selected_fleets = ["fleet-a", "fleet-b"] + selected_fleets: list[FleetReferenceOrShorthand] = ["fleet-a", "fleet-b"] run_spec = get_run_spec( repo_id=repo.name, profile=Profile(name="default", fleets=selected_fleets), @@ -3175,7 +3175,9 @@ async def test_collects_offers_only_if_requested_by_for_offers_only( ), pytest.param( DevEnvironmentConfiguration(ide="vscode", inactivity_duration=False), - DevEnvironmentConfiguration(ide="vscode", inactivity_duration="30m"), + DevEnvironmentConfiguration.model_validate( + {"ide": "vscode", "inactivity_duration": "30m"} + ), "update", id="update-dev-env", ), @@ -3592,6 +3594,7 @@ async def test_updates_run(self, test_db, session: AsyncSession, client: AsyncCl ) run = run_model_to_run(run_model) run_spec.configuration_path = "new.dstack.yml" + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.replicas = Range[int](min=2, max=2) response = await client.post( f"/api/project/{project.name}/runs/apply", @@ -3614,7 +3617,9 @@ async def test_updates_run(self, test_db, session: AsyncSession, client: AsyncCl assert updated_run.deployment_num == 1 assert run.run_spec.configuration_path == "old.dstack.yml" assert updated_run.run_spec.configuration_path == "new.dstack.yml" + assert isinstance(run.run_spec.configuration, ServiceConfiguration) assert run.run_spec.configuration.replicas == Range(min=1, max=1) + assert isinstance(updated_run.run_spec.configuration, ServiceConfiguration) assert updated_run.run_spec.configuration.replicas == Range(min=2, max=2) events = await list_events(session) assert len(events) == 1 @@ -3817,6 +3822,7 @@ async def test_assigns_to_specified_gateway_on_field_change( await create_gateway_replica(session=session, backend=backend, gateway_id=gateway.id) run_model, run_spec = await self._create_service_run(session, project, repo, user) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = "my-gateway" response = await self._apply(client, project, user, run_model, run_spec) @@ -3860,6 +3866,7 @@ async def test_reassigns_between_specific_gateways_on_field_change( run_model, run_spec = await self._create_service_run( session, project, repo, user, gateway=old_gateway, gateway_field="old-gateway" ) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = "new-gateway" response = await self._apply(client, project, user, run_model, run_spec) @@ -3892,6 +3899,7 @@ async def test_unassigns_gateway_when_field_changes_to_false( run_model, run_spec = await self._create_service_run( session, project, repo, user, gateway=gateway, gateway_field="my-gateway" ) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = False response = await self._apply(client, project, user, run_model, run_spec) @@ -3930,6 +3938,7 @@ async def test_reassigns_to_default_gateway_when_field_changes_to_true( run_model, run_spec = await self._create_service_run( session, project, repo, user, gateway_field=False ) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = True response = await self._apply(client, project, user, run_model, run_spec) @@ -3974,6 +3983,7 @@ async def test_reassigns_to_default_gateway_when_field_changes_to_true_even_if_a run_model, run_spec = await self._create_service_run( session, project, repo, user, gateway=gateway, gateway_field="my-gateway" ) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = True response = await self._apply(client, project, user, run_model, run_spec) @@ -4041,6 +4051,7 @@ async def test_rejects_update_to_nonexistent_gateway( repo = await create_repo(session=session, project_id=project.id) run_model, run_spec = await self._create_service_run(session, project, repo, user) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = "nonexistent-gateway" response = await self._apply(client, project, user, run_model, run_spec) @@ -4062,6 +4073,7 @@ async def test_rejects_field_change_to_true_when_no_default_gateway( run_model, run_spec = await self._create_service_run( session, project, repo, user, gateway_field=False ) + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = True response = await self._apply(client, project, user, run_model, run_spec) @@ -4087,6 +4099,7 @@ async def test_no_reassignment_when_resolved_gateway_is_unchanged( session, project, repo, user, gateway_field=None ) assert run_model.service_spec is None + assert isinstance(run_spec.configuration, ServiceConfiguration) run_spec.configuration.gateway = False response = await self._apply(client, project, user, run_model, run_spec) diff --git a/src/tests/_internal/server/services/jobs/configurators/test_base.py b/src/tests/_internal/server/services/jobs/configurators/test_base.py index 0b4fb2cbec..73d797494b 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_base.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_base.py @@ -1,5 +1,3 @@ -from typing import Union - import pytest from dstack._internal.core.errors import ServerClientError @@ -37,7 +35,7 @@ class TestInterpolateJobVolumes: ) def test_interpolates_volumes( self, - run_volumes: list[Union[MountPoint, str]], + run_volumes: list[MountPoint], job_num: int, job_volumes: list[MountPoint], ): @@ -65,7 +63,7 @@ def test_interpolates_volumes( ) def test_raises_server_client_error( self, - run_volumes: list[Union[MountPoint, str]], + run_volumes: list[MountPoint], job_num: int, ): with pytest.raises(ServerClientError): diff --git a/src/tests/_internal/server/services/jobs/configurators/test_service.py b/src/tests/_internal/server/services/jobs/configurators/test_service.py index a13b24701a..0fc1156a80 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_service.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_service.py @@ -169,10 +169,10 @@ async def test_no_probe_when_no_model(self): assert len(job_specs[0].probes) == 0 -def _make_run_spec(replicas, **service_kwargs): +def _make_run_spec(groups: list[ReplicaGroup], **service_kwargs): configuration = ServiceConfiguration( port=80, - replicas=replicas, + groups=groups, **service_kwargs, ) return get_run_spec(run_name="run", repo_id="id", configuration=configuration) @@ -186,10 +186,10 @@ class TestPerGroupOverrides: async def test_image_name_uses_group_image(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), image="custom:1.0", commands=["x"], ) @@ -200,10 +200,10 @@ async def test_image_name_uses_group_image(self): async def test_image_name_uses_dind_when_group_docker_true(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), docker=True, commands=["x"], ) @@ -214,10 +214,10 @@ async def test_image_name_uses_dind_when_group_docker_true(self): async def test_image_name_uses_nvcc_default_when_group_nvcc_true(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), nvcc=True, commands=["x"], ) @@ -229,10 +229,10 @@ async def test_image_name_uses_nvcc_default_when_group_nvcc_true(self): async def test_image_name_falls_back_to_service_image(self): run_spec = _make_run_spec( image="svc:1.0", - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], ) ], @@ -242,10 +242,10 @@ async def test_image_name_falls_back_to_service_image(self): async def test_privileged_true_when_group_docker(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), docker=True, commands=["x"], ) @@ -256,10 +256,10 @@ async def test_privileged_true_when_group_docker(self): async def test_privileged_returns_group_privileged(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), image="x", privileged=True, commands=["x"], @@ -272,10 +272,10 @@ async def test_privileged_returns_group_privileged(self): async def test_privileged_defers_to_super_when_group_unset(self): run_spec = _make_run_spec( image="svc:1.0", - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], ) ], @@ -286,10 +286,10 @@ async def test_privileged_defers_to_super_when_group_unset(self): async def test_dstack_image_commands_injects_start_dockerd_for_docker(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), docker=True, commands=["x"], ) @@ -300,10 +300,10 @@ async def test_dstack_image_commands_injects_start_dockerd_for_docker(self): async def test_dstack_image_commands_empty_for_group_image(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), image="alpine", commands=["x"], ) @@ -314,10 +314,10 @@ async def test_dstack_image_commands_empty_for_group_image(self): async def test_shell_bash_when_group_docker(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), docker=True, commands=["x"], ) @@ -328,10 +328,10 @@ async def test_shell_bash_when_group_docker(self): async def test_shell_sh_when_group_image(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), image="alpine", commands=["x"], ) @@ -342,10 +342,10 @@ async def test_shell_sh_when_group_image(self): async def test_python_uses_group_python(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), python=PythonVersion.PY312, commands=["x"], ) @@ -363,10 +363,10 @@ async def test_user_looks_up_group_image(self, monkeypatch: pytest.MonkeyPatch): Mock(return_value=(image_config, {gpuhunt.CPUArchitecture.X86})), ) run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), image="nginxinc/nginx-unprivileged", commands=["x"], ) @@ -385,10 +385,10 @@ async def test_user_does_not_lookup_for_group_docker(self, monkeypatch: pytest.M mock_get_image_config, ) run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), docker=True, commands=["x"], ) @@ -400,10 +400,10 @@ async def test_user_does_not_lookup_for_group_docker(self, monkeypatch: pytest.M async def test_spot_policy_uses_group_value(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], spot_policy=SpotPolicy.SPOT, ) @@ -414,10 +414,10 @@ async def test_spot_policy_uses_group_value(self): async def test_spot_policy_defaults_to_ondemand_when_group_unset(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], ) ], @@ -427,16 +427,16 @@ async def test_spot_policy_defaults_to_ondemand_when_group_unset(self): async def test_different_groups_different_spot_policies(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="spot", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], spot_policy=SpotPolicy.SPOT, ), ReplicaGroup( name="od", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["y"], spot_policy=SpotPolicy.ONDEMAND, ), @@ -453,10 +453,10 @@ async def test_different_groups_different_spot_policies(self): async def test_reservation_uses_group_value(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], reservation="my-reservation", ) @@ -467,10 +467,10 @@ async def test_reservation_uses_group_value(self): async def test_reservation_defaults_to_none_when_group_unset(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], ) ], @@ -480,16 +480,16 @@ async def test_reservation_defaults_to_none_when_group_unset(self): async def test_different_groups_different_reservations(self): run_spec = _make_run_spec( - replicas=[ + groups=[ ReplicaGroup( name="a", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["x"], reservation="res-a", ), ReplicaGroup( name="b", - count=Range(min=1, max=1), + replicas=Range(min=1, max=1), commands=["y"], reservation="res-b", ), diff --git a/src/tests/_internal/server/services/jobs/configurators/test_task.py b/src/tests/_internal/server/services/jobs/configurators/test_task.py index 74d2d83a7c..723c06a68c 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_task.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_task.py @@ -3,7 +3,8 @@ import pytest -from dstack._internal.core.models.configurations import NodeGroup, TaskConfiguration +from dstack._internal.core.models.configurations import NodeGroup, PythonVersion, TaskConfiguration +from dstack._internal.core.models.envs import Env from dstack._internal.core.models.resources import GPUSpec, ResourcesSpec from dstack._internal.core.models.runs import JobSSHKey from dstack._internal.server.services.docker import ImageConfig @@ -74,13 +75,17 @@ async def test_uses_per_group_commands_and_resources(self): name="head", nodes=1, commands=["echo head"], - resources=ResourcesSpec(gpu=GPUSpec(name=["H100"], count=1)), + resources=ResourcesSpec( + gpu=GPUSpec.model_validate({"name": ["H100"], "count": 1}) + ), ), NodeGroup( name="workers", nodes=1, commands=["echo worker"], - resources=ResourcesSpec(gpu=GPUSpec(name=["A100"], count=2)), + resources=ResourcesSpec( + gpu=GPUSpec.model_validate({"name": ["A100"], "count": 2}) + ), ), ], ) @@ -91,16 +96,20 @@ async def test_uses_per_group_commands_and_resources(self): assert "echo head" in job_specs[0].commands[-1] assert "echo worker" in job_specs[1].commands[-1] - assert job_specs[0].requirements.resources.gpu.name == ["H100"] - assert job_specs[0].requirements.resources.gpu.count.min == 1 - assert job_specs[1].requirements.resources.gpu.name == ["A100"] - assert job_specs[1].requirements.resources.gpu.count.min == 2 + head_gpu = job_specs[0].requirements.resources.gpu + assert head_gpu is not None + assert head_gpu.name == ["H100"] + assert head_gpu.count.min == 1 + worker_gpu = job_specs[1].requirements.resources.gpu + assert worker_gpu is not None + assert worker_gpu.name == ["A100"] + assert worker_gpu.count.min == 2 async def test_group_without_resources_does_not_inherit_top_level(self): """Same as replica groups: omitted group resources → ResourcesSpec(), not top-level.""" configuration = TaskConfiguration( image="debian", - resources=ResourcesSpec(gpu=GPUSpec(name=["H100"], count=1)), + resources=ResourcesSpec(gpu=GPUSpec.model_validate({"name": ["H100"], "count": 1})), groups=[ NodeGroup(name="head", nodes=1, commands=["echo head"]), ], @@ -110,7 +119,9 @@ async def test_group_without_resources_does_not_inherit_top_level(self): job_specs = await configurator.get_job_specs(replica_num=0) - assert job_specs[0].requirements.resources.gpu.name is None + gpu = job_specs[0].requirements.resources.gpu + assert gpu is not None + assert gpu.name is None @pytest.mark.asyncio @@ -142,9 +153,7 @@ async def test_preserves_explicit_transport_env(self): image="debian", commands=["true"], dstack=True, - env={ - "DSTACK_SERVER_URL": "https://server.example.com", - }, + env=Env.model_validate({"DSTACK_SERVER_URL": "https://server.example.com"}), ) run_spec = get_run_spec(run_name="run", repo_id="id", configuration=configuration) configurator = TaskJobConfigurator(run_spec) @@ -205,7 +214,9 @@ async def test_with_commands_and_image(self, shell: Optional[str], expected_shel ], ) async def test_with_commands_no_image(self, shell: Optional[str], expected_shell: str): - configuration = TaskConfiguration(python="3.12", commands=["sleep inf"], shell=shell) + configuration = TaskConfiguration( + python=PythonVersion.PY312, commands=["sleep inf"], shell=shell + ) run_spec = get_run_spec(run_name="run", repo_id="id", configuration=configuration) configurator = TaskJobConfigurator(run_spec) diff --git a/src/tests/_internal/server/services/jobs/test_jobs.py b/src/tests/_internal/server/services/jobs/test_jobs.py index 96b127e823..a2524de11f 100644 --- a/src/tests/_internal/server/services/jobs/test_jobs.py +++ b/src/tests/_internal/server/services/jobs/test_jobs.py @@ -10,7 +10,7 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.repos.local import LocalRunRepoData from dstack._internal.core.models.resources import ResourcesSpec -from dstack._internal.core.models.runs import JobSpec, RunSpec +from dstack._internal.core.models.runs import JobSpec, Requirements, RunSpec from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration from dstack._internal.server.services.docker import ImageConfig from dstack._internal.server.services.jobs import ( @@ -138,7 +138,7 @@ def _job_spec(self, **requirements_overrides) -> JobSpec: commands=["sleep infinity"], env={}, image_name="ubuntu", - requirements={"resources": resources}, + requirements=Requirements.model_validate({"resources": resources}), ) def test_identical_specs(self): diff --git a/src/tests/_internal/server/services/jobs/test_server_connection.py b/src/tests/_internal/server/services/jobs/test_server_connection.py index 260feb893c..438df71889 100644 --- a/src/tests/_internal/server/services/jobs/test_server_connection.py +++ b/src/tests/_internal/server/services/jobs/test_server_connection.py @@ -158,7 +158,7 @@ async def test_replaces_tunnel_with_stale_socket(self, tunnel_mock): async def test_stale_probe_socket_is_not_reachable(self, tunnel_mock): job = Mock(id=uuid.uuid4()) connection = JobServerConnection(job, job_runtime_data=None) - stale_socket = socket.socket(socket.AF_UNIX) + stale_socket = socket.socket(socket.AF_UNIX) # pyright: ignore[reportAttributeAccessIssue] stale_socket.bind(str(connection._probe_socket_path)) stale_socket.close() @@ -175,7 +175,7 @@ async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): await writer.drain() writer.close() - server = await asyncio.start_unix_server(handle, path=connection._probe_socket_path) + server = await asyncio.start_unix_server(handle, path=connection._probe_socket_path) # pyright: ignore[reportAttributeAccessIssue] try: assert await connection._server_is_reachable() finally: diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index e55ff6de67..a4c9cf2621 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -217,7 +217,7 @@ async def test_plans_each_node_group_with_its_own_requirements( nodes=2, commands=["echo prefill"], resources=ResourcesSpec( - gpu=GPUSpec(name=["L40S"], count=1), + gpu=GPUSpec.model_validate({"name": ["L40S"], "count": 1}), ), ), ], @@ -286,16 +286,18 @@ async def test_plans_each_replica_group_with_its_own_requirements( configuration=ServiceConfiguration( port=8080, gateway=False, - replicas=[ + groups=[ ReplicaGroup( name="gpu-group", - count=Range[int](min=1, max=1), - resources=ResourcesSpec(gpu=GPUSpec(name=["L40S"], count=1)), + replicas=Range[int](min=1, max=1), + resources=ResourcesSpec( + gpu=GPUSpec.model_validate({"name": ["L40S"], "count": 1}) + ), commands=["python server.py"], ), ReplicaGroup( name="cpu-group", - count=Range[int](min=1, max=1), + replicas=Range[int](min=1, max=1), resources=ResourcesSpec(cpu=CPUSpec(count=Range[int](min=4, max=4))), commands=["python router.py"], ), @@ -849,7 +851,6 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el assert offers == [(backend, offer)] get_offers_by_requirements_mock.assert_awaited_once() - assert ( - get_offers_by_requirements_mock.await_args.kwargs["master_job_provisioning_data"] - is None - ) + await_args = get_offers_by_requirements_mock.await_args + assert await_args is not None + assert await_args.kwargs["master_job_provisioning_data"] is None diff --git a/src/tests/_internal/server/services/runs/test_router_worker_sync.py b/src/tests/_internal/server/services/runs/test_router_worker_sync.py index 7c411fd829..f494592daa 100644 --- a/src/tests/_internal/server/services/runs/test_router_worker_sync.py +++ b/src/tests/_internal/server/services/runs/test_router_worker_sync.py @@ -3,10 +3,10 @@ import json import logging import uuid -from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence from contextlib import asynccontextmanager, contextmanager from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union, cast from unittest.mock import AsyncMock, patch import grpc @@ -28,6 +28,7 @@ from dstack._internal.server.services.runs import router_worker_sync from dstack._internal.server.services.runs.router_worker_sync import ( _add_worker_to_router, + _CurrentWorker, _get_current_worker_job_id, _get_router_workers, _get_worker_jobs_with_addresses, @@ -441,7 +442,11 @@ async def test_returns_running_worker_jobs_with_addresses( class TestGetCurrentWorkerJobId: def test_returns_job_id_from_label(self): job_id = uuid.uuid4() - worker = {"id": "1", "url": "http://10.0.0.1:8000", "labels": {_JOB_ID_LABEL: str(job_id)}} + worker: _CurrentWorker = { + "id": "1", + "url": "http://10.0.0.1:8000", + "labels": {_JOB_ID_LABEL: str(job_id)}, + } assert _get_current_worker_job_id(worker) == job_id @pytest.mark.parametrize( @@ -942,7 +947,7 @@ class TestProbeSglangGrpcWorker: ) async def test_reports_worker(self, server_args: dict, expected: dict): response = sglang_scheduler_pb2.GetServerInfoResponse(server_args=server_args) - channel = _FakeGrpcChannel({_SGLANG_GET_SERVER_INFO: response}) + channel = _fake_grpc_channel({_SGLANG_GET_SERVER_INFO: response}) worker = await _probe_sglang_grpc_worker( channel, log_prefix="Worker", address="10.0.0.1:8000" @@ -973,7 +978,7 @@ async def test_returns_none_on_expected_error( self, caplog: pytest.LogCaptureFixture, code: grpc.StatusCode ): caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) - channel = _FakeGrpcChannel({_SGLANG_GET_SERVER_INFO: _rpc_error(code)}) + channel = _fake_grpc_channel({_SGLANG_GET_SERVER_INFO: _rpc_error(code)}) assert ( await _probe_sglang_grpc_worker(channel, log_prefix="Worker", address="10.0.0.1:8000") is None @@ -981,7 +986,7 @@ async def test_returns_none_on_expected_error( assert not [r for r in caplog.records if r.levelno > logging.DEBUG] async def test_warns_on_unexpected_error(self, caplog: pytest.LogCaptureFixture): - channel = _FakeGrpcChannel( + channel = _fake_grpc_channel( {_SGLANG_GET_SERVER_INFO: _rpc_error(grpc.StatusCode.PERMISSION_DENIED)} ) assert ( @@ -1013,7 +1018,7 @@ async def test_returns_none_on_invalid_server_args( self, caplog: pytest.LogCaptureFixture, server_args: dict ): response = sglang_scheduler_pb2.GetServerInfoResponse(server_args=server_args) - channel = _FakeGrpcChannel({_SGLANG_GET_SERVER_INFO: response}) + channel = _fake_grpc_channel({_SGLANG_GET_SERVER_INFO: response}) assert ( await _probe_sglang_grpc_worker(channel, log_prefix="Worker", address="10.0.0.1:8000") is None @@ -1062,7 +1067,7 @@ async def test_reports_worker(self, kv_role: str, kv_connector: str, expected: d response = vllm_engine_pb2.GetServerInfoResponse( kv_role=kv_role, kv_connector=kv_connector ) - channel = _FakeGrpcChannel({_VLLM_GET_SERVER_INFO: response}) + channel = _fake_grpc_channel({_VLLM_GET_SERVER_INFO: response}) worker = await _probe_vllm_grpc_worker( channel, log_prefix="Worker", address="10.0.0.1:8000" @@ -1090,7 +1095,7 @@ async def test_returns_none_on_expected_error( self, caplog: pytest.LogCaptureFixture, code: grpc.StatusCode ): caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) - channel = _FakeGrpcChannel({_VLLM_GET_SERVER_INFO: _rpc_error(code)}) + channel = _fake_grpc_channel({_VLLM_GET_SERVER_INFO: _rpc_error(code)}) assert ( await _probe_vllm_grpc_worker(channel, log_prefix="Worker", address="10.0.0.1:8000") is None @@ -1098,7 +1103,7 @@ async def test_returns_none_on_expected_error( assert not [r for r in caplog.records if r.levelno > logging.DEBUG] async def test_warns_on_unexpected_error(self, caplog: pytest.LogCaptureFixture): - channel = _FakeGrpcChannel( + channel = _fake_grpc_channel( {_VLLM_GET_SERVER_INFO: _rpc_error(grpc.StatusCode.PERMISSION_DENIED)} ) assert ( @@ -1198,9 +1203,11 @@ def _job_id_label(job: JobModel) -> dict[str, str]: return {_JOB_ID_LABEL: str(job.id)} -def _router_entry(worker_id: str, url: str, labels: Optional[dict[str, str]] = None) -> dict: +def _router_entry( + worker_id: str, url: str, labels: Optional[dict[str, str]] = None +) -> _CurrentWorker: """A worker as listed by the router `GET /workers`, with the fields the sync reads""" - entry: dict = {"id": worker_id, "url": url} + entry: _CurrentWorker = {"id": worker_id, "url": url} if labels is not None: entry["labels"] = labels return entry @@ -1211,12 +1218,12 @@ class _FakeRouter: def __init__( self, - workers: Optional[list[dict]] = None, + workers: Optional[Sequence[Mapping[str, Any]]] = None, *, connect_error: Optional[Exception] = None, update_error: Optional[Exception] = None, ): - self.workers = list(workers or []) + self.workers: list[Mapping[str, Any]] = list(workers or []) self.connect_error = connect_error self.update_error = update_error self.connections = 0 @@ -1335,6 +1342,18 @@ def _grpc_server_http_handler(request: httpx.Request) -> httpx.Response: raise httpx.RemoteProtocolError("illegal request line") +def _fake_grpc_channel( + responses: Optional[Mapping[str, Union[Message, grpc.aio.AioRpcError]]] = None, + *, + unknown_method_code: grpc.StatusCode = grpc.StatusCode.UNIMPLEMENTED, +) -> grpc.aio.Channel: + """A `_FakeGrpcChannel` passed where the probes expect a real channel""" + return cast( + grpc.aio.Channel, + _FakeGrpcChannel(responses, unknown_method_code=unknown_method_code), + ) + + class _FakeGrpcChannel: """ A channel to a gRPC worker replica that serves the given unary methods. The probes create diff --git a/src/tests/_internal/server/services/runs/test_runs.py b/src/tests/_internal/server/services/runs/test_runs.py index cc283a6812..e9087daa66 100644 --- a/src/tests/_internal/server/services/runs/test_runs.py +++ b/src/tests/_internal/server/services/runs/test_runs.py @@ -8,6 +8,7 @@ from dstack._internal.core.models.profiles import Profile, ProfileRetry, RetryEvent from dstack._internal.core.models.runs import JobStatus, JobTerminationReason, RunStatus from dstack._internal.core.models.users import GlobalRole, ProjectRole +from dstack._internal.core.models.volumes import Volume from dstack._internal.server.models import UserModel from dstack._internal.server.services import runs as runs_services from dstack._internal.server.services.jobs import check_can_attach_job_volumes @@ -20,6 +21,7 @@ create_user, get_run_spec, get_volume, + get_volume_configuration, ) @@ -328,38 +330,30 @@ async def create_listed_run( class TestCanAttachRunVolumes: @pytest.mark.asyncio async def test_can_attach(self): - vol11 = get_volume(name="vol11") - vol11.configuration.backend = BackendType.AWS - vol11.configuration.region = "eu-west-1" - vol12 = get_volume(name="vol12") - vol12.configuration.backend = BackendType.AWS - vol12.configuration.region = "eu-west-2" - vol21 = get_volume(name="vol21") - vol21.configuration.backend = BackendType.AWS - vol21.configuration.region = "eu-west-1" - vol22 = get_volume(name="vol22") - vol22.configuration.backend = BackendType.AWS - vol22.configuration.region = "eu-west-2" + vol11 = _get_aws_volume("vol11", "eu-west-1") + vol12 = _get_aws_volume("vol12", "eu-west-2") + vol21 = _get_aws_volume("vol21", "eu-west-1") + vol22 = _get_aws_volume("vol22", "eu-west-2") volumes = [[vol11, vol12], [vol21, vol22]] check_can_attach_job_volumes(volumes) @pytest.mark.asyncio async def test_cannot_attach_different_mount_points_with_different_backends_regions(self): - vol1 = get_volume(name="vol11") - vol1.configuration.backend = BackendType.AWS - vol1.configuration.region = "eu-west-1" - vol2 = get_volume(name="vol12") - vol2.configuration.backend = BackendType.AWS - vol2.configuration.region = "eu-west-2" + vol1 = _get_aws_volume("vol11", "eu-west-1") + vol2 = _get_aws_volume("vol12", "eu-west-2") volumes = [[vol1], [vol2]] with pytest.raises(ServerClientError): check_can_attach_job_volumes(volumes) @pytest.mark.asyncio async def test_cannot_attach_same_volume_at_different_mount_points(self): - vol1 = get_volume(name="vol11") - vol1.configuration.backend = BackendType.AWS - vol1.configuration.region = "eu-west-1" + vol1 = _get_aws_volume("vol11", "eu-west-1") volumes = [[vol1], [vol1]] with pytest.raises(ServerClientError): check_can_attach_job_volumes(volumes) + + +def _get_aws_volume(name: str, region: str) -> Volume: + return get_volume( + name=name, configuration=get_volume_configuration(backend=BackendType.AWS, region=region) + ) diff --git a/src/tests/_internal/server/services/runs/test_spec.py b/src/tests/_internal/server/services/runs/test_spec.py index 20ac7bd605..bcfe8f17a6 100644 --- a/src/tests/_internal/server/services/runs/test_spec.py +++ b/src/tests/_internal/server/services/runs/test_spec.py @@ -1,7 +1,7 @@ import re import uuid from types import SimpleNamespace -from typing import Any, Optional +from typing import Any, Optional, cast import gpuhunt import pytest @@ -12,10 +12,13 @@ ServiceConfiguration, TaskConfiguration, ) +from dstack._internal.core.models.envs import Env from dstack._internal.core.models.files import FileArchiveMapping from dstack._internal.core.models.profiles import Profile, ProfileRetry from dstack._internal.core.models.repos.local import LocalRunRepoData +from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import RunSpec +from dstack._internal.server.models import UserModel from dstack._internal.server.services.runs.spec import ( _check_can_update_configuration, check_can_update_run_spec, @@ -145,25 +148,23 @@ def _service_run_spec( def _validate(run_spec: RunSpec) -> None: - validate_run_spec_and_set_defaults(SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec) + validate_run_spec_and_set_defaults(_user(), run_spec) class TestValidateRunSpecRetryDuration: def test_model_accepts_negative_retry_duration_for_backward_compatibility(self): - retry = ProfileRetry(duration=-1) + retry = ProfileRetry.model_validate({"duration": -1}) assert retry.duration == -1 def test_rejects_negative_retry_duration_for_new_run_specs(self): run_spec = get_run_spec( repo_id="test-repo", - profile=Profile(name="default", retry=ProfileRetry(duration=-1)), + profile=Profile(name="default", retry=ProfileRetry.model_validate({"duration": -1})), ) with pytest.raises(ServerClientError, match="retry.duration cannot be negative"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) class TestValidateRunSpecGroupsIpRefs: @@ -183,9 +184,7 @@ def test_rejects_typo_groups_ref_in_node_group_commands(self): ) with pytest.raises(ServerClientError, match="Illegal reference name"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_accepts_valid_groups_ref(self): run_spec = get_run_spec( @@ -202,9 +201,7 @@ def test_accepts_valid_groups_ref(self): ), ) - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_groups_ref_in_env(self): run_spec = get_run_spec( @@ -212,14 +209,14 @@ def test_rejects_groups_ref_in_env(self): configuration=TaskConfiguration( image="debian", commands=["echo ok"], - env={"PREFILL_URL": "http://${{ groups[1].nodes[0].IP_ADDRESS }}"}, + env=Env.model_validate( + {"PREFILL_URL": "http://${{ groups[1].nodes[0].IP_ADDRESS }}"} + ), ), ) with pytest.raises(ServerClientError, match="only supported in commands, not in `env`"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_out_of_range_group_index(self): run_spec = get_run_spec( @@ -237,9 +234,7 @@ def test_rejects_out_of_range_group_index(self): ) with pytest.raises(ServerClientError, match="out of range"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_out_of_range_node_index(self): run_spec = get_run_spec( @@ -257,9 +252,7 @@ def test_rejects_out_of_range_node_index(self): ) with pytest.raises(ServerClientError, match="out of range"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_replicas_member_in_task_commands(self): run_spec = get_run_spec( @@ -277,9 +270,7 @@ def test_rejects_replicas_member_in_task_commands(self): ) with pytest.raises(ServerClientError, match="Illegal reference name"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_nodes_member_in_service_commands(self): run_spec = get_run_spec( @@ -300,9 +291,7 @@ def test_rejects_nodes_member_in_service_commands(self): ) with pytest.raises(ServerClientError, match="Illegal reference name"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_accepts_service_replicas_ref_on_fixed_and_min_slot(self): for replicas in (1, "1..4"): @@ -316,9 +305,7 @@ def test_accepts_service_replicas_ref_on_fixed_and_min_slot(self): repo_id="test-repo", configuration=_service_with_groups([group]), ) - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_accepts_service_replicas_indexes_for_fixed_count(self): run_spec = get_run_spec( @@ -335,9 +322,7 @@ def test_accepts_service_replicas_indexes_for_fixed_count(self): ] ), ) - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_service_replicas_index_above_min(self): run_spec = get_run_spec( @@ -354,9 +339,7 @@ def test_rejects_service_replicas_index_above_min(self): ) with pytest.raises(ServerClientError, match="out of range"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_service_replicas_ref_into_scale_to_zero_group(self): run_spec = get_run_spec( @@ -373,9 +356,7 @@ def test_rejects_service_replicas_ref_into_scale_to_zero_group(self): ) with pytest.raises(ServerClientError, match="scales to zero"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_service_replicas_group_out_of_range(self): run_spec = get_run_spec( @@ -391,9 +372,7 @@ def test_rejects_service_replicas_group_out_of_range(self): ) with pytest.raises(ServerClientError, match="out of range"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_accepts_service_ref_to_another_group_min_slot(self): run_spec = get_run_spec( @@ -413,9 +392,7 @@ def test_accepts_service_ref_to_another_group_min_slot(self): ] ), ) - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) def test_rejects_service_groups_ref_in_env(self): run_spec = get_run_spec( @@ -434,9 +411,7 @@ def test_rejects_service_groups_ref_in_env(self): ) with pytest.raises(ServerClientError, match="only supported in commands, not in `env`"): - validate_run_spec_and_set_defaults( - SimpleNamespace(ssh_public_key="ssh-rsa test"), run_spec - ) + validate_run_spec_and_set_defaults(_user(), run_spec) class TestRunSpecHasReplicaIpRefs: @@ -679,7 +654,7 @@ def test_sets_vendor_detected_by_gpu_names( set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor == expected_vendor + assert _gpu_vendor(run_spec.configuration.resources) == expected_vendor @pytest.mark.parametrize( "gpu_spec", @@ -694,7 +669,7 @@ def test_does_not_set_vendor_if_gpu_names_are_ambiguous(self, gpu_spec: str): set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor is None + assert _gpu_vendor(run_spec.configuration.resources) is None def test_does_not_override_vendor_set_by_the_user(self): run_spec = _task_run_spec( @@ -703,14 +678,14 @@ def test_does_not_override_vendor_set_by_the_user(self): set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor == gpuhunt.AcceleratorVendor.AMD + assert _gpu_vendor(run_spec.configuration.resources) == gpuhunt.AcceleratorVendor.AMD def test_sets_nvidia_if_the_default_image_is_used(self): run_spec = _task_run_spec(resources={"gpu": "1"}) set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor == gpuhunt.AcceleratorVendor.NVIDIA + assert _gpu_vendor(run_spec.configuration.resources) == gpuhunt.AcceleratorVendor.NVIDIA @pytest.mark.parametrize( ["image", "docker"], @@ -726,14 +701,14 @@ def test_does_not_set_vendor_if_the_default_image_is_not_used( set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor is None + assert _gpu_vendor(run_spec.configuration.resources) is None def test_does_not_set_vendor_if_no_gpu_requested(self): run_spec = _task_run_spec(resources={"gpu": "0"}) set_run_spec_resources_defaults(run_spec) - assert run_spec.configuration.resources.gpu.vendor is None + assert _gpu_vendor(run_spec.configuration.resources) is None def test_sets_default_gpu_spec_if_gpu_is_null(self): run_spec = _task_run_spec(resources={"gpu": None}) @@ -760,9 +735,10 @@ def test_sets_defaults_for_every_replica_group(self): set_run_spec_resources_defaults(run_spec) + assert isinstance(run_spec.configuration, ServiceConfiguration) groups = run_spec.configuration.groups assert groups is not None - assert [g.resources.gpu.vendor for g in groups] == [ + assert [_gpu_vendor(g.resources) for g in groups] == [ gpuhunt.AcceleratorVendor.AMD, gpuhunt.AcceleratorVendor.NVIDIA, gpuhunt.AcceleratorVendor.NVIDIA, @@ -789,8 +765,9 @@ def test_infers_vendor_from_the_image_used_by_the_group( set_run_spec_resources_defaults(run_spec) + assert isinstance(run_spec.configuration, ServiceConfiguration) assert run_spec.configuration.groups is not None - assert run_spec.configuration.groups[0].resources.gpu.vendor == expected_vendor + assert _gpu_vendor(run_spec.configuration.groups[0].resources) == expected_vendor def test_sets_defaults_for_top_level_resources(self): # The top-level resources are ignored when replica groups are set, but they are still @@ -803,7 +780,7 @@ def test_sets_defaults_for_top_level_resources(self): set_run_spec_resources_defaults(run_spec) resources = run_spec.configuration.resources - assert resources.gpu.vendor == gpuhunt.AcceleratorVendor.NVIDIA + assert _gpu_vendor(resources) == gpuhunt.AcceleratorVendor.NVIDIA class TestSetRunSpecResourcesDefaultsNodeGroups: @@ -818,8 +795,9 @@ def test_sets_defaults_for_every_node_group(self): set_run_spec_resources_defaults(run_spec) + assert isinstance(run_spec.configuration, TaskConfiguration) groups = run_spec.configuration.node_groups - assert [g.resources.gpu.vendor for g in groups] == [ + assert [_gpu_vendor(g.resources) for g in groups] == [ gpuhunt.AcceleratorVendor.AMD, gpuhunt.AcceleratorVendor.NVIDIA, gpuhunt.AcceleratorVendor.NVIDIA, @@ -836,7 +814,7 @@ def test_sets_defaults_for_top_level_resources(self): set_run_spec_resources_defaults(run_spec) resources = run_spec.configuration.resources - assert resources.gpu.vendor == gpuhunt.AcceleratorVendor.NVIDIA + assert _gpu_vendor(resources) == gpuhunt.AcceleratorVendor.NVIDIA class TestValidateRunSpecGpuVendorAndImage: @@ -976,3 +954,13 @@ def test_allows_node_group_with_task_level_image(self): ) _validate(run_spec) + + +def _gpu_vendor(resources: ResourcesSpec) -> Optional[gpuhunt.AcceleratorVendor]: + assert resources.gpu is not None + return resources.gpu.vendor + + +def _user() -> UserModel: + # Only the attribute that `validate_run_spec_and_set_defaults()` reads + return cast(UserModel, SimpleNamespace(ssh_public_key="ssh-rsa test")) diff --git a/src/tests/_internal/server/services/test_backend_configs.py b/src/tests/_internal/server/services/test_backend_configs.py index 5b2c15e1b2..6558cc1358 100644 --- a/src/tests/_internal/server/services/test_backend_configs.py +++ b/src/tests/_internal/server/services/test_backend_configs.py @@ -1,11 +1,16 @@ import json from pathlib import Path from textwrap import dedent +from typing import cast from unittest.mock import patch import yaml from dstack._internal.core.backends.kubernetes.backend import KubernetesBackend +from dstack._internal.core.backends.kubernetes.models import ( + KubernetesBackendConfigWithCreds, + KubernetesConfig, +) from dstack._internal.server import settings from dstack._internal.server.services.config import ( ServerConfigManager, @@ -181,10 +186,12 @@ def test_ui_config_embedded_kubeconfig_initializes_backend(self): ) backend_config = config_yaml_to_backend_config(config_yaml) - backend = KubernetesBackend(backend_config) + assert isinstance(backend_config, KubernetesBackendConfigWithCreds) + # `KubernetesConfig` adds no fields to the parsed config + backend = KubernetesBackend(cast(KubernetesConfig, backend_config)) cluster = backend.compute().region_cluster_map[""] - assert cluster.api_client.configuration.host == ( + assert cluster.api_client.configuration.host == ( # pyright: ignore[reportAttributeAccessIssue] "https://gpu-cluster.internal.example.com:6443" ) assert cluster.proxy_jump.hostname == "204.12.171.137" @@ -225,7 +232,9 @@ def test_kubeconfig_context_namespace_does_not_set_backend_namespace(self): ) backend_config = config_yaml_to_backend_config(config_yaml) - backend = KubernetesBackend(backend_config) + assert isinstance(backend_config, KubernetesBackendConfigWithCreds) + # `KubernetesConfig` adds no fields to the parsed config + backend = KubernetesBackend(cast(KubernetesConfig, backend_config)) cluster = backend.compute().region_cluster_map[""] assert cluster.namespace == "default" diff --git a/src/tests/_internal/server/services/test_fluentbit_logs.py b/src/tests/_internal/server/services/test_fluentbit_logs.py index f652e0c3aa..3ace3bc68d 100644 --- a/src/tests/_internal/server/services/test_fluentbit_logs.py +++ b/src/tests/_internal/server/services/test_fluentbit_logs.py @@ -10,6 +10,7 @@ from dstack._internal.server.models import ProjectModel from dstack._internal.server.schemas.logs import PollLogsRequest from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent +from dstack._internal.server.services.logs import fluentbit from dstack._internal.server.services.logs.base import LogStorageError from dstack._internal.server.services.logs.fluentbit import ( ELASTICSEARCH_AVAILABLE, @@ -19,24 +20,12 @@ pytestmark = pytest.mark.skipif(not FLUENTBIT_AVAILABLE, reason="fluent-logger not installed") -# Conditionally import classes that are only defined when FLUENTBIT_AVAILABLE is True -if FLUENTBIT_AVAILABLE: - from dstack._internal.server.services.logs.fluentbit import ( - FluentBitLogStorage, - ForwardFluentBitWriter, - HTTPFluentBitWriter, - NullLogReader, - ) - - if ELASTICSEARCH_AVAILABLE: - from dstack._internal.server.services.logs.fluentbit import ElasticsearchReader - class TestNullLogReader: - """Tests for the NullLogReader (ship-only mode).""" + """Tests for the fluentbit.NullLogReader (ship-only mode).""" def test_read_returns_empty_logs(self): - reader = NullLogReader() + reader = fluentbit.NullLogReader() request = PollLogsRequest( run_name="test-run", job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), @@ -48,12 +37,12 @@ def test_read_returns_empty_logs(self): assert result.next_token is None def test_close_does_nothing(self): - reader = NullLogReader() + reader = fluentbit.NullLogReader() reader.close() # Should not raise class TestHTTPFluentBitWriter: - """Tests for the HTTPFluentBitWriter.""" + """Tests for the fluentbit.HTTPFluentBitWriter.""" @pytest.fixture def mock_httpx_client(self): @@ -61,12 +50,12 @@ def mock_httpx_client(self): yield mock.return_value def test_init_creates_client(self, mock_httpx_client): - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") assert writer._endpoint == "http://localhost:8080" assert writer._tag_prefix == "dstack" def test_write_posts_records(self, mock_httpx_client): - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") records = [ {"message": "Hello", "@timestamp": "2023-10-06T10:00:00Z"}, {"message": "World", "@timestamp": "2023-10-06T10:00:01Z"}, @@ -89,7 +78,7 @@ def test_write_calls_raise_for_status(self, mock_httpx_client): """Test that response.raise_for_status() is called to detect non-2xx responses.""" mock_response = Mock() mock_httpx_client.post.return_value = mock_response - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") writer.write(tag="test-tag", records=[{"message": "test"}]) @@ -106,7 +95,7 @@ def test_write_raises_on_http_status_error(self, mock_httpx_client): mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( "Server Error", request=Mock(), response=mock_response ) - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") with pytest.raises(LogStorageError, match="Fluent-bit HTTP error: status 500"): writer.write(tag="test-tag", records=[{"message": "test"}]) @@ -115,19 +104,19 @@ def test_write_raises_on_transport_error(self, mock_httpx_client): import httpx mock_httpx_client.post.side_effect = httpx.HTTPError("Connection failed") - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") with pytest.raises(LogStorageError, match="Fluent-bit HTTP error"): writer.write(tag="test-tag", records=[{"message": "test"}]) def test_close_closes_client(self, mock_httpx_client): - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") writer.close() mock_httpx_client.close.assert_called_once() def test_write_applies_tag_prefix(self, mock_httpx_client): """Test that tag prefix is applied to tags in HTTP requests.""" - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") records = [{"message": "test"}] writer.write(tag="project/run/job", records=records) @@ -139,7 +128,7 @@ def test_write_applies_tag_prefix(self, mock_httpx_client): def test_write_with_empty_tag_prefix(self, mock_httpx_client): """Test that empty tag prefix doesn't break the tag.""" - writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="") + writer = fluentbit.HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="") records = [{"message": "test"}] writer.write(tag="test-tag", records=records) @@ -151,7 +140,7 @@ def test_write_with_empty_tag_prefix(self, mock_httpx_client): class TestForwardFluentBitWriter: - """Tests for the ForwardFluentBitWriter.""" + """Tests for the fluentbit.ForwardFluentBitWriter.""" @pytest.fixture def mock_fluent_sender(self): @@ -168,7 +157,7 @@ def test_init_creates_sender(self, mock_fluent_sender): "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" ) as mock: mock.return_value = mock_fluent_sender - ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + fluentbit.ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") mock.assert_called_once_with("dstack", host="localhost", port=24224) def test_write_emits_records(self, mock_fluent_sender): @@ -176,7 +165,9 @@ def test_write_emits_records(self, mock_fluent_sender): "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" ) as mock: mock.return_value = mock_fluent_sender - writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + writer = fluentbit.ForwardFluentBitWriter( + host="localhost", port=24224, tag_prefix="dstack" + ) records = [ {"message": "Hello"}, @@ -194,7 +185,9 @@ def test_write_raises_on_emit_failure(self, mock_fluent_sender): "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" ) as mock: mock.return_value = mock_fluent_sender - writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + writer = fluentbit.ForwardFluentBitWriter( + host="localhost", port=24224, tag_prefix="dstack" + ) with pytest.raises(LogStorageError, match="Fluent-bit Forward error"): writer.write(tag="test-tag", records=[{"message": "test"}]) @@ -206,13 +199,15 @@ def test_close_closes_sender(self, mock_fluent_sender): "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" ) as mock: mock.return_value = mock_fluent_sender - writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + writer = fluentbit.ForwardFluentBitWriter( + host="localhost", port=24224, tag_prefix="dstack" + ) writer.close() mock_fluent_sender.close.assert_called_once() class TestFluentBitLogStorage: - """Tests for the FluentBitLogStorage.""" + """Tests for the fluentbit.FluentBitLogStorage.""" @pytest_asyncio.fixture async def project(self, test_db, session: AsyncSession) -> ProjectModel: @@ -247,19 +242,19 @@ def test_init_with_forward_protocol(self, mock_forward_writer): "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", tag_prefix="dstack", ) mock.assert_called_once_with(host="localhost", port=24224, tag_prefix="dstack") - assert isinstance(storage._reader, NullLogReader) + assert isinstance(storage._reader, fluentbit.NullLogReader) def test_init_with_http_protocol(self, mock_http_writer): with patch("dstack._internal.server.services.logs.fluentbit.HTTPFluentBitWriter") as mock: mock.return_value = mock_http_writer - FluentBitLogStorage( + fluentbit.FluentBitLogStorage( host="localhost", port=8080, protocol="http", @@ -269,7 +264,7 @@ def test_init_with_http_protocol(self, mock_http_writer): def test_init_with_unsupported_protocol_raises(self): with pytest.raises(LogStorageError, match="Unsupported Fluent-bit protocol"): - FluentBitLogStorage( + fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="grpc", @@ -282,14 +277,14 @@ def test_init_ship_only_mode(self, mock_forward_writer): "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", tag_prefix="dstack", es_host=None, ) - assert isinstance(storage._reader, NullLogReader) + assert isinstance(storage._reader, fluentbit.NullLogReader) @pytest.mark.skipif(not ELASTICSEARCH_AVAILABLE, reason="elasticsearch not installed") def test_init_with_elasticsearch(self, mock_forward_writer, mock_es_reader): @@ -305,7 +300,7 @@ def test_init_with_elasticsearch(self, mock_forward_writer, mock_es_reader): writer_mock.return_value = mock_forward_writer reader_mock.return_value = mock_es_reader - FluentBitLogStorage( + fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -328,7 +323,7 @@ async def test_write_logs(self, test_db, project: ProjectModel, mock_forward_wri "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -359,7 +354,7 @@ async def test_write_logs_empty_logs_not_written( "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -381,7 +376,7 @@ async def test_write_logs_empty_logs_not_written( async def test_poll_logs_ship_only_mode(self, test_db, project: ProjectModel): """Test that ship-only mode returns empty logs.""" with patch("dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter"): - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -404,7 +399,7 @@ def test_close_closes_writer_and_reader(self, mock_forward_writer): "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -422,7 +417,7 @@ def test_close_closes_reader_even_if_writer_fails(self, mock_forward_writer): ) as mock: mock_forward_writer.close.side_effect = Exception("Writer close failed") mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -444,7 +439,7 @@ def test_get_stream_name(self, mock_forward_writer): "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" ) as mock: mock.return_value = mock_forward_writer - storage = FluentBitLogStorage( + storage = fluentbit.FluentBitLogStorage( host="localhost", port=24224, protocol="forward", @@ -466,7 +461,7 @@ def test_get_stream_name(self, mock_forward_writer): reason="fluent-logger or elasticsearch not installed", ) class TestElasticsearchReader: - """Tests for the ElasticsearchReader.""" + """Tests for the fluentbit.ElasticsearchReader.""" @pytest.fixture def mock_es_client(self): @@ -480,7 +475,7 @@ def mock_es_client(self): def test_init_verifies_connection(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - ElasticsearchReader( + fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -489,7 +484,7 @@ def test_init_verifies_connection(self, mock_es_client): def test_init_with_api_key(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - ElasticsearchReader( + fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", api_key="test-api-key", @@ -505,7 +500,7 @@ def test_init_connection_error_raises(self): mock.return_value = mock_instance with pytest.raises(LogStorageError, match="Failed to connect"): - ElasticsearchReader( + fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -536,7 +531,7 @@ def test_read_returns_logs(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -556,7 +551,7 @@ def test_read_returns_logs(self, mock_es_client): def test_read_with_time_filtering(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -578,7 +573,7 @@ def test_read_with_time_filtering(self, mock_es_client): def test_read_descending_order(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -600,7 +595,7 @@ def test_read_descending_order(self, mock_es_client): def test_read_with_next_token(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -620,7 +615,7 @@ def test_read_with_malformed_next_token_raises_client_error(self, mock_es_client """Test that malformed next_token raises ServerClientError (400) instead of IndexError (500).""" with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) @@ -651,7 +646,7 @@ def test_read_with_malformed_next_token_raises_client_error(self, mock_es_client def test_close_closes_client(self, mock_es_client): with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: mock.return_value = mock_es_client - reader = ElasticsearchReader( + reader = fluentbit.ElasticsearchReader( host="http://localhost:9200", index="dstack-logs", ) diff --git a/src/tests/_internal/server/services/test_instances.py b/src/tests/_internal/server/services/test_instances.py index 6b332a3e73..e54f0ad97d 100644 --- a/src/tests/_internal/server/services/test_instances.py +++ b/src/tests/_internal/server/services/test_instances.py @@ -454,7 +454,7 @@ async def test_fleet_instance_selector_respects_project_reference( res = await instances_services.select_instances_by_selectors( session=session, project=project, - selectors=[FleetInstanceSelector(fleet=selector, instance=1)], + selectors=[FleetInstanceSelector.model_validate({"fleet": selector, "instance": 1})], ) assert [instance.name for instance in res] == [expected_name] diff --git a/src/tests/_internal/server/services/test_templates.py b/src/tests/_internal/server/services/test_templates.py index 45fcaf5628..d49670af06 100644 --- a/src/tests/_internal/server/services/test_templates.py +++ b/src/tests/_internal/server/services/test_templates.py @@ -1,5 +1,7 @@ import uuid from pathlib import Path +from types import SimpleNamespace +from typing import cast from unittest.mock import patch import pytest @@ -10,6 +12,7 @@ EnvUITemplateParameter, NameUITemplateParameter, ) +from dstack._internal.server.models import ProjectModel from dstack._internal.server.services import templates as templates_service @@ -39,7 +42,7 @@ class TestListTemplates: @pytest.mark.asyncio async def test_returns_empty_when_no_repo_configured(self): with patch.object(templates_service.settings, "SERVER_TEMPLATES_REPO", None): - project = type("Project", (), {"templates_repo": None, "id": "project-id"})() + project = cast(ProjectModel, SimpleNamespace(templates_repo=None, id="project-id")) result = await templates_service.list_templates(project) assert result == [] diff --git a/src/tests/_internal/server/services/test_volumes.py b/src/tests/_internal/server/services/test_volumes.py index e77e2591db..5807fa23b0 100644 --- a/src/tests/_internal/server/services/test_volumes.py +++ b/src/tests/_internal/server/services/test_volumes.py @@ -45,11 +45,13 @@ def test_regional_volume_requires_size_only_when_managed(self, backend, identifi def test_external_volume_with_auto_cleanup_duration_raises_error(self): """External volumes (with volume_id) should not allow auto_cleanup_duration""" - config = AWSVolumeConfiguration( - backend=BackendType.AWS, - region="us-east-1", - volume_id="vol-123456", - auto_cleanup_duration="1h", + config = AWSVolumeConfiguration.model_validate( + { + "backend": BackendType.AWS, + "region": "us-east-1", + "volume_id": "vol-123456", + "auto_cleanup_duration": "1h", + } ) with pytest.raises( ServerClientError, match="External volumes.*do not support auto_cleanup_duration" @@ -71,11 +73,13 @@ def test_external_volume_with_auto_cleanup_duration_int_raises_error(self): def test_external_volume_with_auto_cleanup_disabled_succeeds(self): """External volumes with auto_cleanup_duration='off' or -1 should be allowed""" - config1 = AWSVolumeConfiguration( - backend=BackendType.AWS, - region="us-east-1", - volume_id="vol-123456", - auto_cleanup_duration="off", + config1 = AWSVolumeConfiguration.model_validate( + { + "backend": BackendType.AWS, + "region": "us-east-1", + "volume_id": "vol-123456", + "auto_cleanup_duration": "off", + } ) config2 = AWSVolumeConfiguration( backend=BackendType.AWS, @@ -97,8 +101,13 @@ def test_external_volume_without_auto_cleanup_succeeds(self): def test_new_volume_with_auto_cleanup_duration_succeeds(self): """New volumes (without volume_id) with auto_cleanup_duration should be allowed""" - config = AWSVolumeConfiguration( - backend=BackendType.AWS, region="us-east-1", size=100, auto_cleanup_duration="1h" + config = AWSVolumeConfiguration.model_validate( + { + "backend": BackendType.AWS, + "region": "us-east-1", + "size": 100, + "auto_cleanup_duration": "1h", + } ) # Should not raise any errors _validate_volume_configuration(config) diff --git a/src/tests/_internal/server/utils/test_tracing.py b/src/tests/_internal/server/utils/test_tracing.py index e9168992e1..6e2bdd2b53 100644 --- a/src/tests/_internal/server/utils/test_tracing.py +++ b/src/tests/_internal/server/utils/test_tracing.py @@ -55,6 +55,8 @@ async def task(): task_span, outer_span = span_exporter.get_finished_spans() assert task_span.name == "task" assert task_span.parent is None + assert task_span.context is not None + assert outer_span.context is not None assert task_span.context.trace_id != outer_span.context.trace_id @@ -63,7 +65,7 @@ class TestRecordTaskRun: async def test_counts_runs_by_status(self): from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import InMemoryMetricReader + from opentelemetry.sdk.metrics.export import InMemoryMetricReader, NumberDataPoint reader = InMemoryMetricReader() # The global meter provider can only be set once per process @@ -85,16 +87,16 @@ async def failing_task(): with pytest.raises(ValueError): await failing_task() - points = ( - reader.get_metrics_data() - .resource_metrics[0] - .scope_metrics[0] - .metrics[0] - .data.data_points - ) - counts = {p.attributes["status"]: p.value for p in points} + metrics_data = reader.get_metrics_data() + assert metrics_data is not None + points = metrics_data.resource_metrics[0].scope_metrics[0].metrics[0].data.data_points + counts = {} + for point in points: + assert isinstance(point, NumberDataPoint) + assert point.attributes is not None + assert point.attributes["task"] == "task" + counts[point.attributes["status"]] = point.value assert counts == {"success": 2, "error": 1} - assert all(p.attributes["task"] == "task" for p in points) class TestBuildLogHandler: diff --git a/src/tests/_internal/utils/test_ssh.py b/src/tests/_internal/utils/test_ssh.py index 92ed9bef3b..b78b4658f3 100644 --- a/src/tests/_internal/utils/test_ssh.py +++ b/src/tests/_internal/utils/test_ssh.py @@ -226,6 +226,7 @@ def test_converts_pkcs8_private_key_to_pem(self, tmp_path: Path): assert public_key_path is None assert public_key == public_key_bytes.decode().rsplit(" ", 1)[0] + assert private_key is not None assert private_key.startswith("-----BEGIN RSA PRIVATE KEY-----") assert pkey_from_str(private_key) diff --git a/src/tests/api/test_projects.py b/src/tests/api/test_projects.py index 6e8bb61248..eca666f150 100644 --- a/src/tests/api/test_projects.py +++ b/src/tests/api/test_projects.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from uuid import UUID -from dstack._internal.utils.common import render_datetime_as_api +from dstack._internal.utils.common import get_or_error, render_datetime_as_api from dstack.api.server._projects import ProjectsAPIClient from tests.api.common import RequestRecorder @@ -43,7 +43,7 @@ def test_projects_list_serializes_pagination_and_parses_info_list(self): ascending=True, ) - payload = json.loads(request.last_body) + payload = json.loads(get_or_error(request.last_body)) assert request.last_path == "/api/projects/list" assert payload["include_not_joined"] is True assert payload["return_total_count"] is True diff --git a/src/tests/api/test_runs.py b/src/tests/api/test_runs.py index 34017acf3e..af65c1d871 100644 --- a/src/tests/api/test_runs.py +++ b/src/tests/api/test_runs.py @@ -2,7 +2,8 @@ import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Optional +from typing import Optional, cast +from unittest.mock import MagicMock import pytest @@ -21,50 +22,27 @@ ) from dstack._internal.core.models.runs import Run as RunModel from dstack._internal.server.schemas.logs import PollLogsRequest +from dstack.api import Client from dstack.api._public.runs import Run, RunCollection from tests._internal.utils.test_ssh import PRIVATE_KEY, PUBLIC_KEY, PUBLIC_KEY_NO_COMMENT -class _RunsAPI: - def __init__(self): - self.calls = [] - - def list(self, **kwargs): - self.calls.append(kwargs) - if len(self.calls) == 1: - return [] - return ["finished-run"] - - -class _APIClient: - def __init__(self): - self.runs = _RunsAPI() - - -class _PlansAPI: - def __init__(self): - self.run_specs: list[RunSpec] = [] - - def get_plan(self, project_name: str, run_spec: RunSpec, **kwargs) -> str: - self.run_specs.append(run_spec) - return "run-plan" - - -class _PlansAPIClient: - def __init__(self): - self.runs = _PlansAPI() +def _get_run_collection(api_client: MagicMock) -> RunCollection: + # `RunCollection` only stores the client and never uses it + return RunCollection(api_client=api_client, project="main", client=cast(Client, None)) class TestRunCollectionList: def test_default_list_fallback_limits_job_submissions(self): - api_client = _APIClient() - runs = RunCollection(api_client=api_client, project="main", client=None) - runs._model_to_run = lambda run: run + api_client = MagicMock() + api_client.runs.list.side_effect = [[], [_get_run_model(RunStatus.DONE, jobs=[])]] - assert runs.list() == ["finished-run"] + runs = _get_run_collection(api_client).list() - assert api_client.runs.calls[0]["job_submissions_limit"] == 1 - assert api_client.runs.calls[1]["job_submissions_limit"] == 1 + assert [run.name for run in runs] == ["test-run"] + assert [ + call.kwargs["job_submissions_limit"] for call in api_client.runs.list.call_args_list + ] == [1, 1] class _LogsAPI: @@ -127,7 +105,7 @@ def _get_run_model(status: RunStatus, jobs: list[Job]) -> RunModel: def _get_run(run_model: RunModel, logs_api: _LogsAPI) -> Run: - api_client = _APIClient() + api_client = MagicMock() api_client.logs = logs_api return Run(api_client=api_client, project="main", run=run_model) @@ -192,19 +170,17 @@ class TestRunCollectionGetRunPlan: def _get_run_plan( self, ssh_identity_file: Optional[Path] = None, ssh_key_pub: Optional[str] = None ) -> RunSpec: - api_client = _PlansAPIClient() - runs = RunCollection(api_client=api_client, project="main", client=None) - - assert ( - runs.get_run_plan( - configuration=TaskConfiguration(commands=["echo hello"], image="ubuntu:latest"), - ssh_identity_file=ssh_identity_file, - ssh_key_pub=ssh_key_pub, - ) - == "run-plan" + api_client = MagicMock() + api_client.runs.get_plan.return_value = "run-plan" + + plan = _get_run_collection(api_client).get_run_plan( + configuration=TaskConfiguration(commands=["echo hello"], image="ubuntu:latest"), + ssh_identity_file=ssh_identity_file, + ssh_key_pub=ssh_key_pub, ) - return api_client.runs.run_specs[0] + assert plan == "run-plan" + return api_client.runs.get_plan.call_args.kwargs["run_spec"] def test_uses_public_key_file(self, tmp_path: Path): private_key_path = tmp_path / "id_ed25519" diff --git a/src/tests/api/test_users.py b/src/tests/api/test_users.py index e5c8f9afb6..3e10536310 100644 --- a/src/tests/api/test_users.py +++ b/src/tests/api/test_users.py @@ -3,7 +3,8 @@ from datetime import datetime, timezone from uuid import UUID -from dstack._internal.utils.common import render_datetime_as_api +from dstack._internal.core.models.users import UsersInfoList +from dstack._internal.utils.common import get_or_error, render_datetime_as_api from dstack.api.server._users import UsersAPIClient from tests.api.common import RequestRecorder @@ -35,7 +36,7 @@ def test_serializes_pagination_and_parses_info_list(self): ascending=True, ) - payload = json.loads(recorder.last_body) + payload = json.loads(get_or_error(recorder.last_body)) assert recorder.last_path == "/api/users/list" assert payload["return_total_count"] is True assert payload["name_pattern"] == "user" @@ -43,6 +44,7 @@ def test_serializes_pagination_and_parses_info_list(self): assert payload["prev_id"] == str(uid) assert payload["limit"] == 1 assert payload["ascending"] is True + assert isinstance(result, UsersInfoList) assert result.total_count == 1 assert result.users[0].username == "user" diff --git a/src/tests/plugins/test_rest_plugin.py b/src/tests/plugins/test_rest_plugin.py index d17e58c5dd..2876d5ee57 100644 --- a/src/tests/plugins/test_rest_plugin.py +++ b/src/tests/plugins/test_rest_plugin.py @@ -33,7 +33,7 @@ async def create_run_spec( session: AsyncSession, project: ProjectModel, - replicas: str = 1, + replicas: int = 1, ) -> RunSpec: repo = await create_repo(session=session, project_id=project.id) run_name = "test-run" @@ -111,7 +111,8 @@ async def test_on_apply_plugin_service_returns_mutated_spec( mock_response = Mock() response_dict = {"spec": spec.model_dump(), "error": None} - if isinstance(spec, (RunSpec, FleetSpec)): + has_profile = isinstance(spec, (RunSpec, FleetSpec)) + if has_profile: response_dict["spec"]["profile"]["tags"] = {"env": "test", "team": "qa"} else: response_dict["spec"]["configuration_path"] = "/path/to/something"