Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
a58b27e
Reach optional fluentbit classes through the module in tests
r4victor Oct 28, 2026
f33de4a
Build AWS volumes directly in run volume tests
r4victor Oct 28, 2026
ff2527e
Use replicas instead of the legacy count in replica group tests
r4victor Oct 28, 2026
b67ad58
Compare proxy repo listings as sorted lists in tests
r4victor Oct 28, 2026
d62355e
Fix loose annotations in server tests
r4victor Oct 28, 2026
202a018
Narrow optional values in server, proxy and utils tests
r4victor Oct 28, 2026
ccffb4d
Use model_validate and typed values for shorthand in server and proxy…
r4victor Oct 28, 2026
776b682
Narrow configuration and metric data point unions in server tests
r4victor Oct 28, 2026
b42adc6
Type fake users and projects in server service tests
r4victor Oct 28, 2026
40d974a
Ignore POSIX-only Unix socket APIs in tests on Windows
r4victor Oct 28, 2026
2923d49
Narrow run configurations in compatibility tests
r4victor Oct 28, 2026
c5fbfdc
Fix remaining pyright errors in proxy and backend config tests
r4victor Oct 28, 2026
e4b0b3f
Check all tests except CLI with pyright
r4victor Oct 28, 2026
e0f27b0
Fix pyright errors in CLI tests and check all tests
r4victor Oct 28, 2026
d207836
Address review comments on test typing
r4victor Oct 28, 2026
99faef7
Fix pyright errors in API and plugin tests and check all tests
r4victor Oct 28, 2026
c1376ed
Mock API clients in run collection tests
r4victor Oct 28, 2026
b5d7c00
Clarify why run collection tests pass no client
r4victor Oct 28, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ include = [
"src/dstack/_internal/core/backends",
"src/dstack/_internal/cli/services/configurators",
"src/dstack/_internal/cli/commands",
"src/tests/_internal/core",
"src/tests/_internal/server/background/pipeline_tasks",
"src/tests",
]
exclude = [
# OCI has pyright errors. Excluded rather than ignored because checking it makes pyright analyze
Expand Down
6 changes: 3 additions & 3 deletions src/tests/_internal/cli/commands/test_preset.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from dstack._internal.cli.services.presets.store import PresetStore
from dstack._internal.compat import IS_WINDOWS
from dstack._internal.core.errors import CLIError
from dstack._internal.utils.common import render_datetime_as_api
from dstack._internal.utils.common import get_or_error, render_datetime_as_api
from tests._internal.cli.common import (
get_preset,
get_session_run,
Expand Down Expand Up @@ -543,7 +543,7 @@ def test_create_detaches_the_name_from_the_old_preset(self, tmp_path):

assert exit_code == 0
create.assert_called_once()
assert store.get(preset.id).name is None
assert get_or_error(store.get(preset.id)).name is None

def test_create_without_confirmation_exits_before_creating(self, tmp_path):
preset = get_preset().model_copy(update={"name": "qwen"})
Expand All @@ -569,7 +569,7 @@ def test_create_without_confirmation_exits_before_creating(self, tmp_path):

assert exit_code == 0
create.assert_not_called()
assert store.get(preset.id).name == "qwen"
assert get_or_error(store.get(preset.id)).name == "qwen"

def test_get_and_delete_resolve_names(self, tmp_path, capsys):
preset = get_preset().model_copy(update={"name": "qwen"})
Expand Down
97 changes: 60 additions & 37 deletions src/tests/_internal/cli/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import IO, Any, List, Optional
from typing import IO, Any, List, Optional, cast
from unittest.mock import patch
from uuid import uuid4

Expand All @@ -19,6 +19,7 @@
)
from dstack._internal.cli.models.presets import VerifiedPreset
from dstack._internal.cli.services.presets.agents.base import PresetAgentSpec
from dstack._internal.cli.services.presets.store import PresetStore
from dstack._internal.compat import IS_WINDOWS
from dstack._internal.core.models.configurations import (
DEFAULT_REPLICA_GROUP_NAME,
Expand All @@ -32,6 +33,7 @@
)
from dstack._internal.core.models.resources import ResourcesSpec
from dstack._internal.core.models.runs import JobStatus, Run, RunStatus, ServiceSpec
from dstack.api import Client


def plain_console(file: IO[str], *, width: int = 250) -> Console:
Expand All @@ -50,20 +52,26 @@ def run_dstack_cli(
repo_dir: Optional[Path] = None,
) -> int:
exit_code = 0
cwd = os.getcwd()
prev_home_dir = os.environ.get("HOME")
prev_userprofile = os.environ.get("USERPROFILE")
if repo_dir is not None:
cwd = os.getcwd()
os.chdir(repo_dir)
if home_dir is not None:
prev_home_dir = os.environ.get("HOME")
os.environ["HOME"] = str(home_dir)
if IS_WINDOWS:
prev_userprofile = os.environ.get("USERPROFILE")
os.environ["USERPROFILE"] = str(home_dir)
with patch("sys.argv", ["dstack"] + cli_args):
try:
main()
except SystemExit as e:
exit_code = e.code
# Same as the process exit status: no code is success, a non-int code is an error
if e.code is None:
exit_code = 0
elif isinstance(e.code, int):
exit_code = e.code
else:
exit_code = 1
finally:
if home_dir is not None:
if prev_home_dir is None:
Expand Down Expand Up @@ -95,28 +103,30 @@ def run_dstack_cli(


def get_preset_benchmark() -> PresetBenchmark:
benchmark = PresetBenchmark(
tool="vllm bench serve",
tool_version="0.11.0",
command="vllm bench serve --base-url $SERVICE_URL",
workload={
"api": "chat_completions",
"num_requests": 16,
"input_tokens": 1024,
"output_tokens": 128,
"concurrency": 1,
},
metrics={
"successful_requests": 16,
"failed_requests": 0,
"duration_seconds": 48.64,
"total_input_tokens": 16384,
"total_output_tokens": 2048,
"output_tok_per_s": 42.1,
"per_user_tok_per_s": 42.1,
"ttft_ms": {"mean": 110.9, "p50": 108.2, "p99": 121.6},
"tpot_ms": {"mean": 7.5, "p50": 7.4, "p99": 8.1},
},
benchmark = PresetBenchmark.model_validate(
{
"tool": "vllm bench serve",
"tool_version": "0.11.0",
"command": "vllm bench serve --base-url $SERVICE_URL",
"workload": {
"api": "chat_completions",
"num_requests": 16,
"input_tokens": 1024,
"output_tokens": 128,
"concurrency": 1,
},
"metrics": {
"successful_requests": 16,
"failed_requests": 0,
"duration_seconds": 48.64,
"total_input_tokens": 16384,
"total_output_tokens": 2048,
"output_tok_per_s": 42.1,
"per_user_tok_per_s": 42.1,
"ttft_ms": {"mean": 110.9, "p50": 108.2, "p99": 121.6},
"tpot_ms": {"mean": 7.5, "p50": 7.4, "p99": 8.1},
},
}
)
return benchmark

Expand Down Expand Up @@ -258,14 +268,27 @@ def get_session_run(**overrides: Any) -> PresetSessionRun:


def get_successful_preset_report(run: Run) -> PresetAgentSuccess:
return PresetAgentSuccess(
success=True,
run_id=run.id,
run_name=run.run_spec.run_name,
service_yaml="type: service",
trial=1,
base="Qwen/Qwen3.5-27B",
model="community/Qwen3.5-27B-GPTQ-Int4",
context_length=32768,
benchmark=get_preset_benchmark(),
return PresetAgentSuccess.model_validate(
{
"success": True,
"run_id": run.id,
"run_name": run.run_spec.run_name,
"service_yaml": "type: service",
"trial": 1,
"base": "Qwen/Qwen3.5-27B",
"model": "community/Qwen3.5-27B-GPTQ-Int4",
"context_length": 32768,
"benchmark": get_preset_benchmark(),
}
)


def get_verified_preset(store: PresetStore, preset_id: str) -> VerifiedPreset:
preset = store.get(preset_id)
assert isinstance(preset, VerifiedPreset)
return preset


def get_fake_api(**attrs: Any) -> Client:
# Only the attributes that the code under test reads
return cast(Client, SimpleNamespace(**attrs))
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def test_no_retry(self):
def test_retry_duration(self):
profile = Profile(name="test")
modified, _ = apply_args(profile, ["--retry-duration", "1h"])
profile.retry = ProfileRetry(on_events=None, duration="1h")
profile.retry = ProfileRetry.model_validate({"on_events": None, "duration": "1h"})
assert profile.model_dump() == modified.model_dump()


Expand Down
8 changes: 5 additions & 3 deletions src/tests/_internal/cli/services/configurators/test_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def test_env_override(self):
assert modified.model_dump() == conf.model_dump()

def test_ports_override(self):
conf = TaskConfiguration(commands=["whoami"], ports=["80"])
conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["80"]})
modified, args = self.apply_args(conf, ["-p", "8000:80", "--port", "8001:8000"])
conf.ports = [
PortMapping(local_port=8000, container_port=80),
Expand All @@ -85,12 +85,12 @@ def test_ports_override(self):
assert modified.model_dump() == conf.model_dump()

def test_local_ports_conflict(self):
conf = TaskConfiguration(commands=["whoami"], ports=["3000"])
conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["3000"]})
with pytest.raises(ConfigurationError):
self.apply_args(conf, ["-p", "3000:4000"])

def test_any_port(self):
conf = TaskConfiguration(commands=["whoami"], ports=["8000"])
conf = TaskConfiguration.model_validate({"commands": ["whoami"], "ports": ["8000"]})
modified, args = self.apply_args(conf, ["-p", "*:8000"])
conf.ports = [PortMapping(local_port=None, container_port=8000)]
assert modified.model_dump() == conf.model_dump()
Expand Down Expand Up @@ -132,6 +132,7 @@ def test_interpolates_service_group_commands(self):
}
)
modified, _ = self.apply_args(conf, ["hello"])
assert isinstance(modified, ServiceConfiguration)
assert modified.groups is not None
assert modified.groups[0].commands == [
"echo hello ${{ groups[0].replicas[0].IP_ADDRESS }}"
Expand Down Expand Up @@ -164,6 +165,7 @@ def test_homogeneous_service_interpolates_commands_once(self):
}
)
modified, _ = self.apply_args(conf, [])
assert isinstance(modified, ServiceConfiguration)
assert modified.commands == ["echo ${{ env.FOO }}"]


Expand Down
43 changes: 29 additions & 14 deletions src/tests/_internal/cli/services/presets/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from contextlib import suppress
from pathlib import Path
from types import SimpleNamespace
from typing import cast

import psutil
import pytest
Expand Down Expand Up @@ -60,7 +61,14 @@
from dstack._internal.core.errors import CLIError
from dstack._internal.core.models.configurations import PresetAgentConfig, PresetConfiguration
from dstack._internal.core.services.configs import ConfigManager
from tests._internal.cli.common import get_agent_spec, get_session_run, get_session_state
from dstack._internal.utils.common import get_or_error
from dstack.api import Client
from tests._internal.cli.common import (
get_agent_spec,
get_fake_api,
get_session_run,
get_session_state,
)


def _record_run(session, workspace_record):
Expand Down Expand Up @@ -477,7 +485,7 @@ def test_inherits_only_required_environment(self, tmp_path, monkeypatch):
monkeypatch.setenv("PATH", "/usr/bin")
monkeypatch.setenv("HOME", "/home/test")
monkeypatch.setenv("UNRELATED_SECRET", "must-not-be-inherited")
api = SimpleNamespace(
api = get_fake_api(
project="main",
client=SimpleNamespace(base_url="http://127.0.0.1:3000"),
)
Expand Down Expand Up @@ -547,11 +555,13 @@ def _session_workspace(tmp_path):

class TestAgentSession:
def _configuration(self) -> PresetConfiguration:
return PresetConfiguration(
name="qwen",
base="Qwen/Qwen3.5-27B",
max_price=0.5,
env=["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"],
return PresetConfiguration.model_validate(
{
"name": "qwen",
"base": "Qwen/Qwen3.5-27B",
"max_price": 0.5,
"env": ["HF_TOKEN", "TOKENIZERS_PARALLELISM=false"],
}
)

def _home(self, tmp_path, monkeypatch) -> None:
Expand Down Expand Up @@ -630,7 +640,8 @@ def test_reports_invalid_existing_parent(self, tmp_path, monkeypatch):

with pytest.raises(CLIError, match="Could not create agent output"):
create_preset_session(
PresetConfiguration(name="qwen", base="Qwen/Qwen3.5-27B"), previous=()
PresetConfiguration.model_validate({"name": "qwen", "base": "Qwen/Qwen3.5-27B"}),
previous=(),
)

def test_log_write_failure_warns_once(self, tmp_path, capsys):
Expand Down Expand Up @@ -1325,7 +1336,7 @@ async def test_does_not_resurrect_an_externally_stopped_agent(self, tmp_path, mo
workspace, session = _agent_setup(tmp_path)
# Another CLI recorded a stop: the death must read as a decision, not an
# outage to retry.
state = session.read_state()
state = get_or_error(session.read_state())
state.status = "interrupted"
session.write_state(state)

Expand Down Expand Up @@ -1622,6 +1633,7 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path):
trials_dir = _write_trials(tmp_path, [record, rerun, record, nameless])

summary = _summarize_session_trials(trials_dir)
assert summary is not None

# 4 records = 4 trials: one long-lived task commonly hosts several
# trials, so identical records must not collapse the count.
Expand Down Expand Up @@ -1649,6 +1661,7 @@ def test_trials_are_ordered_numerically_beyond_nine(self, tmp_path):
trials_dir = _write_trials(tmp_path, records)

summary = _summarize_session_trials(trials_dir)
assert summary is not None

assert summary["series"] == [float(i * 100) for i in range(1, 13)]

Expand All @@ -1666,6 +1679,7 @@ def test_a_trial_still_in_flight_is_not_counted(self, tmp_path):
(trials_dir / "2" / "task.dstack.yml").write_text("type: task\n")

summary = _summarize_session_trials(trials_dir)
assert summary is not None

assert summary["count"] == 1
assert summary["series"] == [2000.0]
Expand All @@ -1676,6 +1690,7 @@ def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path):
(trials_dir / "1" / "trial.json").write_text('{"benchmark": {"met')

summary = _summarize_session_trials(trials_dir)
assert summary is not None

assert summary["count"] == 0

Expand Down Expand Up @@ -1780,18 +1795,18 @@ def test_detach_keeps_the_agent_and_stop_terminates_it(self, tmp_path, monkeypat
session.write_state(state)

monkeypatch.setattr(create_module, "confirm_ask", lambda *_: False)
_stop_or_detach_agent_session(session, None)
_stop_or_detach_agent_session(session, cast(Client, None))
assert agent.poll() is None
assert session.read_state().status == "running"
assert get_or_error(session.read_state()).status == "running"
assert "Detached" in capsys.readouterr().out

monkeypatch.setattr(create_module, "confirm_ask", lambda *_: True)
_stop_or_detach_agent_session(session, None)
_stop_or_detach_agent_session(session, cast(Client, None))
psutil.Process(agent.pid).wait(timeout=10)
assert session.read_state().status == "interrupted"
assert get_or_error(session.read_state()).status == "interrupted"
finally:
with suppress(OSError):
os.killpg(agent.pid, signal.SIGKILL)
os.killpg(agent.pid, signal.SIGKILL) # pyright: ignore[reportAttributeAccessIssue]


class TestOffsetStoreSharing:
Expand Down
Loading
Loading