From 63d7386c9a7f7c5e7731a6d462b1ec63d9d57809 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 10 Nov 2026 09:43:58 +0000 Subject: [PATCH] Allow retries with `type: dynamo` router Retry was rejected because workers got the router's internal IP once, via DSTACK_ROUTER_INTERNAL_IP, and a retried router could come back at a different address. Since #4402, Dynamo services run etcd in a dedicated group referenced via replica IP interpolation (#4177), and #4473 removed the variable, so nothing depends on the router's address anymore. --- src/dstack/_internal/core/models/runs.py | 26 ------ src/tests/_internal/core/models/test_runs.py | 94 -------------------- 2 files changed, 120 deletions(-) diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index 9f4dad43a..a888c9907 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -42,7 +42,6 @@ ) from dstack._internal.core.models.repos import AnyRunRepoData from dstack._internal.core.models.resources import Memory, ResourcesSpec -from dstack._internal.core.models.routers import RouterType from dstack._internal.core.models.unix import UnixUser from dstack._internal.core.models.volumes import MountPoint from dstack._internal.utils import common as common_utils @@ -629,31 +628,6 @@ def _merged_profile(self) -> Self: self.merged_profile = merged_profile return self - @model_validator(mode="after") - def _validate_dynamo_no_retry(self) -> Self: - """Reject `retry` for services with a Dynamo router replica group. - Dynamo workers cache the router's internal IP at provisioning time. A - retry would produce a new router and likely a new internal_ip, leaving workers bound - to a router that no longer exists. - """ - merged_profile = self.merged_profile - cfg = self.configuration - if merged_profile is None or merged_profile.retry is None: - return self - if not isinstance(cfg, ServiceConfiguration): - return self - for g in cfg.replica_groups: - if g.router is not None and g.router.type == RouterType.DYNAMO: - raise ValueError( - "Retry cannot be configured for services with a Dynamo " - "router replica group. The router's address must remain " - "stable for the life of the run; allowing retry would " - "leave workers bound to a router that no longer exists. " - "Remove `retry` from the profile/configuration and " - "re-apply." - ) - return self - class ServiceModelSpec(CoreModel): name: str diff --git a/src/tests/_internal/core/models/test_runs.py b/src/tests/_internal/core/models/test_runs.py index a617ff1bd..54f07c1f1 100644 --- a/src/tests/_internal/core/models/test_runs.py +++ b/src/tests/_internal/core/models/test_runs.py @@ -117,100 +117,6 @@ def test_run_get_error_returns_none_for_specific_reasons(): assert reason in no_error_reasons -def _service_run_spec_dict(router_type=None, retry=None, top_level_extras=None): - """Build a minimal RunSpec dict for a service. - - `router_type`: None | "smg" | "dynamo" — controls whether/how the - second replica group has a router field. - `retry`: optional dict passed as `profile.retry`. - `top_level_extras`: optional dict merged into the service configuration. - """ - replicas = [{"name": "worker", "commands": ["echo hi"], "count": 1}] - if router_type is not None: - replicas.append( - { - "name": "router", - "router": {"type": router_type}, - "commands": ["echo router"], - "count": 1, - } - ) - configuration = { - "type": "service", - "port": 8000, - "replicas": replicas, - } - if top_level_extras: - configuration.update(top_level_extras) - profile = {"name": "default"} - if retry is not None: - profile["retry"] = retry - return { - "run_name": "test-run", - "repo_id": "test-repo", - "configuration_path": "dstack.yaml", - "configuration": configuration, - "profile": profile, - "ssh_key_pub": "ssh-rsa AAAA...", - "repo_data": {"repo_type": "virtual"}, - } - - -class TestDynamoNoRetryValidator: - def test_dynamo_router_with_retry_at_profile_level_is_rejected(self): - spec = _service_run_spec_dict( - router_type="dynamo", - retry={"on_events": ["error"]}, - ) - with pytest.raises(ValidationError, match="Dynamo"): - RunSpec.model_validate(spec) - - def test_dynamo_router_with_retry_in_configuration_is_rejected(self): - # retry can also be specified at configuration level; _merged_profile - # folds it into merged_profile.retry, so the validator should still - # catch it. - spec = _service_run_spec_dict( - router_type="dynamo", - top_level_extras={"retry": {"on_events": ["error"]}}, - ) - with pytest.raises(ValidationError, match="Dynamo"): - RunSpec.model_validate(spec) - - def test_dynamo_router_without_retry_is_accepted(self): - spec = _service_run_spec_dict(router_type="dynamo", retry=None) - # Should not raise: - RunSpec.model_validate(spec) - - def test_smg_router_with_retry_is_accepted(self): - spec = _service_run_spec_dict( - router_type="smg", - retry={"on_events": ["error"]}, - ) - # SMG services are unaffected by the validator. - RunSpec.model_validate(spec) - - def test_service_without_router_with_retry_is_accepted(self): - spec = _service_run_spec_dict(router_type=None, retry={"on_events": ["error"]}) - RunSpec.model_validate(spec) - - def test_non_service_run_with_retry_is_accepted(self): - # Validator is service-only. A task or dev-environment with retry - # shouldn't be flagged regardless of the rest of the spec. - spec = { - "run_name": "test-run", - "repo_id": "test-repo", - "configuration_path": "dstack.yaml", - "configuration": { - "type": "task", - "commands": ["echo hi"], - }, - "profile": {"name": "default", "retry": {"on_events": ["error"]}}, - "ssh_key_pub": "ssh-rsa AAAA...", - "repo_data": {"repo_type": "virtual"}, - } - RunSpec.model_validate(spec) - - class TestRunSpec: @pytest.mark.parametrize( "spec",