From f43c05815a435fa005b51ac422a00c5a309c0394 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 10 Nov 2026 11:03:37 +0000 Subject: [PATCH] Allow in-place updates of services with `type: dynamo` router The check rejected changes that would roll the router or change the router type, since workers got the router's internal IP only once. DSTACK_ROUTER_INTERNAL_IP was removed in #4473, so Dynamo services can now be updated like any other service. Router type changes are allowed too: only the SMG worker sync reads the type, and it follows the new spec. Mid-rollout errors are expected, as old and new routers and workers are incompatible, but the error window is no worse than with stopping and re-applying. Co-Authored-By: Claude Opus 5.5 --- .../_internal/server/services/runs/spec.py | 65 ------- .../server/services/runs/test_spec.py | 178 ++++-------------- 2 files changed, 35 insertions(+), 208 deletions(-) diff --git a/src/dstack/_internal/server/services/runs/spec.py b/src/dstack/_internal/server/services/runs/spec.py index 1727c39ee..7c5b20f14 100644 --- a/src/dstack/_internal/server/services/runs/spec.py +++ b/src/dstack/_internal/server/services/runs/spec.py @@ -13,7 +13,6 @@ from dstack._internal.core.models.profiles import ProfileRetry from dstack._internal.core.models.repos.virtual import DEFAULT_VIRTUAL_REPO_ID, VirtualRunRepoData from dstack._internal.core.models.resources import GPUSpec, ResourcesSpec -from dstack._internal.core.models.routers import RouterType from dstack._internal.core.models.runs import LEGACY_REPO_DIR, AnyRunConfiguration, RunSpec from dstack._internal.core.models.volumes import InstanceMountPoint from dstack._internal.core.services import validate_dstack_resource_name @@ -387,69 +386,6 @@ def _get_replica_group_image_and_docker( return image, docker -def _check_dynamo_in_place_update_compatibility( - current_run_spec: RunSpec, new_run_spec: RunSpec -) -> None: - """Reject in-place updates that would re-provision a Dynamo router. - - Workers cache the router internal IP at provisioning time; changes that - trigger a rolling router update must not be applied in place. - """ - current_cfg = current_run_spec.configuration - new_cfg = new_run_spec.configuration - if not isinstance(current_cfg, ServiceConfiguration) or not isinstance( - new_cfg, ServiceConfiguration - ): - return - - current_router_group = next( - (g for g in current_cfg.replica_groups if g.router is not None), None - ) - new_router_group = next((g for g in new_cfg.replica_groups if g.router is not None), None) - current_router_type = ( - current_router_group.router.type - if current_router_group is not None and current_router_group.router is not None - else None - ) - new_router_type = ( - new_router_group.router.type - if new_router_group is not None and new_router_group.router is not None - else None - ) - if ( - current_router_type is not None - and new_router_type is not None - and current_router_type != new_router_type - ): - raise ServerClientError( - "Cannot change router.type in place. Stop the run with `dstack stop` and re-apply." - ) - if RouterType.DYNAMO not in (current_router_type, new_router_type): - return - if current_router_group != new_router_group: - raise ServerClientError( - "Cannot update a Dynamo router replica group in place. " - "Stop the run with `dstack stop` and re-apply." - ) - _router_affecting_top_level_fields = tuple( - f - for f in _TYPE_SPECIFIC_CONF_UPDATABLE_FIELDS.get("service", []) - if f not in ("replicas", "groups", "scaling", "gateway") - ) - for field in _router_affecting_top_level_fields: - if getattr(current_cfg, field, None) != getattr(new_cfg, field, None): - raise ServerClientError( - f"Cannot change top-level `{field}` in place when the service has a Dynamo router." - f" Stop the run with `dstack stop` and re-apply." - ) - for field in _TYPE_SPECIFIC_UPDATABLE_SPEC_FIELDS.get("service", []): - if getattr(current_run_spec, field, None) != getattr(new_run_spec, field, None): - raise ServerClientError( - f"Cannot change top-level `{field}` in place when the service has a Dynamo router." - f" Stop the run with `dstack stop` and re-apply." - ) - - def check_can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec) -> ModelDiff: """ Check if in-place update is possible. @@ -468,7 +404,6 @@ def check_can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec) f"Failed to update fields {changed_spec_fields}." f" Can only update {updatable_spec_fields}." ) - _check_dynamo_in_place_update_compatibility(current_run_spec, new_run_spec) # We don't allow update if the order of archives has been changed, as even if the archives # are the same (the same id => hash => content and the same container path), the order of # unpacking matters when one path is a subpath of another. diff --git a/src/tests/_internal/server/services/runs/test_spec.py b/src/tests/_internal/server/services/runs/test_spec.py index bcfe8f17a..c37a49216 100644 --- a/src/tests/_internal/server/services/runs/test_spec.py +++ b/src/tests/_internal/server/services/runs/test_spec.py @@ -1,5 +1,4 @@ import re -import uuid from types import SimpleNamespace from typing import Any, Optional, cast @@ -13,9 +12,7 @@ TaskConfiguration, ) from dstack._internal.core.models.envs import Env -from dstack._internal.core.models.files import FileArchiveMapping from dstack._internal.core.models.profiles import Profile, ProfileRetry -from dstack._internal.core.models.repos.local import LocalRunRepoData from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import RunSpec from dstack._internal.server.models import UserModel @@ -34,21 +31,10 @@ def _service_configuration( router_type=None, image=None, env=None, - worker_count_min=None, router_commands="echo router", - worker_commands="echo worker", ): # Build a ServiceConfiguration instance for the in-place update tests. - worker = { - "name": "worker", - "commands": [worker_commands], - } - if worker_count_min is None: - worker["count"] = 1 - else: - worker["count"] = {"min": worker_count_min, "max": worker_count_min + 1} - worker["scaling"] = {"metric": "rps", "target": 4} - replicas = [worker] + replicas = [{"name": "worker", "commands": ["echo worker"], "count": 1}] if router_type is not None: replicas.append( { @@ -87,21 +73,6 @@ def _run_spec(configuration: ServiceConfiguration, **kwargs): ) -def _run_spec_with_overrides(configuration: ServiceConfiguration, **overrides) -> RunSpec: - get_run_spec_keys = {"repo_code_hash", "repo_data"} - get_run_spec_kwargs = {k: v for k, v in overrides.items() if k in get_run_spec_keys} - run_spec_overrides = {k: v for k, v in overrides.items() if k not in get_run_spec_keys} - run_spec = get_run_spec( - repo_id="test-repo", - run_name="test-run", - configuration=configuration, - **get_run_spec_kwargs, - ) - if not run_spec_overrides: - return run_spec - return RunSpec.model_validate({**run_spec.model_dump(), **run_spec_overrides}) - - def _task_run_spec( *, resources: Optional[dict] = None, @@ -456,134 +427,55 @@ def test_false_for_task_node_refs(self): assert not run_spec_has_replica_ip_refs(run_spec) -class TestCheckCanUpdateConfigurationRouterType: - def test_smg_to_dynamo_router_type_change_is_rejected(self): - current = _run_spec(_service_configuration(router_type="smg")) - new = _run_spec(_service_configuration(router_type="dynamo")) - with pytest.raises(ServerClientError, match="router.type"): - check_can_update_run_spec(current, new) - - def test_dynamo_to_smg_router_type_change_is_rejected(self): - current = _run_spec(_service_configuration(router_type="dynamo")) - new = _run_spec(_service_configuration(router_type="smg")) - with pytest.raises(ServerClientError, match="router.type"): - check_can_update_run_spec(current, new) - - def test_same_router_type_no_other_changes_succeeds(self): - current = _run_spec(_service_configuration(router_type="dynamo")) - new = _run_spec(_service_configuration(router_type="dynamo")) - check_can_update_run_spec(current, new) - - -class TestCheckCanUpdateConfigurationDynamoRouterGroup: - def test_dynamo_router_group_commands_change_is_rejected(self): - current = _run_spec(_service_configuration(router_type="dynamo", router_commands="a")) - new = _run_spec(_service_configuration(router_type="dynamo", router_commands="b")) - with pytest.raises(ServerClientError, match="Dynamo router replica group"): - check_can_update_run_spec(current, new) - - -class TestCheckCanUpdateConfigurationDynamoTopLevel: - def test_dynamo_top_level_image_change_is_rejected(self): - current = _run_spec(_service_configuration(router_type="dynamo", image="img:1")) - new = _run_spec(_service_configuration(router_type="dynamo", image="img:2")) - with pytest.raises(ServerClientError, match="image.*Dynamo"): - check_can_update_run_spec(current, new) - - def test_dynamo_top_level_env_change_is_rejected(self): - current = _run_spec(_service_configuration(router_type="dynamo", env={"FOO": "1"})) - new = _run_spec(_service_configuration(router_type="dynamo", env={"FOO": "2"})) - with pytest.raises(ServerClientError, match="env.*Dynamo"): - check_can_update_run_spec(current, new) - - -class TestCheckCanUpdateRunSpecDynamoSpecLevel: +class TestCheckCanUpdateRunSpecRouter: @pytest.mark.parametrize( - ("field", "current_overrides", "new_overrides"), + ("current_router_type", "new_router_type"), [ - pytest.param( - "repo_code_hash", - {"repo_code_hash": "hash-a"}, - {"repo_code_hash": "hash-b"}, - id="repo_code_hash", - ), - pytest.param( - "repo_data", - {"repo_data": LocalRunRepoData(repo_dir="/repo/a")}, - {"repo_data": LocalRunRepoData(repo_dir="/repo/b")}, - id="repo_data", - ), - pytest.param( - "file_archives", - { - "file_archives": [ - FileArchiveMapping( - id=uuid.UUID("00000000-0000-0000-0000-000000000001"), - path="/work/a.txt", - ), - ], - }, - { - "file_archives": [ - FileArchiveMapping( - id=uuid.UUID("00000000-0000-0000-0000-000000000002"), - path="/work/b.txt", - ), - ], - }, - id="file_archives", - ), - pytest.param( - "working_dir", - {"working_dir": "/old-top"}, - {"working_dir": "/new-top"}, - id="working_dir", - ), + pytest.param("smg", "dynamo", id="smg-to-dynamo"), + pytest.param("dynamo", "smg", id="dynamo-to-smg"), ], ) - def test_dynamo_spec_level_field_change_is_rejected( - self, field: str, current_overrides: dict, new_overrides: dict - ) -> None: - cfg = _service_configuration(router_type="dynamo") - current = _run_spec_with_overrides(cfg, **current_overrides) - new = _run_spec_with_overrides(cfg, **new_overrides) - - with pytest.raises(ServerClientError, match=re.escape(field)): - check_can_update_run_spec(current, new) - - -class TestCheckCanUpdateConfigurationWorkerOnlyChangesAllowed: - def test_dynamo_worker_count_min_change_is_allowed(self): - current = _run_spec(_service_configuration(router_type="dynamo", worker_count_min=1)) - new = _run_spec(_service_configuration(router_type="dynamo", worker_count_min=2)) - # Worker group count change is allowed on a Dynamo service. + def test_allows_changing_router_type(self, current_router_type: str, new_router_type: str): + current = _run_spec(_service_configuration(router_type=current_router_type)) + new = _run_spec(_service_configuration(router_type=new_router_type)) check_can_update_run_spec(current, new) - def test_dynamo_worker_commands_change_is_allowed(self): - current = _run_spec(_service_configuration(router_type="dynamo", worker_commands="x")) - new = _run_spec(_service_configuration(router_type="dynamo", worker_commands="y")) - # Non-router replica group's commands change is allowed. + @pytest.mark.parametrize("router_type", ["smg", "dynamo"]) + def test_allows_changing_router_group(self, router_type: str): + current = _run_spec(_service_configuration(router_type=router_type, router_commands="a")) + new = _run_spec(_service_configuration(router_type=router_type, router_commands="b")) check_can_update_run_spec(current, new) - -class TestCheckCanUpdateConfigurationNonDynamoUnchanged: - def test_smg_top_level_image_change_is_allowed(self): - current = _run_spec(_service_configuration(router_type="smg", image="img:1")) - new = _run_spec(_service_configuration(router_type="smg", image="img:2")) - # Top-level changes on SMG services flow through to the existing - # rolling-deployment path; no Dynamo gate fires. + @pytest.mark.parametrize("router_type", [None, "smg", "dynamo"]) + def test_allows_changing_top_level_fields(self, router_type: Optional[str]): + current = _run_spec( + _service_configuration(router_type=router_type, image="img:1", env={"FOO": "1"}) + ) + new = _run_spec( + _service_configuration(router_type=router_type, image="img:2", env={"FOO": "2"}) + ) check_can_update_run_spec(current, new) - def test_no_router_top_level_image_change_is_allowed(self): - current = _run_spec(_service_configuration(router_type=None, image="img:1")) - new = _run_spec(_service_configuration(router_type=None, image="img:2")) - check_can_update_run_spec(current, new) + @pytest.mark.parametrize( + ("current_router_type", "new_router_type"), + [ + pytest.param(None, "smg", id="add-router"), + pytest.param("dynamo", None, id="remove-router"), + ], + ) + def test_rejects_adding_or_removing_router( + self, current_router_type: Optional[str], new_router_type: Optional[str] + ): + current = _run_spec(_service_configuration(router_type=current_router_type)) + new = _run_spec(_service_configuration(router_type=new_router_type)) + with pytest.raises(ServerClientError, match="router replica groups"): + check_can_update_run_spec(current, new) class TestCheckCanUpdateConfigurationFieldAllowlist: """`_check_can_update_configuration` is also called directly with configs only.""" - def test_non_dynamo_image_change_passes_configuration_gate(self): + def test_image_change_passes_configuration_gate(self): current = _service_configuration(router_type="smg", image="img:1") new = _service_configuration(router_type="smg", image="img:2") _check_can_update_configuration(current, new, ignore_files=True)