From bf6d00e23476e7805d52a75fc2b2c732a4252ac4 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 9 Nov 2026 13:30:20 +0000 Subject: [PATCH] Remove DSTACK_ROUTER_INTERNAL_IP The variable predates replica IP interpolation and is replaced by ${{ groups[i].replicas[j].IP_ADDRESS }}. Dynamo workers no longer wait for the router replica's internal IP, and their provisioning no longer loads every replica of the run. --- mkdocs/blog/posts/nvidia-dynamo.md | 3 + mkdocs/docs/guides/upgrade.md | 4 +- .../background/pipeline_tasks/jobs_running.py | 115 +-------- .../server/services/runner/client.py | 11 +- .../server/services/runs/replicas.py | 111 +-------- .../_internal/server/services/runs/spec.py | 14 +- .../pipeline_tasks/test_jobs_running.py | 222 +----------------- 7 files changed, 27 insertions(+), 453 deletions(-) diff --git a/mkdocs/blog/posts/nvidia-dynamo.md b/mkdocs/blog/posts/nvidia-dynamo.md index b46bc76150..0a7ac8adb4 100644 --- a/mkdocs/blog/posts/nvidia-dynamo.md +++ b/mkdocs/blog/posts/nvidia-dynamo.md @@ -18,6 +18,9 @@ With the latest update, `dstack` added native support for NVIDIA Dynamo with Pre +!!! note "Deprecation notice" + The `DSTACK_ROUTER_INTERNAL_IP` environment variable used in this blog post is no longer supported. For the modern approach, see the [NVIDIA Dynamo example](../../docs/examples/inference/dynamo.md). + ## About NVIDIA Dynamo [NVIDIA Dynamo](https://docs.nvidia.com/dynamo/getting-started/introduction) is an open-source, high-throughput, low-latency inference framework for serving generative AI workloads in distributed environments. It adds a system-level layer above inference engines such as SGLang, vLLM, and TensorRT-LLM, coordinating them across GPUs and nodes. diff --git a/mkdocs/docs/guides/upgrade.md b/mkdocs/docs/guides/upgrade.md index 5c69bef9fe..802c6894e9 100644 --- a/mkdocs/docs/guides/upgrade.md +++ b/mkdocs/docs/guides/upgrade.md @@ -28,9 +28,9 @@ Exceptions are noted in the sections below. Support for [exports](../concepts/exports.md) and the global `Events` page has moved from the open-source version to [dstack Sky](https://dstack.ai/products/sky/) and [dstack Factory](https://dstack.ai/products/factory/). The `dstack event` CLI command and the `Events` tabs on resource pages remain available in the open-source version. -### Deprecations +### `DSTACK_ROUTER_INTERNAL_IP` -The `DSTACK_ROUTER_INTERNAL_IP` environment variable is deprecated in favor of [`${{ groups[i].replicas[j].IP_ADDRESS }}`](../concepts/services.md#replica-groups). +Starting with `0.22.8`, `dstack` no longer sets the `DSTACK_ROUTER_INTERNAL_IP` environment variable. Use [`${{ groups[i].replicas[j].IP_ADDRESS }}`](../concepts/services.md#replica-groups) instead, e.g. to point Dynamo workers at a dedicated `etcd` replica group as shown in the [NVIDIA Dynamo example](../examples/inference/dynamo.md). ## 0.21.* { #0_21 } diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py index 01d6628af1..56dcbf0076 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py @@ -4,7 +4,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime, timedelta -from typing import Dict, Iterable, Optional, Sequence +from typing import Iterable, Optional, Sequence from sqlalchemy import and_, exists, false, func, or_, select, true, update from sqlalchemy.ext.asyncio import AsyncSession @@ -26,7 +26,6 @@ from dstack._internal.core.models.metrics import Metric from dstack._internal.core.models.profiles import StartupOrder from dstack._internal.core.models.repos import RemoteRepoCreds -from dstack._internal.core.models.routers import RouterType from dstack._internal.core.models.runs import ( ClusterInfo, ImagePullProgress, @@ -116,11 +115,6 @@ from dstack._internal.server.services.runner import client from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel from dstack._internal.server.services.runs import is_job_ready, run_model_to_run -from dstack._internal.server.services.runs.replicas import ( - RouterEnvStatus, - get_router_env_for_job, - get_router_replica_group, -) from dstack._internal.server.services.runs.spec import run_spec_has_replica_ip_refs from dstack._internal.server.services.secrets import get_project_secrets_mapping from dstack._internal.server.services.storage import get_default_storage @@ -143,8 +137,6 @@ JOB_STATUSES_WITH_MIN_PROCESSING_INTERVAL = [JobStatus.PROVISIONING, JobStatus.PULLING] -ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS = 30 * 60 - JOB_DISCONNECTED_RETRY_TIMEOUT = timedelta(minutes=2) """`The minimum time before terminating active job in case of connectivity issues.""" @@ -421,12 +413,8 @@ async def _load_process_context(item: JobRunningPipelineItem) -> Optional[_Proce ) else: # PROVISIONING/PULLING jobs need same-replica siblings for cluster - # coordination, plus — when the run has a router replica group — - # the router replica's job (cross-replica) so the router checks in - # _terminate_if_router_unavailable and _prepare_runner_submission - # can read its status / IP. - # _fetch_run_model handles both: same-replica jobs always, plus - # all non-terminated jobs when one exists. + # coordination, plus all non-terminated jobs of the run when commands + # reference other replicas' IPs. _fetch_run_model handles both. run_spec = validate_json_extra_ignore(RunSpec, job_model.run.run_spec) run_model = await _fetch_run_model( session=session, @@ -477,12 +465,8 @@ async def _process_running_job(context: _ProcessContext) -> _ProcessResult: return result if context.job_model.status == JobStatus.PROVISIONING: - if _terminate_if_router_unavailable(context=context, result=result): - return result await _process_provisioning_status(context=context, result=result) elif context.job_model.status == JobStatus.PULLING: - if _terminate_if_router_unavailable(context=context, result=result): - return result await _process_pulling_status(context=context, result=result) elif context.job_model.status == JobStatus.RUNNING: if _server_access_enabled(context): @@ -509,53 +493,6 @@ async def _process_running_job(context: _ProcessContext) -> _ProcessResult: return result -def _terminate_if_router_unavailable(context: _ProcessContext, result: _ProcessResult) -> bool: - """ - Terminates a worker job whose router replica is in a terminal state or did not acquire an - internal IP in time. Waiting for the router IP is up to `_prepare_runner_submission`. - - Returns `True` if the job was terminated. - """ - # `get_router_env_for_job` returns None for the router itself and for runs without - # a router group, so this check is a no-op in those cases. - router_env = get_router_env_for_job( - run_model=context.run_model, - run_spec=context.run.run_spec, - job_model=context.job_model, - ) - if router_env is RouterEnvStatus.FAILED: - # Router has reached a terminal state — the worker cannot recover by - # waiting. Terminate it now with a clear reason instead of letting it - # idle until the run-level reconciler tears the whole run down. - _terminate_job( - job_model=context.job_model, - job_update_map=result.job_update_map, - termination_reason=JobTerminationReason.TERMINATED_BY_SERVER, - termination_reason_message=( - "Router replica is in a terminal state; cannot provision worker " - "without a running router." - ), - ) - return True - if router_env is not RouterEnvStatus.NOT_PROVISIONED: - return False - # Bound the wait for the router IP so a router that is genuinely stuck can't burn - # worker instance-hours forever; see ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS. - waited_seconds = (get_current_datetime() - context.job_model.submitted_at).total_seconds() - if waited_seconds <= ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS: - return False - _terminate_job( - job_model=context.job_model, - job_update_map=result.job_update_map, - termination_reason=JobTerminationReason.TERMINATED_BY_SERVER, - termination_reason_message=( - f"Router replica did not acquire an internal IP within " - f"{ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS}s; terminating worker." - ), - ) - return True - - async def _refetch_locked_job_model( session: AsyncSession, item: JobRunningPipelineItem ) -> Optional[JobModel]: @@ -589,12 +526,10 @@ async def _fetch_run_model( Args: replica_num: If None, skip loading jobs (for RUNNING jobs that don't need siblings). If set, load only latest-submission jobs for that replica (for PROVISIONING/PULLING - jobs that need same-replica siblings for cluster coordination). When the run has - a Dynamo router replica group, or any command contains - ${{ groups[i].replicas[j].IP_ADDRESS }}, all non-terminated latest-submission - jobs for the run are loaded so sibling replica IPs are visible. - run_spec: Required whenever `replica_num` is set. Used to detect whether the run - has a Dynamo router replica group or replica IP refs. + jobs that need same-replica siblings for cluster coordination). When any command + contains ${{ groups[i].replicas[j].IP_ADDRESS }}, all non-terminated + latest-submission jobs for the run are loaded so sibling replica IPs are visible. + run_spec: Required whenever `replica_num` is set. Used to detect replica IP refs. """ query = ( select(RunModel) @@ -616,13 +551,7 @@ async def _fetch_run_model( ) if replica_num is not None: assert run_spec is not None, "run_spec must be provided when replica_num is set" - router_group = get_router_replica_group(run_spec) - is_dynamo = ( - router_group is not None - and router_group.router is not None - and router_group.router.type == RouterType.DYNAMO - ) - load_all_replicas = is_dynamo or run_spec_has_replica_ip_refs(run_spec) + load_all_replicas = run_spec_has_replica_ip_refs(run_spec) latest_submissions_sq = ( select( @@ -633,8 +562,8 @@ async def _fetch_run_model( ) .where( JobModel.run_id == run_id, - # Dynamo and replica-IP interpolation need sibling replicas. - # Other services load only the worker's own replica. + # Replica-IP interpolation needs sibling replicas. + # Other runs load only the job's own replica. true() if load_all_replicas else JobModel.replica_num == replica_num, ) .group_by(JobModel.run_id, JobModel.replica_num, JobModel.job_num) @@ -794,7 +723,6 @@ async def _process_provisioning_status( code=code, file_archives=file_archives, repo_credentials=runner_submission.repo_creds, - router_env=runner_submission.router_env, success_if_not_available=False, ) _apply_submit_job_to_runner_result( @@ -914,7 +842,6 @@ async def _process_pulling_status( code=code, file_archives=file_archives, repo_credentials=runner_submission.repo_creds, - router_env=runner_submission.router_env, success_if_not_available=True, ) _apply_submit_job_to_runner_result( @@ -1347,12 +1274,6 @@ def _interpolate_secrets_or_terminate( class _RunnerSubmissionContext: cluster_info: ClusterInfo repo_creds: Optional[RemoteRepoCreds] - router_env: Optional[Dict[str, str]] = None - """Dynamo-specific env (e.g. DSTACK_ROUTER_INTERNAL_IP) computed from the - router replica's state. Passed through to RunnerClient.submit_job, which - merges it into a deep-copied job_spec.env so the shared job_spec is not - mutated. None for SMG services, non-router runs, and the router - replica itself.""" async def _prepare_runner_submission( @@ -1368,19 +1289,6 @@ async def _prepare_runner_submission( if _should_wait_for_replica_provisioning(context): return None - router_env = get_router_env_for_job( - run_model=context.run_model, - run_spec=context.run.run_spec, - job_model=context.job_model, - ) - if isinstance(router_env, RouterEnvStatus): - # `FAILED` and a timed-out wait are handled by `_terminate_if_router_unavailable` - logger.debug( - "%s: waiting for router replica to be provisioned", - fmt(context.job_model), - ) - return None - cluster_info = _get_cluster_info( jobs=context.run.jobs, replica_num=context.job.job_spec.replica_num, @@ -1426,7 +1334,6 @@ async def _prepare_runner_submission( return _RunnerSubmissionContext( cluster_info=cluster_info, repo_creds=repo_creds, - router_env=router_env, ) @@ -1697,7 +1604,6 @@ def _submit_job_to_runner( code: Optional[bytes], file_archives: Iterable[tuple[uuid.UUID, bytes]], repo_credentials: Optional[RemoteRepoCreds], - router_env: Optional[Dict[str, str]], success_if_not_available: bool, ) -> _SubmitJobToRunnerResult: logger.debug("%s: submitting job spec", fmt(job_model)) @@ -1726,7 +1632,6 @@ def _submit_job_to_runner( secrets={}, repo_credentials=repo_credentials, instance_env=instance_env, - router_env=router_env, ) for archive_id, archive in file_archives: logger.debug("%s: uploading file archive: %s", fmt(job_model), archive_id) diff --git a/src/dstack/_internal/server/services/runner/client.py b/src/dstack/_internal/server/services/runner/client.py index c85465fa70..10ae8ff138 100644 --- a/src/dstack/_internal/server/services/runner/client.py +++ b/src/dstack/_internal/server/services/runner/client.py @@ -272,15 +272,14 @@ def submit_job( secrets: Dict[str, str], repo_credentials: Optional[RemoteRepoCreds], instance_env: Optional[Union[Env, Dict[str, str]]] = None, - router_env: Optional[Dict[str, str]] = None, ): # XXX: This is a quick-and-dirty hack to deliver InstanceModel-specific - # and Dynamo-router environment variables to the runner without runner - # API modification. Both layers are merged into a deep-copied job_spec - # so the shared spec object held by the caller is not mutated. + # environment variables to the runner without runner API modification. + # They are merged into a deep-copied job_spec so the shared spec object + # held by the caller is not mutated. job_spec = job.job_spec server_access = bool(getattr(run.run_spec.configuration, "dstack", False)) - if instance_env is not None or router_env is not None or server_access: + if instance_env is not None or server_access: merged_env: Dict[str, str] = {} if instance_env is not None: if isinstance(instance_env, Env): @@ -288,8 +287,6 @@ def submit_job( else: merged_env.update(instance_env) merged_env.update(job_spec.env) - if router_env is not None: - merged_env.update(router_env) if server_access: merged_env.setdefault(DSTACK_PROJECT_ENV, run.project_name) job_spec = job_spec.model_copy(deep=True) diff --git a/src/dstack/_internal/server/services/runs/replicas.py b/src/dstack/_internal/server/services/runs/replicas.py index d917599472..df29a65b9c 100644 --- a/src/dstack/_internal/server/services/runs/replicas.py +++ b/src/dstack/_internal/server/services/runs/replicas.py @@ -1,15 +1,12 @@ from dataclasses import dataclass -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union +from typing import List, Optional, Tuple -from dstack._internal.core.models.configurations import ReplicaGroup, ServiceConfiguration +from dstack._internal.core.models.configurations import ReplicaGroup from dstack._internal.core.models.gateways import GatewayReplicaStatus -from dstack._internal.core.models.routers import RouterType -from dstack._internal.core.models.runs import JobStatus, JobTerminationReason, RunSpec +from dstack._internal.core.models.runs import JobStatus, JobTerminationReason from dstack._internal.server.models import JobModel, RunModel from dstack._internal.server.services.gateways import get_gateway_replica_models from dstack._internal.server.services.jobs import ( - get_job_provisioning_data, get_job_spec, group_jobs_by_replica_latest, ) @@ -25,31 +22,6 @@ class GroupRolloutState: receiving_traffic_non_terminating_replica_count: int -class RouterEnvStatus(str, Enum): - """Outcomes returned from get_router_env_for_job() when no env dict is - appropriate. Each value carries a distinct caller-side action. - - Using an enum (rather than empty-dict sentinels) means callers can rely - on either `is` or `==` to compare — both yield correct, unambiguous - results — and stray dicts from elsewhere can never accidentally match. - - NOT_PROVISIONED — router job exists but its internal_ip is not yet - known. Transient; caller should defer this worker - and retry on the next pipeline tick (subject to - ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS in - jobs_running.py). - FAILED — router job has reached a terminal state - (TERMINATING/TERMINATED/FAILED/ABORTED/DONE). - Permanent; caller should stop deferring and - terminate this worker — waiting longer cannot - recover because the router will not come back with - a fresh internal_ip. - """ - - NOT_PROVISIONED = "not_provisioned" - FAILED = "failed" - - def build_replica_lists( run_model: RunModel, group_filter: Optional[str] = None, @@ -183,80 +155,3 @@ def is_replica_receiving_traffic(run_model: RunModel, jobs: list[JobModel]) -> b if registration is None or not registration.is_registered: return False return True - - -def get_router_replica_group(run_spec: RunSpec) -> Optional[ReplicaGroup]: - """Return the (single) replica group with a `router:` field, or None. - - `validate_at_most_one_router_replica_group` guarantees at most one such - group exists, so we can safely return on the first match. - """ - cfg = run_spec.configuration - if not isinstance(cfg, ServiceConfiguration): - return None - for g in cfg.replica_groups: - if g.router is not None: - return g - return None - - -def find_router_job(run_model: RunModel, router_group_name: str) -> Optional[JobModel]: - for j in run_model.jobs: - if job_belongs_to_group(j, router_group_name): - return j - return None - - -def get_router_env_for_job( - run_model: RunModel, run_spec: RunSpec, job_model: JobModel -) -> Optional[Union[Dict[str, str], RouterEnvStatus]]: - """Compute env vars exposing the router replica's address to a worker job. - - Returns one of four values, each communicating a distinct outcome: - - None -> not applicable. Either the - run has no router replica - group, or this job IS the - router replica. Caller does - nothing. - RouterEnvStatus.NOT_PROVISIONED -> router job exists but has no - internal_ip yet. Caller defers. - RouterEnvStatus.FAILED -> router job has reached a - terminal state and can never - expose an internal_ip. Caller - terminates this worker; - waiting cannot recover. - {"DSTACK_ROUTER_INTERNAL_IP": ...} -> ready-to-merge env dict - containing the router - replica's internal IP. - """ - router_group = get_router_replica_group(run_spec) - if router_group is None or router_group.name is None: - return None - # DSTACK_ROUTER_INTERNAL_IP is Dynamo-specific. SMG workers - # are registered via the worker-sync pipeline (ServiceRouterWorkerSyncModel) - if router_group.router is None or router_group.router.type != RouterType.DYNAMO: - return None - if job_belongs_to_group(job_model, router_group.name): - # Router replica itself doesn't need to be told its own IP. - return None - - router_job = find_router_job(run_model, router_group.name) - if router_job is None: - # The router's latest submission is in a terminal state and was - # filtered out by _fetch_run_model's not-terminated predicate. - return RouterEnvStatus.FAILED - - # If the router has reached a terminal state, the worker cannot recover - # by waiting — the router will not come back with a fresh internal_ip - # under the same job. Surface this as FAILED so the caller can stop - # the wait loop and terminate the worker with a clear reason. - if router_job.status == JobStatus.TERMINATING or router_job.status.is_finished(): - return RouterEnvStatus.FAILED - - # Router is alive but may not yet have been assigned a machine. - jpd = get_job_provisioning_data(router_job) - if jpd is None or not jpd.internal_ip: - return RouterEnvStatus.NOT_PROVISIONED - - return {"DSTACK_ROUTER_INTERNAL_IP": jpd.internal_ip} diff --git a/src/dstack/_internal/server/services/runs/spec.py b/src/dstack/_internal/server/services/runs/spec.py index 47e3143b3f..1727c39ee8 100644 --- a/src/dstack/_internal/server/services/runs/spec.py +++ b/src/dstack/_internal/server/services/runs/spec.py @@ -439,20 +439,14 @@ def _check_dynamo_in_place_update_compatibility( for field in _router_affecting_top_level_fields: if getattr(current_cfg, field, None) != getattr(new_cfg, field, None): raise ServerClientError( - f"Cannot change top-level `{field}` in place when the " - f"service has a Dynamo router (would re-provision the " - f"router and invalidate workers' cached " - f"DSTACK_ROUTER_INTERNAL_IP). Stop the run with " - f"`dstack stop` and re-apply." + f"Cannot change top-level `{field}` in place when the service has a Dynamo router." + f" Stop the run with `dstack stop` and re-apply." ) for field in _TYPE_SPECIFIC_UPDATABLE_SPEC_FIELDS.get("service", []): if getattr(current_run_spec, field, None) != getattr(new_run_spec, field, None): raise ServerClientError( - f"Cannot change top-level `{field}` in place when the " - f"service has a Dynamo router (would re-provision the " - f"router and invalidate workers' cached " - f"DSTACK_ROUTER_INTERNAL_IP). Stop the run with " - f"`dstack stop` and re-apply." + f"Cannot change top-level `{field}` in place when the service has a Dynamo router." + f" Stop the run with `dstack stop` and re-apply." ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_running.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_running.py index 2de105396f..e5b115ce44 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_running.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_running.py @@ -46,7 +46,6 @@ from dstack._internal.server.background.pipeline_tasks.jobs_running import ( JOB_DISCONNECTED_RETRY_TIMEOUT, MAX_DURATION_ENFORCEMENT_GRACE, - ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS, JobRunningFetcher, JobRunningPipeline, JobRunningPipelineItem, @@ -61,7 +60,6 @@ _referenced_ips_ready, _should_wait_for_replica_provisioning, _SubmitJobToRunnerResult, - _terminate_if_router_unavailable, ) from dstack._internal.server.background.pipeline_tasks.runs import RunPipeline from dstack._internal.server.models import JobModel, ProbeModel, ServiceReplicaRegistrationModel @@ -76,7 +74,6 @@ from dstack._internal.server.services.jobs import server_connection from dstack._internal.server.services.jobs.server_connection import job_server_connections_pool from dstack._internal.server.services.runner.client import RunnerClient, ShimClient -from dstack._internal.server.services.runs.replicas import RouterEnvStatus from dstack._internal.server.services.volumes import volume_model_to_volume from dstack._internal.server.testing.common import ( clear_events, @@ -3354,142 +3351,6 @@ def _replica_ip_refs_service_configuration() -> ServiceConfiguration: ) -def _make_router_worker_context(*, submitted_at: datetime) -> _ProcessContext: - job_model = MagicMock() - job_model.submitted_at = submitted_at - job = MagicMock() - job.job_spec.replica_num = 0 - run = MagicMock() - run.jobs = [] - run.run_spec = MagicMock() - return _ProcessContext( - job_model=job_model, - run_model=MagicMock(), - run=run, - job=job, - job_submission=MagicMock(job_runtime_data=None), - job_provisioning_data=MagicMock(), - instance_access_revoked=False, - ) - - -class TestTerminateIfRouterUnavailable: - def test_router_failed_terminates_worker(self): - context = _make_router_worker_context( - submitted_at=datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - ) - result = _ProcessResult() - with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=RouterEnvStatus.FAILED, - ): - assert _terminate_if_router_unavailable(context=context, result=result) - assert ( - result.job_update_map.get("termination_reason") - == JobTerminationReason.TERMINATED_BY_SERVER - ) - assert "Router replica is in a terminal state" in ( - result.job_update_map.get("termination_reason_message") or "" - ) - - @time_machine.travel(datetime(2023, 1, 1, 12, tzinfo=timezone.utc), tick=False) - def test_router_not_provisioned_within_timeout_does_not_terminate(self): - context = _make_router_worker_context( - submitted_at=datetime(2023, 1, 1, 11, 45, 0, tzinfo=timezone.utc), - ) - result = _ProcessResult() - with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=RouterEnvStatus.NOT_PROVISIONED, - ): - assert not _terminate_if_router_unavailable(context=context, result=result) - assert result.job_update_map == {} - - @time_machine.travel(datetime(2023, 1, 1, 12, tzinfo=timezone.utc), tick=False) - def test_router_not_provisioned_past_timeout_terminates(self): - context = _make_router_worker_context( - submitted_at=datetime(2023, 1, 1, 10, 0, 0, tzinfo=timezone.utc), - ) - result = _ProcessResult() - with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=RouterEnvStatus.NOT_PROVISIONED, - ): - assert _terminate_if_router_unavailable(context=context, result=result) - assert ( - result.job_update_map.get("termination_reason") - == JobTerminationReason.TERMINATED_BY_SERVER - ) - msg = result.job_update_map.get("termination_reason_message") or "" - assert str(ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS) in msg - assert "internal IP" in msg - - -@pytest.mark.asyncio -class TestPrepareRunnerSubmissionRouterEnv: - @time_machine.travel(datetime(2023, 1, 1, 12, tzinfo=timezone.utc), tick=False) - async def test_router_not_provisioned_waits(self): - context = _make_router_worker_context( - submitted_at=datetime(2023, 1, 1, 11, 45, 0, tzinfo=timezone.utc), - ) - result = _ProcessResult() - with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=RouterEnvStatus.NOT_PROVISIONED, - ): - out = await _prepare_runner_submission(context=context, result=result) - assert out is None - assert result.job_update_map == {} - - async def test_router_env_dict_populates_submission_context(self): - context = _make_router_worker_context( - submitted_at=datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc), - ) - result = _ProcessResult() - router_env = {"DSTACK_ROUTER_INTERNAL_IP": "10.1.2.3"} - - @asynccontextmanager - async def _fake_session_ctx(): - yield MagicMock() - - with ( - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=router_env, - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_session_ctx", - _fake_session_ctx, - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_repo_creds", - new_callable=AsyncMock, - return_value=None, - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_project_secrets_mapping", - new_callable=AsyncMock, - return_value={}, - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.repo_model_to_repo_head_with_creds", - return_value=MagicMock(repo_creds=None), - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.interpolate_job_spec_secrets", - ), - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running._get_cluster_info", - return_value=ClusterInfo( - job_ips=["10.0.0.1"], master_job_ip="10.0.0.1", gpus_per_job=0 - ), - ), - ): - out = await _prepare_runner_submission(context=context, result=result) - assert out is not None - assert out.router_env == router_env - - @pytest.mark.asyncio class TestPrepareRunnerSubmissionClusterWait: def _make_context(self) -> _ProcessContext: @@ -3561,10 +3422,6 @@ async def _fake_session_ctx(): yield MagicMock() return ( - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=None, - ), patch( "dstack._internal.server.background.pipeline_tasks.jobs_running.get_session_ctx", _fake_session_ctx, @@ -3753,10 +3610,6 @@ async def _fake_session_ctx(): yield MagicMock() return ( - patch( - "dstack._internal.server.background.pipeline_tasks.jobs_running.get_router_env_for_job", - return_value=None, - ), patch( "dstack._internal.server.background.pipeline_tasks.jobs_running.get_session_ctx", _fake_session_ctx, @@ -3830,80 +3683,7 @@ async def test_nodes_ref_in_service_terminates(self): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) -class TestFetchRunModelDynamoBranch: - async def test_dynamo_run_loads_all_non_terminated_replicas( - self, test_db, session: AsyncSession - ): - project = await create_project(session=session) - user = await create_user(session=session) - repo = await create_repo(session=session, project_id=project.id) - run_spec = get_run_spec( - repo_id=repo.name, - configuration=_router_service_configuration("dynamo"), - ) - run = await create_run( - session=session, project=project, repo=repo, user=user, run_spec=run_spec - ) - await create_job( - session=session, - run=run, - replica_num=0, - status=JobStatus.PROVISIONING, - ) - await create_job( - session=session, - run=run, - replica_num=1, - status=JobStatus.PROVISIONING, - ) - run_id = run.id - parsed = validate_json_extra_ignore(RunSpec, get_or_error(run.run_spec)) - await session.commit() - session.expire_all() - run_model = await _fetch_run_model( - session=session, - run_id=run_id, - replica_num=0, - run_spec=parsed, - ) - assert {j.replica_num for j in run_model.jobs} == {0, 1} - - async def test_non_dynamo_loads_only_own_replica(self, test_db, session: AsyncSession): - """Without a Dynamo router, eager-load only the worker replica's jobs.""" - project = await create_project(session=session) - user = await create_user(session=session) - repo = await create_repo(session=session, project_id=project.id) - run_spec = get_run_spec( - repo_id=repo.name, - configuration=_router_service_configuration("smg"), - ) - run = await create_run( - session=session, project=project, repo=repo, user=user, run_spec=run_spec - ) - await create_job( - session=session, - run=run, - replica_num=0, - status=JobStatus.PROVISIONING, - ) - await create_job( - session=session, - run=run, - replica_num=1, - status=JobStatus.PROVISIONING, - ) - run_id = run.id - parsed = validate_json_extra_ignore(RunSpec, get_or_error(run.run_spec)) - await session.commit() - session.expire_all() - run_model = await _fetch_run_model( - session=session, - run_id=run_id, - replica_num=0, - run_spec=parsed, - ) - assert {j.replica_num for j in run_model.jobs} == {0} - +class TestFetchRunModel: async def test_smg_with_replica_ip_refs_loads_all_replicas( self, test_db, session: AsyncSession ):