Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions mkdocs/blog/posts/nvidia-dynamo.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ With the latest update, `dstack` added native support for NVIDIA Dynamo with Pre

<!-- more -->

!!! note "Deprecation notice"
The `DSTACK_ROUTER_INTERNAL_IP` environment variable used in this blog post is no longer supported. For the modern approach, see the [NVIDIA Dynamo example](../../docs/examples/inference/dynamo.md).

## About NVIDIA Dynamo

[NVIDIA Dynamo](https://docs.nvidia.com/dynamo/getting-started/introduction) is an open-source, high-throughput, low-latency inference framework for serving generative AI workloads in distributed environments. It adds a system-level layer above inference engines such as SGLang, vLLM, and TensorRT-LLM, coordinating them across GPUs and nodes.
Expand Down
4 changes: 2 additions & 2 deletions mkdocs/docs/guides/upgrade.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ Exceptions are noted in the sections below.

Support for [exports](../concepts/exports.md) and the global `Events` page has moved from the open-source version to [dstack Sky](https://dstack.ai/products/sky/) and [dstack Factory](https://dstack.ai/products/factory/). The `dstack event` CLI command and the `Events` tabs on resource pages remain available in the open-source version.

### Deprecations
### `DSTACK_ROUTER_INTERNAL_IP`

The `DSTACK_ROUTER_INTERNAL_IP` environment variable is deprecated in favor of [`${{ groups[i].replicas[j].IP_ADDRESS }}`](../concepts/services.md#replica-groups).
Starting with `0.22.8`, `dstack` no longer sets the `DSTACK_ROUTER_INTERNAL_IP` environment variable. Use [`${{ groups[i].replicas[j].IP_ADDRESS }}`](../concepts/services.md#replica-groups) instead, e.g. to point Dynamo workers at a dedicated `etcd` replica group as shown in the [NVIDIA Dynamo example](../examples/inference/dynamo.md).

## 0.21.* { #0_21 }

Expand Down
115 changes: 10 additions & 105 deletions src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from typing import Dict, Iterable, Optional, Sequence
from typing import Iterable, Optional, Sequence

from sqlalchemy import and_, exists, false, func, or_, select, true, update
from sqlalchemy.ext.asyncio import AsyncSession
Expand All @@ -26,7 +26,6 @@
from dstack._internal.core.models.metrics import Metric
from dstack._internal.core.models.profiles import StartupOrder
from dstack._internal.core.models.repos import RemoteRepoCreds
from dstack._internal.core.models.routers import RouterType
from dstack._internal.core.models.runs import (
ClusterInfo,
ImagePullProgress,
Expand Down Expand Up @@ -116,11 +115,6 @@
from dstack._internal.server.services.runner import client
from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
from dstack._internal.server.services.runs import is_job_ready, run_model_to_run
from dstack._internal.server.services.runs.replicas import (
RouterEnvStatus,
get_router_env_for_job,
get_router_replica_group,
)
from dstack._internal.server.services.runs.spec import run_spec_has_replica_ip_refs
from dstack._internal.server.services.secrets import get_project_secrets_mapping
from dstack._internal.server.services.storage import get_default_storage
Expand All @@ -143,8 +137,6 @@

JOB_STATUSES_WITH_MIN_PROCESSING_INTERVAL = [JobStatus.PROVISIONING, JobStatus.PULLING]

ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS = 30 * 60

JOB_DISCONNECTED_RETRY_TIMEOUT = timedelta(minutes=2)
"""`The minimum time before terminating active job in case of connectivity issues."""

Expand Down Expand Up @@ -421,12 +413,8 @@ async def _load_process_context(item: JobRunningPipelineItem) -> Optional[_Proce
)
else:
# PROVISIONING/PULLING jobs need same-replica siblings for cluster
# coordination, plus — when the run has a router replica group —
# the router replica's job (cross-replica) so the router checks in
# _terminate_if_router_unavailable and _prepare_runner_submission
# can read its status / IP.
# _fetch_run_model handles both: same-replica jobs always, plus
# all non-terminated jobs when one exists.
# coordination, plus all non-terminated jobs of the run when commands
# reference other replicas' IPs. _fetch_run_model handles both.
run_spec = validate_json_extra_ignore(RunSpec, job_model.run.run_spec)
run_model = await _fetch_run_model(
session=session,
Expand Down Expand Up @@ -477,12 +465,8 @@ async def _process_running_job(context: _ProcessContext) -> _ProcessResult:
return result

if context.job_model.status == JobStatus.PROVISIONING:
if _terminate_if_router_unavailable(context=context, result=result):
return result
await _process_provisioning_status(context=context, result=result)
elif context.job_model.status == JobStatus.PULLING:
if _terminate_if_router_unavailable(context=context, result=result):
return result
await _process_pulling_status(context=context, result=result)
elif context.job_model.status == JobStatus.RUNNING:
if _server_access_enabled(context):
Expand All @@ -509,53 +493,6 @@ async def _process_running_job(context: _ProcessContext) -> _ProcessResult:
return result


def _terminate_if_router_unavailable(context: _ProcessContext, result: _ProcessResult) -> bool:
"""
Terminates a worker job whose router replica is in a terminal state or did not acquire an
internal IP in time. Waiting for the router IP is up to `_prepare_runner_submission`.

Returns `True` if the job was terminated.
"""
# `get_router_env_for_job` returns None for the router itself and for runs without
# a router group, so this check is a no-op in those cases.
router_env = get_router_env_for_job(
run_model=context.run_model,
run_spec=context.run.run_spec,
job_model=context.job_model,
)
if router_env is RouterEnvStatus.FAILED:
# Router has reached a terminal state — the worker cannot recover by
# waiting. Terminate it now with a clear reason instead of letting it
# idle until the run-level reconciler tears the whole run down.
_terminate_job(
job_model=context.job_model,
job_update_map=result.job_update_map,
termination_reason=JobTerminationReason.TERMINATED_BY_SERVER,
termination_reason_message=(
"Router replica is in a terminal state; cannot provision worker "
"without a running router."
),
)
return True
if router_env is not RouterEnvStatus.NOT_PROVISIONED:
return False
# Bound the wait for the router IP so a router that is genuinely stuck can't burn
# worker instance-hours forever; see ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS.
waited_seconds = (get_current_datetime() - context.job_model.submitted_at).total_seconds()
if waited_seconds <= ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS:
return False
_terminate_job(
job_model=context.job_model,
job_update_map=result.job_update_map,
termination_reason=JobTerminationReason.TERMINATED_BY_SERVER,
termination_reason_message=(
f"Router replica did not acquire an internal IP within "
f"{ROUTER_PROVISIONING_WAIT_TIMEOUT_SECONDS}s; terminating worker."
),
)
return True


async def _refetch_locked_job_model(
session: AsyncSession, item: JobRunningPipelineItem
) -> Optional[JobModel]:
Expand Down Expand Up @@ -589,12 +526,10 @@ async def _fetch_run_model(
Args:
replica_num: If None, skip loading jobs (for RUNNING jobs that don't need siblings).
If set, load only latest-submission jobs for that replica (for PROVISIONING/PULLING
jobs that need same-replica siblings for cluster coordination). When the run has
a Dynamo router replica group, or any command contains
${{ groups[i].replicas[j].IP_ADDRESS }}, all non-terminated latest-submission
jobs for the run are loaded so sibling replica IPs are visible.
run_spec: Required whenever `replica_num` is set. Used to detect whether the run
has a Dynamo router replica group or replica IP refs.
jobs that need same-replica siblings for cluster coordination). When any command
contains ${{ groups[i].replicas[j].IP_ADDRESS }}, all non-terminated
latest-submission jobs for the run are loaded so sibling replica IPs are visible.
run_spec: Required whenever `replica_num` is set. Used to detect replica IP refs.
"""
query = (
select(RunModel)
Expand All @@ -616,13 +551,7 @@ async def _fetch_run_model(
)
if replica_num is not None:
assert run_spec is not None, "run_spec must be provided when replica_num is set"
router_group = get_router_replica_group(run_spec)
is_dynamo = (
router_group is not None
and router_group.router is not None
and router_group.router.type == RouterType.DYNAMO
)
load_all_replicas = is_dynamo or run_spec_has_replica_ip_refs(run_spec)
load_all_replicas = run_spec_has_replica_ip_refs(run_spec)

latest_submissions_sq = (
select(
Expand All @@ -633,8 +562,8 @@ async def _fetch_run_model(
)
.where(
JobModel.run_id == run_id,
# Dynamo and replica-IP interpolation need sibling replicas.
# Other services load only the worker's own replica.
# Replica-IP interpolation needs sibling replicas.
# Other runs load only the job's own replica.
true() if load_all_replicas else JobModel.replica_num == replica_num,
)
.group_by(JobModel.run_id, JobModel.replica_num, JobModel.job_num)
Expand Down Expand Up @@ -794,7 +723,6 @@ async def _process_provisioning_status(
code=code,
file_archives=file_archives,
repo_credentials=runner_submission.repo_creds,
router_env=runner_submission.router_env,
success_if_not_available=False,
)
_apply_submit_job_to_runner_result(
Expand Down Expand Up @@ -914,7 +842,6 @@ async def _process_pulling_status(
code=code,
file_archives=file_archives,
repo_credentials=runner_submission.repo_creds,
router_env=runner_submission.router_env,
success_if_not_available=True,
)
_apply_submit_job_to_runner_result(
Expand Down Expand Up @@ -1347,12 +1274,6 @@ def _interpolate_secrets_or_terminate(
class _RunnerSubmissionContext:
cluster_info: ClusterInfo
repo_creds: Optional[RemoteRepoCreds]
router_env: Optional[Dict[str, str]] = None
"""Dynamo-specific env (e.g. DSTACK_ROUTER_INTERNAL_IP) computed from the
router replica's state. Passed through to RunnerClient.submit_job, which
merges it into a deep-copied job_spec.env so the shared job_spec is not
mutated. None for SMG services, non-router runs, and the router
replica itself."""


async def _prepare_runner_submission(
Expand All @@ -1368,19 +1289,6 @@ async def _prepare_runner_submission(
if _should_wait_for_replica_provisioning(context):
return None

router_env = get_router_env_for_job(
run_model=context.run_model,
run_spec=context.run.run_spec,
job_model=context.job_model,
)
if isinstance(router_env, RouterEnvStatus):
# `FAILED` and a timed-out wait are handled by `_terminate_if_router_unavailable`
logger.debug(
"%s: waiting for router replica to be provisioned",
fmt(context.job_model),
)
return None

cluster_info = _get_cluster_info(
jobs=context.run.jobs,
replica_num=context.job.job_spec.replica_num,
Expand Down Expand Up @@ -1426,7 +1334,6 @@ async def _prepare_runner_submission(
return _RunnerSubmissionContext(
cluster_info=cluster_info,
repo_creds=repo_creds,
router_env=router_env,
)


Expand Down Expand Up @@ -1697,7 +1604,6 @@ def _submit_job_to_runner(
code: Optional[bytes],
file_archives: Iterable[tuple[uuid.UUID, bytes]],
repo_credentials: Optional[RemoteRepoCreds],
router_env: Optional[Dict[str, str]],
success_if_not_available: bool,
) -> _SubmitJobToRunnerResult:
logger.debug("%s: submitting job spec", fmt(job_model))
Expand Down Expand Up @@ -1726,7 +1632,6 @@ def _submit_job_to_runner(
secrets={},
repo_credentials=repo_credentials,
instance_env=instance_env,
router_env=router_env,
)
for archive_id, archive in file_archives:
logger.debug("%s: uploading file archive: %s", fmt(job_model), archive_id)
Expand Down
11 changes: 4 additions & 7 deletions src/dstack/_internal/server/services/runner/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,24 +272,21 @@ def submit_job(
secrets: Dict[str, str],
repo_credentials: Optional[RemoteRepoCreds],
instance_env: Optional[Union[Env, Dict[str, str]]] = None,
router_env: Optional[Dict[str, str]] = None,
):
# XXX: This is a quick-and-dirty hack to deliver InstanceModel-specific
# and Dynamo-router environment variables to the runner without runner
# API modification. Both layers are merged into a deep-copied job_spec
# so the shared spec object held by the caller is not mutated.
# environment variables to the runner without runner API modification.
# They are merged into a deep-copied job_spec so the shared spec object
# held by the caller is not mutated.
job_spec = job.job_spec
server_access = bool(getattr(run.run_spec.configuration, "dstack", False))
if instance_env is not None or router_env is not None or server_access:
if instance_env is not None or server_access:
merged_env: Dict[str, str] = {}
if instance_env is not None:
if isinstance(instance_env, Env):
merged_env.update(instance_env.as_dict())
else:
merged_env.update(instance_env)
merged_env.update(job_spec.env)
if router_env is not None:
merged_env.update(router_env)
if server_access:
merged_env.setdefault(DSTACK_PROJECT_ENV, run.project_name)
job_spec = job_spec.model_copy(deep=True)
Expand Down
Loading
Loading