Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
c5b63b6
Use the Nebius SDK sync interface instead of a dstack event loop
r4victor Nov 3, 2026
f2b77c9
Add Compute.close() and close Nebius SDKs
r4victor Nov 3, 2026
c532223
Lease cached backends and close retired backends
r4victor Nov 3, 2026
764fdac
Close retired backends in the background
r4victor Nov 3, 2026
f8ffe03
Annotate lease context managers with AsyncGenerator
r4victor Nov 3, 2026
2e92079
Simplify selecting a leased backend
r4victor Nov 3, 2026
c0bd7de
Move the backends cache to cache.py and backend model helpers to stor…
r4victor Nov 3, 2026
3ae28e2
Move backend persistence to store.py and backend offers to services/o…
r4victor Nov 3, 2026
1a63759
Rename _close_retired_backends to _retire_and_close_backends
r4victor Nov 3, 2026
b930e29
Explain the backends cache in a module docstring
r4victor Nov 3, 2026
e58cee0
Fix why backends are cached in the cache docstring
r4victor Nov 3, 2026
f3e5e0f
Return only offers from run plan functions
r4victor Nov 3, 2026
242edd9
Narrow backend leases in instance check and termination
r4victor Nov 3, 2026
858966a
Lease the gateway backend through backends_cache
r4victor Nov 3, 2026
2fad06d
Type the backend selector and document Compute.close()
r4victor Nov 3, 2026
9361cc5
Do not close backends on shutdown and document cancellation limits
r4victor Nov 3, 2026
4e0dcc3
Reduce backends cache lease API to three functions
r4victor Nov 4, 2026
09c32b2
Simplify termination lease and lease mocks in tests
r4victor Nov 4, 2026
b980253
Look up backends by type in cache tests
r4victor Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions contributing/BACKENDS.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ Refer to examples:
Go to `compute.py` and implement `Compute` methods.
Optionally, extend and implement `ComputeWith*` classes to support additional features such as fleets, volumes, gateways, placement groups, etc. For example, extend `ComputeWithCreateInstanceSupport` to support fleets.
To support the cleanup of orphaned resources, extend `ComputeWithResourceListingSupport` as described in [Orphan cleanup support](#32-orphan-cleanup-support).
If the compute holds resources that garbage collection does not free, such as SDK clients that run background threads, implement `Compute.close()` to release them. The server caches backends and calls `close()` once a backend is no longer used.

Refer to examples:
[verda](https://github.com/dstackai/dstack/blob/master/src/dstack/_internal/core/backends/verda/compute.py),
Expand Down
7 changes: 7 additions & 0 deletions src/dstack/_internal/core/backends/base/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,10 @@ def compute(self) -> Compute:
Returns Compute instance.
"""
pass

def close(self) -> None:
"""
Releases resources held by the backend. The backend must not be used after `close()`.
See `Compute.close()`.
"""
self.compute().close()
8 changes: 8 additions & 0 deletions src/dstack/_internal/core/backends/base/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,14 @@ def update_provisioning_data(
"""
pass

def close(self) -> None:
"""
Releases resources that are not freed by garbage collection, e.g. SDK clients that run
background threads. The compute must not be used after `close()`.
It is called from async code in executor, so it can block. Must be idempotent.
"""
pass


class ComputeWithAllOffersCached(ABC):
"""
Expand Down
10 changes: 10 additions & 0 deletions src/dstack/_internal/core/backends/configurators.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import List, Optional, Type, Union

from dstack._internal.core.backends.base.configurator import Configurator
from dstack._internal.core.errors import BackendNotAvailable
from dstack._internal.core.models.backends.base import BackendType

_CONFIGURATOR_CLASSES: List[Type[Configurator]] = []
Expand Down Expand Up @@ -223,3 +224,12 @@ def register_configurator(configurator: Type[Configurator]):
Can be used to extend dstack functionality.
"""
_BACKEND_TYPE_TO_CONFIGURATOR_CLASS_MAP[configurator.TYPE] = configurator


def check_backend_type_available(backend_type: BackendType):
if backend_type not in list_available_backend_types():
raise BackendNotAvailable(
f"Backend {backend_type.value} not available."
" Ensure that backend dependencies are installed."
f" Available backends: {[b.value for b in list_available_backend_types()]}."
)
15 changes: 10 additions & 5 deletions src/dstack/_internal/core/backends/nebius/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,13 @@ def _sdk(self) -> SDK:
assert isinstance(self.config.creds, NebiusServiceAccountCreds)
return resources.make_sdk(self.config.creds)

def close(self) -> None:
# `_sdk` is created on first use. Close only an existing SDK and only once
if "_sdk" in self.__dict__:
sdk = self._sdk
del self._sdk
resources.close_sdk(sdk)

@cached_property
def _region_to_project_id(self) -> dict[str, str]:
return resources.get_region_to_project_id_map(
Expand Down Expand Up @@ -496,11 +503,9 @@ def _wait_for_instance(sdk: SDK, op: resources.NebiusOperation) -> None:
op.status(),
)
time.sleep(WAIT_FOR_INSTANCE_UPDATE_INTERVAL)
resources.LOOP.await_(
op.update(
per_retry_timeout=resources.REQUEST_TIMEOUT,
auth_options=resources.REQUEST_AUTH_OPTIONS,
)
op.sync_update(
per_retry_timeout=resources.REQUEST_TIMEOUT,
auth_options=resources.REQUEST_AUTH_OPTIONS,
)


Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/core/backends/nebius/configurator.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class NebiusConfigurator(

def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_enabled: bool):
assert isinstance(config.creds, NebiusServiceAccountCreds)
sdk = None
try:
sdk = resources.make_sdk(config.creds)
# check that it's possible to build the projects map with configured settings
Expand All @@ -46,6 +47,9 @@ def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_en
fields=[["creds"]],
details=str(e),
)
finally:
if sdk is not None:
resources.close_sdk(sdk)
valid_fabrics = get_all_infiniband_fabrics()
if invalid_fabrics := set(config.fabrics or []) - valid_fabrics:
raise_invalid_credentials_error(
Expand Down
130 changes: 70 additions & 60 deletions src/dstack/_internal/core/backends/nebius/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
import re
import time
from collections import defaultdict
from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence
from collections.abc import Callable, Generator, Iterable, Sequence
from collections.abc import Container as ContainerT
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from typing import Any, Dict, Optional

from grpc import StatusCode
from nebius.aio.operation import Operation as SDKOperation
from nebius.aio.request import Request
from nebius.aio.service_error import RequestError
from nebius.aio.token.options import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS
from nebius.api.nebius.billing.v1 import (
Expand Down Expand Up @@ -78,16 +79,15 @@
from dstack._internal.core.models.backends.base import BackendType
from dstack._internal.core.models.common import validate_extra_ignore
from dstack._internal.utils.common import get_or_error
from dstack._internal.utils.event_loop import DaemonEventLoop
from dstack._internal.utils.logging import get_logger
from dstack.version import __version__

#
# Guidelines on using the Nebius SDK:
#
# Do not use Request.wait() or other sync SDK methods, they suffer from deadlocks.
# Instead, use async methods and await them with LOOP.await_()
LOOP = DaemonEventLoop()
# Use sync methods such as Request.wait() and Operation.sync_update(). The SDK runs requests
# on its own event loop thread, and its sync methods raise LoopError if called from a thread
# with a running event loop, so call them only from sync code, e.g. via run_async().
# Pass a timeout to all methods to avoid infinite waiting
REQUEST_TIMEOUT = 10
# Pass REQUEST_AUTH_OPTIONS to all methods to avoid infinite retries in case of invalid credentials
Expand All @@ -96,6 +96,9 @@
OPTION_RENEW_REQUEST_TIMEOUT: "5",
}
LIST_PAGE_SIZE = 100
# Close every SDK created with make_sdk() using close_sdk(). An SDK runs an event loop thread
# that keeps it alive, so an SDK that is not closed is never garbage collected.
SDK_CLOSE_TIMEOUT = 10
# The SDK's own `Operation` does not satisfy the `OperationMessage` bound of `SDKOperation`:
# the protocol declares a mutable `id: str`, while the `Operation` setter accepts `str | None`
NebiusOperation = SDKOperation[Operation] # pyright: ignore[reportInvalidTypeArguments]
Expand Down Expand Up @@ -136,6 +139,13 @@ def make_sdk(creds: NebiusServiceAccountCreds) -> SDK:
)


def close_sdk(sdk: SDK) -> None:
try:
sdk.sync_close(timeout=SDK_CLOSE_TIMEOUT)
except Exception:
logger.warning("Failed to close Nebius SDK", exc_info=True)


def wait_for_operation(
op: NebiusOperation,
timeout: float,
Expand All @@ -147,9 +157,7 @@ def wait_for_operation(
if time.monotonic() + interval > deadline:
raise TimeoutError(f"Operation {op.id} wait timeout")
time.sleep(interval)
LOOP.await_(
op.update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS)
)
op.sync_update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS)


def get_operation_error(op: NebiusOperation) -> str:
Expand Down Expand Up @@ -188,22 +196,26 @@ def validate_regions(configured: set[str], available: set[str]) -> None:


def list_tenant_projects(sdk: SDK) -> Sequence[Container]:
tenants = LOOP.await_(
TenantServiceClient(sdk).list(
tenants = (
TenantServiceClient(sdk)
.list(
ListTenantsRequest(),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)
if len(tenants.items) != 1:
raise ValueError(f"Expected to find 1 tenant, found {(len(tenants.items))}")
tenant_id = tenants.items[0].metadata.id
projects = LOOP.await_(
ProjectServiceClient(sdk).list(
projects = (
ProjectServiceClient(sdk)
.list(
ListProjectsRequest(parent_id=tenant_id, page_size=999),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)
return projects.items

Expand Down Expand Up @@ -272,12 +284,14 @@ def _get_region_to_configured_project_id_map(


def get_default_subnet(sdk: SDK, project_id: str) -> Subnet:
subnets = LOOP.await_(
SubnetServiceClient(sdk).list(
subnets = (
SubnetServiceClient(sdk)
.list(
ListSubnetsRequest(parent_id=project_id, page_size=999),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)
for subnet in subnets.items:
if subnet.metadata.name.startswith("default-subnet"):
Expand Down Expand Up @@ -311,21 +325,17 @@ def create_disk(
),
)
with wrap_capacity_errors():
return LOOP.await_(
client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
)
)
return client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
).wait()


def delete_disk(sdk: SDK, disk_id: str) -> None:
LOOP.await_(
DiskServiceClient(sdk).delete(
DeleteDiskRequest(id=disk_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
)
DiskServiceClient(sdk).delete(
DeleteDiskRequest(id=disk_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
).wait()


def list_disks(sdk: SDK, project_id: str) -> list[Disk]:
Expand All @@ -352,31 +362,29 @@ def create_pricing_policy(
pricing=PricingMethod(max_price_v1=MaxPriceV1(max_price=max_price)),
),
)
return LOOP.await_(
client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
)
)
return client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
).wait()


def get_pricing_policy(sdk: SDK, pricing_policy_id: str) -> PricingPolicy:
return LOOP.await_(
PricingPolicyServiceClient(sdk).get(
return (
PricingPolicyServiceClient(sdk)
.get(
GetPricingPolicyRequest(id=pricing_policy_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)


def delete_pricing_policy(sdk: SDK, pricing_policy_id: str) -> None:
LOOP.await_(
PricingPolicyServiceClient(sdk).delete(
DeletePricingPolicyRequest(id=pricing_policy_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
)
PricingPolicyServiceClient(sdk).delete(
DeletePricingPolicyRequest(id=pricing_policy_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
).wait()


def list_pricing_policies(sdk: SDK, project_id: str) -> list[PricingPolicy]:
Expand Down Expand Up @@ -446,30 +454,32 @@ def create_instance(
),
)
with wrap_capacity_errors():
return LOOP.await_(
client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
)
)
return client.create(
request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS
).wait()


def get_instance(sdk: SDK, instance_id: str) -> Instance:
return LOOP.await_(
InstanceServiceClient(sdk).get(
return (
InstanceServiceClient(sdk)
.get(
GetInstanceRequest(id=instance_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)


def delete_instance(sdk: SDK, instance_id: str) -> NebiusOperation:
return LOOP.await_(
InstanceServiceClient(sdk).delete(
return (
InstanceServiceClient(sdk)
.delete(
DeleteInstanceRequest(id=instance_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)


Expand All @@ -488,26 +498,26 @@ def list_instances(sdk: SDK, project_id: str) -> list[Instance]:

def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusOperation:
with wrap_capacity_errors():
return LOOP.await_(
GpuClusterServiceClient(sdk).create(
return (
GpuClusterServiceClient(sdk)
.create(
CreateGpuClusterRequest(
metadata=ResourceMetadata(name=name, parent_id=project_id),
spec=GpuClusterSpec(infiniband_fabric=fabric),
),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
.wait()
)


def delete_cluster(sdk: SDK, cluster_id: str) -> None:
LOOP.await_(
GpuClusterServiceClient(sdk).delete(
DeleteGpuClusterRequest(id=cluster_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
)
)
GpuClusterServiceClient(sdk).delete(
DeleteGpuClusterRequest(id=cluster_id),
per_retry_timeout=REQUEST_TIMEOUT,
auth_options=REQUEST_AUTH_OPTIONS,
).wait()


def filter_invalid_labels(labels: Dict[str, str]) -> Dict[str, str]:
Expand Down Expand Up @@ -549,14 +559,14 @@ def is_valid_label_value(value: str) -> bool:
return match is not None


def _list_all_pages(list_page: Callable[[str], Awaitable[Any]]) -> list:
def _list_all_pages(list_page: Callable[[str], Request[Any, Any]]) -> list:
"""
Calls `list_page` with the next page token until all pages are fetched.
"""
items = []
page_token = ""
while True:
response = LOOP.await_(list_page(page_token))
response = list_page(page_token).wait()
items.extend(response.items)
page_token = response.next_page_token
if not page_token:
Expand Down
Loading
Loading