From c5b63b6b6d966d27a51be86209dce6a9f4d7786a Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:04:19 +0500 Subject: [PATCH 01/19] Use the Nebius SDK sync interface instead of a dstack event loop Since nebius 0.5.0, the SDK runs requests on its own event loop thread and its sync methods are safe to call from sync code, so the separate DaemonEventLoop is no longer needed. --- .../_internal/core/backends/nebius/compute.py | 8 +- .../core/backends/nebius/resources.py | 120 +++++++++--------- src/dstack/_internal/utils/event_loop.py | 30 ----- src/tests/_internal/utils/test_event_loop.py | 18 --- 4 files changed, 63 insertions(+), 113 deletions(-) delete mode 100644 src/dstack/_internal/utils/event_loop.py delete mode 100644 src/tests/_internal/utils/test_event_loop.py diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index 89b5795a69..ff1f85329c 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -496,11 +496,9 @@ def _wait_for_instance(sdk: SDK, op: resources.NebiusOperation) -> None: op.status(), ) time.sleep(WAIT_FOR_INSTANCE_UPDATE_INTERVAL) - resources.LOOP.await_( - op.update( - per_retry_timeout=resources.REQUEST_TIMEOUT, - auth_options=resources.REQUEST_AUTH_OPTIONS, - ) + op.sync_update( + per_retry_timeout=resources.REQUEST_TIMEOUT, + auth_options=resources.REQUEST_AUTH_OPTIONS, ) diff --git a/src/dstack/_internal/core/backends/nebius/resources.py b/src/dstack/_internal/core/backends/nebius/resources.py index 2928cfc7ab..364d94ac17 100644 --- a/src/dstack/_internal/core/backends/nebius/resources.py +++ b/src/dstack/_internal/core/backends/nebius/resources.py @@ -2,7 +2,7 @@ import re import time from collections import defaultdict -from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence +from collections.abc import Callable, Generator, Iterable, Sequence from collections.abc import Container as ContainerT from contextlib import contextmanager from tempfile import NamedTemporaryFile @@ -10,6 +10,7 @@ from grpc import StatusCode from nebius.aio.operation import Operation as SDKOperation +from nebius.aio.request import Request from nebius.aio.service_error import RequestError from nebius.aio.token.options import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS from nebius.api.nebius.billing.v1 import ( @@ -78,16 +79,15 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.utils.common import get_or_error -from dstack._internal.utils.event_loop import DaemonEventLoop from dstack._internal.utils.logging import get_logger from dstack.version import __version__ # # Guidelines on using the Nebius SDK: # -# Do not use Request.wait() or other sync SDK methods, they suffer from deadlocks. -# Instead, use async methods and await them with LOOP.await_() -LOOP = DaemonEventLoop() +# Use sync methods such as Request.wait() and Operation.sync_update(). The SDK runs requests +# on its own event loop thread, and its sync methods raise LoopError if called from a thread +# with a running event loop, so call them only from sync code, e.g. via run_async(). # Pass a timeout to all methods to avoid infinite waiting REQUEST_TIMEOUT = 10 # Pass REQUEST_AUTH_OPTIONS to all methods to avoid infinite retries in case of invalid credentials @@ -147,9 +147,7 @@ def wait_for_operation( if time.monotonic() + interval > deadline: raise TimeoutError(f"Operation {op.id} wait timeout") time.sleep(interval) - LOOP.await_( - op.update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS) - ) + op.sync_update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS) def get_operation_error(op: NebiusOperation) -> str: @@ -188,22 +186,26 @@ def validate_regions(configured: set[str], available: set[str]) -> None: def list_tenant_projects(sdk: SDK) -> Sequence[Container]: - tenants = LOOP.await_( - TenantServiceClient(sdk).list( + tenants = ( + TenantServiceClient(sdk) + .list( ListTenantsRequest(), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) if len(tenants.items) != 1: raise ValueError(f"Expected to find 1 tenant, found {(len(tenants.items))}") tenant_id = tenants.items[0].metadata.id - projects = LOOP.await_( - ProjectServiceClient(sdk).list( + projects = ( + ProjectServiceClient(sdk) + .list( ListProjectsRequest(parent_id=tenant_id, page_size=999), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) return projects.items @@ -272,12 +274,14 @@ def _get_region_to_configured_project_id_map( def get_default_subnet(sdk: SDK, project_id: str) -> Subnet: - subnets = LOOP.await_( - SubnetServiceClient(sdk).list( + subnets = ( + SubnetServiceClient(sdk) + .list( ListSubnetsRequest(parent_id=project_id, page_size=999), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) for subnet in subnets.items: if subnet.metadata.name.startswith("default-subnet"): @@ -311,21 +315,17 @@ def create_disk( ), ) with wrap_capacity_errors(): - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def delete_disk(sdk: SDK, disk_id: str) -> None: - LOOP.await_( - DiskServiceClient(sdk).delete( - DeleteDiskRequest(id=disk_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + DiskServiceClient(sdk).delete( + DeleteDiskRequest(id=disk_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def list_disks(sdk: SDK, project_id: str) -> list[Disk]: @@ -352,31 +352,29 @@ def create_pricing_policy( pricing=PricingMethod(max_price_v1=MaxPriceV1(max_price=max_price)), ), ) - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def get_pricing_policy(sdk: SDK, pricing_policy_id: str) -> PricingPolicy: - return LOOP.await_( - PricingPolicyServiceClient(sdk).get( + return ( + PricingPolicyServiceClient(sdk) + .get( GetPricingPolicyRequest(id=pricing_policy_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_pricing_policy(sdk: SDK, pricing_policy_id: str) -> None: - LOOP.await_( - PricingPolicyServiceClient(sdk).delete( - DeletePricingPolicyRequest(id=pricing_policy_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + PricingPolicyServiceClient(sdk).delete( + DeletePricingPolicyRequest(id=pricing_policy_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def list_pricing_policies(sdk: SDK, project_id: str) -> list[PricingPolicy]: @@ -446,30 +444,32 @@ def create_instance( ), ) with wrap_capacity_errors(): - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def get_instance(sdk: SDK, instance_id: str) -> Instance: - return LOOP.await_( - InstanceServiceClient(sdk).get( + return ( + InstanceServiceClient(sdk) + .get( GetInstanceRequest(id=instance_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_instance(sdk: SDK, instance_id: str) -> NebiusOperation: - return LOOP.await_( - InstanceServiceClient(sdk).delete( + return ( + InstanceServiceClient(sdk) + .delete( DeleteInstanceRequest(id=instance_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) @@ -488,8 +488,9 @@ def list_instances(sdk: SDK, project_id: str) -> list[Instance]: def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusOperation: with wrap_capacity_errors(): - return LOOP.await_( - GpuClusterServiceClient(sdk).create( + return ( + GpuClusterServiceClient(sdk) + .create( CreateGpuClusterRequest( metadata=ResourceMetadata(name=name, parent_id=project_id), spec=GpuClusterSpec(infiniband_fabric=fabric), @@ -497,17 +498,16 @@ def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusO per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_cluster(sdk: SDK, cluster_id: str) -> None: - LOOP.await_( - GpuClusterServiceClient(sdk).delete( - DeleteGpuClusterRequest(id=cluster_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + GpuClusterServiceClient(sdk).delete( + DeleteGpuClusterRequest(id=cluster_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def filter_invalid_labels(labels: Dict[str, str]) -> Dict[str, str]: @@ -549,14 +549,14 @@ def is_valid_label_value(value: str) -> bool: return match is not None -def _list_all_pages(list_page: Callable[[str], Awaitable[Any]]) -> list: +def _list_all_pages(list_page: Callable[[str], Request[Any, Any]]) -> list: """ Calls `list_page` with the next page token until all pages are fetched. """ items = [] page_token = "" while True: - response = LOOP.await_(list_page(page_token)) + response = list_page(page_token).wait() items.extend(response.items) page_token = response.next_page_token if not page_token: diff --git a/src/dstack/_internal/utils/event_loop.py b/src/dstack/_internal/utils/event_loop.py deleted file mode 100644 index 29de015c01..0000000000 --- a/src/dstack/_internal/utils/event_loop.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import threading -from collections.abc import Awaitable -from typing import TypeVar - -T = TypeVar("T") - - -class DaemonEventLoop: - """ - A wrapper around asyncio.EventLoop that runs the loop in a daemon thread. - The thread is started with the first `await_` call. - """ - - def __init__(self) -> None: - self._loop = asyncio.new_event_loop() - self._start_lock = threading.Lock() - self._started = False - - def await_(self, awaitable: Awaitable[T]) -> T: - with self._start_lock: - if not self._started: - threading.Thread(target=self._loop.run_forever, daemon=True).start() - self._started = True - future = asyncio.run_coroutine_threadsafe(_coroutine(awaitable), self._loop) - return future.result() - - -async def _coroutine(awaitable: Awaitable[T]) -> T: - return await awaitable diff --git a/src/tests/_internal/utils/test_event_loop.py b/src/tests/_internal/utils/test_event_loop.py deleted file mode 100644 index 98812b52ac..0000000000 --- a/src/tests/_internal/utils/test_event_loop.py +++ /dev/null @@ -1,18 +0,0 @@ -import asyncio - -from dstack._internal.utils.event_loop import DaemonEventLoop - - -def test_daemon_event_loop(): - q = asyncio.Queue() - - async def worker(i): - await q.put(i) - - async def all_workers(): - await asyncio.gather(*[worker(i) for i in range(3)]) - - loop = DaemonEventLoop() - loop.await_(all_workers()) - assert q.qsize() == 3 - assert {loop.await_(q.get()) for _ in range(3)} == {0, 1, 2} From f2b77c99306d718620ad53ead6fd1605a41351ba Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:09:19 +0500 Subject: [PATCH 02/19] Add Compute.close() and close Nebius SDKs A Nebius SDK runs an event loop thread that keeps it alive, so an SDK that is not closed leaks a thread and file descriptors. Part of #4454. --- .../_internal/core/backends/base/backend.py | 7 +++ .../_internal/core/backends/base/compute.py | 8 ++++ .../_internal/core/backends/nebius/compute.py | 7 +++ .../core/backends/nebius/configurator.py | 4 ++ .../core/backends/nebius/resources.py | 10 ++++ .../core/backends/nebius/test_compute.py | 24 ++++++++++ .../core/backends/nebius/test_configurator.py | 48 +++++++++++++++++++ 7 files changed, 108 insertions(+) create mode 100644 src/tests/_internal/core/backends/nebius/test_configurator.py diff --git a/src/dstack/_internal/core/backends/base/backend.py b/src/dstack/_internal/core/backends/base/backend.py index ee19fd9897..c43fba5c7f 100644 --- a/src/dstack/_internal/core/backends/base/backend.py +++ b/src/dstack/_internal/core/backends/base/backend.py @@ -16,3 +16,10 @@ def compute(self) -> Compute: Returns Compute instance. """ pass + + def close(self) -> None: + """ + Releases resources held by the backend. The backend must not be used after `close()`. + See `Compute.close()`. + """ + self.compute().close() diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 7616f4bcb7..1b4fbc094a 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -204,6 +204,14 @@ def update_provisioning_data( """ pass + def close(self) -> None: + """ + Releases resources that are not freed by garbage collection, e.g. SDK clients that run + background threads. The compute must not be used after `close()`. + It is called from async code in executor, so it can block. Must be idempotent. + """ + pass + class ComputeWithAllOffersCached(ABC): """ diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index ff1f85329c..23b3ba24e4 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -132,6 +132,13 @@ def _sdk(self) -> SDK: assert isinstance(self.config.creds, NebiusServiceAccountCreds) return resources.make_sdk(self.config.creds) + def close(self) -> None: + # `_sdk` is created on first use. Close only an existing SDK and only once + if "_sdk" in self.__dict__: + sdk = self._sdk + del self._sdk + resources.close_sdk(sdk) + @cached_property def _region_to_project_id(self) -> dict[str, str]: return resources.get_region_to_project_id_map( diff --git a/src/dstack/_internal/core/backends/nebius/configurator.py b/src/dstack/_internal/core/backends/nebius/configurator.py index 483d4c4137..71c13aa993 100644 --- a/src/dstack/_internal/core/backends/nebius/configurator.py +++ b/src/dstack/_internal/core/backends/nebius/configurator.py @@ -35,6 +35,7 @@ class NebiusConfigurator( def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_enabled: bool): assert isinstance(config.creds, NebiusServiceAccountCreds) + sdk = None try: sdk = resources.make_sdk(config.creds) # check that it's possible to build the projects map with configured settings @@ -46,6 +47,9 @@ def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_en fields=[["creds"]], details=str(e), ) + finally: + if sdk is not None: + resources.close_sdk(sdk) valid_fabrics = get_all_infiniband_fabrics() if invalid_fabrics := set(config.fabrics or []) - valid_fabrics: raise_invalid_credentials_error( diff --git a/src/dstack/_internal/core/backends/nebius/resources.py b/src/dstack/_internal/core/backends/nebius/resources.py index 364d94ac17..5ede70067f 100644 --- a/src/dstack/_internal/core/backends/nebius/resources.py +++ b/src/dstack/_internal/core/backends/nebius/resources.py @@ -96,6 +96,9 @@ OPTION_RENEW_REQUEST_TIMEOUT: "5", } LIST_PAGE_SIZE = 100 +# Close every SDK created with make_sdk() using close_sdk(). An SDK runs an event loop thread +# that keeps it alive, so an SDK that is not closed is never garbage collected. +SDK_CLOSE_TIMEOUT = 10 # The SDK's own `Operation` does not satisfy the `OperationMessage` bound of `SDKOperation`: # the protocol declares a mutable `id: str`, while the `Operation` setter accepts `str | None` NebiusOperation = SDKOperation[Operation] # pyright: ignore[reportInvalidTypeArguments] @@ -136,6 +139,13 @@ def make_sdk(creds: NebiusServiceAccountCreds) -> SDK: ) +def close_sdk(sdk: SDK) -> None: + try: + sdk.sync_close(timeout=SDK_CLOSE_TIMEOUT) + except Exception: + logger.warning("Failed to close Nebius SDK", exc_info=True) + + def wait_for_operation( op: NebiusOperation, timeout: float, diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py index ab1441434a..c016d4b139 100644 --- a/src/tests/_internal/core/backends/nebius/test_compute.py +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -486,3 +486,27 @@ def test_waits_for_pricing_policy_being_created(self, resources): make_compute().delete_resource_group(group) resources.delete_pricing_policy.assert_not_called() + + +class TestClose: + def test_closes_created_sdk_once(self, mocker): + sdk = Mock() + mocker.patch.object(compute_module.resources, "make_sdk", return_value=sdk) + compute = make_compute() + assert compute._sdk is sdk + compute.close() + compute.close() + sdk.sync_close.assert_called_once_with(timeout=compute_module.resources.SDK_CLOSE_TIMEOUT) + + def test_does_not_create_sdk(self, mocker): + make_sdk = mocker.patch.object(compute_module.resources, "make_sdk") + make_compute().close() + make_sdk.assert_not_called() + + def test_does_not_raise_if_sdk_fails_to_close(self, mocker): + sdk = Mock() + sdk.sync_close.side_effect = TimeoutError() + mocker.patch.object(compute_module.resources, "make_sdk", return_value=sdk) + compute = make_compute() + assert compute._sdk is sdk + compute.close() diff --git a/src/tests/_internal/core/backends/nebius/test_configurator.py b/src/tests/_internal/core/backends/nebius/test_configurator.py new file mode 100644 index 0000000000..86949d73d6 --- /dev/null +++ b/src/tests/_internal/core/backends/nebius/test_configurator.py @@ -0,0 +1,48 @@ +from unittest.mock import Mock + +import pytest + +from dstack._internal.core.backends.nebius import configurator as configurator_module +from dstack._internal.core.backends.nebius.configurator import NebiusConfigurator +from dstack._internal.core.backends.nebius.models import ( + NebiusBackendConfigWithCreds, + NebiusServiceAccountCreds, +) +from dstack._internal.core.errors import BackendInvalidCredentialsError + + +def make_config() -> NebiusBackendConfigWithCreds: + return NebiusBackendConfigWithCreds( + creds=NebiusServiceAccountCreds( + service_account_id="service-account-id", + public_key_id="public-key-id", + private_key_content="private-key", + ) + ) + + +class TestValidateConfig: + @pytest.fixture(autouse=True) + def _mock_fabrics(self, mocker): + mocker.patch.object(configurator_module, "get_all_infiniband_fabrics", return_value=set()) + + def test_closes_sdk_if_valid(self, mocker): + sdk = Mock() + mocker.patch.object(configurator_module.resources, "make_sdk", return_value=sdk) + mocker.patch.object( + configurator_module.resources, "get_region_to_project_id_map", return_value={} + ) + NebiusConfigurator().validate_config(make_config(), default_creds_enabled=False) + sdk.sync_close.assert_called_once() + + def test_closes_sdk_if_invalid(self, mocker): + sdk = Mock() + mocker.patch.object(configurator_module.resources, "make_sdk", return_value=sdk) + mocker.patch.object( + configurator_module.resources, + "get_region_to_project_id_map", + side_effect=ValueError("Expected to find 1 tenant, found 0"), + ) + with pytest.raises(BackendInvalidCredentialsError): + NebiusConfigurator().validate_config(make_config(), default_creds_enabled=False) + sdk.sync_close.assert_called_once() From c53222383d5e2ca9721c938940d6f7740cf4050e Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:35:46 +0500 Subject: [PATCH 03/19] Lease cached backends and close retired backends Callers now borrow backends with lease_project_backend*() context managers. The cache retires a backend if its config changes, if it is deleted from the project, or if it is not leased for 5 minutes, and closes it once its last lease is released. --- src/dstack/_internal/server/app.py | 2 + .../pipeline_tasks/compute_groups.py | 80 +-- .../pipeline_tasks/gateway_replicas.py | 198 +++---- .../background/pipeline_tasks/gateways.py | 157 +++--- .../pipeline_tasks/instances/check.py | 104 ++-- .../instances/cloud_provisioning.py | 256 ++++----- .../pipeline_tasks/instances/termination.py | 92 ++-- .../pipeline_tasks/jobs_submitted.py | 492 +++++++++--------- .../pipeline_tasks/jobs_terminating.py | 46 +- .../pipeline_tasks/orphan_cleanup.py | 116 +++-- .../pipeline_tasks/placement_groups.py | 49 +- .../background/pipeline_tasks/volumes.py | 152 +++--- .../background/scheduled_tasks/__init__.py | 2 + .../server/services/backends/__init__.py | 429 ++++++++++----- .../_internal/server/services/fleets.py | 26 +- .../server/services/gateways/__init__.py | 8 +- .../_internal/server/services/offers.py | 10 +- .../_internal/server/services/runs/plan.py | 53 +- .../_internal/server/services/volumes.py | 31 +- src/dstack/_internal/server/testing/common.py | 24 +- .../pipeline_tasks/instances/test_check.py | 7 +- .../instances/test_cloud_provisioning.py | 73 +-- .../instances/test_termination.py | 7 +- .../pipeline_tasks/test_compute_groups.py | 13 +- .../pipeline_tasks/test_gateway_replicas.py | 73 +-- .../pipeline_tasks/test_gateways.py | 31 +- .../pipeline_tasks/test_jobs_submitted.py | 183 ++++--- .../pipeline_tasks/test_jobs_terminating.py | 23 +- .../pipeline_tasks/test_orphan_cleanup.py | 7 +- .../pipeline_tasks/test_placement_groups.py | 9 +- .../background/pipeline_tasks/test_volumes.py | 33 +- .../scheduled_tasks/test_idle_volumes.py | 13 +- .../_internal/server/routers/test_fleets.py | 25 +- .../_internal/server/routers/test_gpus.py | 79 ++- .../_internal/server/routers/test_runs.py | 119 +++-- .../server/services/backends/test_cache.py | 165 ++++++ .../server/services/runs/test_plan.py | 6 + .../_internal/server/services/test_fleets.py | 23 +- .../_internal/server/services/test_offers.py | 322 ++++++------ 39 files changed, 1995 insertions(+), 1543 deletions(-) create mode 100644 src/tests/_internal/server/services/backends/test_cache.py diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 00fd7ac6c8..ad7fbff443 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -48,6 +48,7 @@ users, volumes, ) +from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import prometheus as prometheus_service from dstack._internal.server.services.config import ServerConfigManager from dstack._internal.server.services.gateways import gateway_connections_pool @@ -224,6 +225,7 @@ async def lifespan(app: FastAPI): scheduler.shutdown() if pipeline_manager is not None: await pipeline_manager.drain() + await backends_services.close_all_backends() await gateway_connections_pool.remove_all() await job_server_connections_pool.remove_all() service_conn_pool = await get_injector_from_app(app).get_service_connection_pool() diff --git a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py index 5cccbd5204..11fbf33f00 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py @@ -289,52 +289,52 @@ async def _terminate_compute_group(compute_group_model: ComputeGroupModel) -> _T return result compute_group = compute_group_model_to_compute_group(compute_group_model) cgpd = compute_group.provisioning_data - backend = await backends_services.get_project_backend_by_type( + async with backends_services.lease_project_backend_by_type( project=compute_group_model.project, backend_type=cgpd.backend, - ) - if backend is None: - logger.error( - "Failed to terminate compute group %s. Backend %s not available." - " Please terminate it manually to avoid unexpected charges.", - compute_group.name, - cgpd.backend, - ) - return _get_terminated_result() - logger.debug("Terminating compute group %s", compute_group.name) - compute = backend.compute() - assert isinstance(compute, ComputeWithGroupProvisioningSupport) - try: - await run_async( - compute.terminate_compute_group, - compute_group, - ) - except Exception as e: - retry_at = get_current_datetime() - first_termination_retry_at = compute_group_model.first_termination_retry_at - if compute_group_model.first_termination_retry_at is None: - result.compute_group_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER - first_termination_retry_at = retry_at - assert first_termination_retry_at is not None - result.compute_group_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER - if _next_termination_retry_at(retry_at) < _get_termination_deadline( - first_termination_retry_at - ): - logger.warning( - "Failed to terminate compute group %s. Will retry. Error: %r", + ) as backend: + if backend is None: + logger.error( + "Failed to terminate compute group %s. Backend %s not available." + " Please terminate it manually to avoid unexpected charges.", + compute_group.name, + cgpd.backend, + ) + return _get_terminated_result() + logger.debug("Terminating compute group %s", compute_group.name) + compute = backend.compute() + assert isinstance(compute, ComputeWithGroupProvisioningSupport) + try: + await run_async( + compute.terminate_compute_group, + compute_group, + ) + except Exception as e: + retry_at = get_current_datetime() + first_termination_retry_at = compute_group_model.first_termination_retry_at + if compute_group_model.first_termination_retry_at is None: + result.compute_group_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER + first_termination_retry_at = retry_at + assert first_termination_retry_at is not None + result.compute_group_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER + if _next_termination_retry_at(retry_at) < _get_termination_deadline( + first_termination_retry_at + ): + logger.warning( + "Failed to terminate compute group %s. Will retry. Error: %r", + compute_group.name, + e, + exc_info=not isinstance(e, BackendError), + ) + return result + logger.error( + "Failed all attempts to terminate compute group %s." + " Please terminate it manually to avoid unexpected charges." + " Error: %r", compute_group.name, e, exc_info=not isinstance(e, BackendError), ) - return result - logger.error( - "Failed all attempts to terminate compute group %s." - " Please terminate it manually to avoid unexpected charges." - " Error: %r", - compute_group.name, - e, - exc_info=not isinstance(e, BackendError), - ) terminated_result = _get_terminated_result() terminated_result.compute_group_update_map.update(result.compute_group_update_map) terminated_result.instances_update_map.update(result.instances_update_map) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index 2530e10ec3..a989e59eb1 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -14,7 +14,7 @@ ComputeWithGatewayLoadBalancerSupport, ComputeWithGatewaySupport, ) -from dstack._internal.core.errors import BackendError, BackendNotAvailable, GatewayError +from dstack._internal.core.errors import BackendError, GatewayError from dstack._internal.core.models.common import validate_json_extra_ignore from dstack._internal.core.models.gateways import GatewayReplicaStatus, GatewayStatus from dstack._internal.core.models.runs import JobSpec, JobStatus, RunStatus, ServiceSpec @@ -454,45 +454,46 @@ async def _provision_gateway_replica( gateway_model: GatewayModel, replica_model: GatewayReplicaModel, ) -> _GatewayReplicaUpdateMap: - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, backend_id=replica_model.backend_id - ) - except BackendNotAvailable: - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message="Backend not available", - active=False, - deleted=True, - ) - - compute = backend.compute() - assert isinstance(compute, ComputeWithGatewaySupport) - replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) + async with backends_services.lease_project_backend_with_model_by_id( + project=gateway_model.project, backend_id=replica_model.backend_id + ) as backend_with_model: + if backend_with_model is None: + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message="Backend not available", + active=False, + deleted=True, + ) + _, backend = backend_with_model + compute = backend.compute() + assert isinstance(compute, ComputeWithGatewaySupport) + replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) - logger.debug("%s: creating gateway replica", fmt(replica_model)) - try: - gpd = await run_async( - compute.create_gateway_replica, replica_configuration, gateway_model.backend_data - ) - except BackendError as e: - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message=status_message, - active=False, - deleted=True, - ) - except Exception: - logger.exception("%s: unexpected error when creating gateway replica", fmt(replica_model)) - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message="Unexpected error", - active=False, - deleted=True, - ) + logger.debug("%s: creating gateway replica", fmt(replica_model)) + try: + gpd = await run_async( + compute.create_gateway_replica, replica_configuration, gateway_model.backend_data + ) + except BackendError as e: + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message=status_message, + active=False, + deleted=True, + ) + except Exception: + logger.exception( + "%s: unexpected error when creating gateway replica", fmt(replica_model) + ) + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message="Unexpected error", + active=False, + deleted=True, + ) return _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.PROVISIONING, @@ -580,26 +581,26 @@ async def _register_replica_with_load_balancer( """ if replica_model.instance_id is None: return "instance_id is None, cannot register with load balancer" - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, backend_id=replica_model.backend_id - ) - except BackendNotAvailable: - return "Backend not available" - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - return "Backend does not support load balancer operations" - lb_configuration = get_gateway_lb_configuration(gateway_model) - try: - await run_async( - compute.register_gateway_replica_with_load_balancer, - replica_model.instance_id, - lb_configuration, - gateway_model.backend_data, - ) - except Exception: - logger.exception("%s: error registering with load balancer", fmt(replica_model)) - return "Error registering with load balancer" + async with backends_services.lease_project_backend_with_model_by_id( + project=gateway_model.project, backend_id=replica_model.backend_id + ) as backend_with_model: + if backend_with_model is None: + return "Backend not available" + _, backend = backend_with_model + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + return "Backend does not support load balancer operations" + lb_configuration = get_gateway_lb_configuration(gateway_model) + try: + await run_async( + compute.register_gateway_replica_with_load_balancer, + replica_model.instance_id, + lb_configuration, + gateway_model.backend_data, + ) + except Exception: + logger.exception("%s: error registering with load balancer", fmt(replica_model)) + return "Error registering with load balancer" logger.info("%s: registered with load balancer", fmt(replica_model)) return None @@ -1467,46 +1468,49 @@ async def _process_terminating_item(item: GatewayReplicaPipelineItem): mark_terminated_update_map = _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.TERMINATED, active=False, deleted=True ) - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, - backend_id=replica_model.backend_id, - ) - except BackendNotAvailable: - logger.error( - "%s: backend not available, cannot terminate. Marking TERMINATED without termination", - fmt(replica_model), - ) - await _commit_update(item, replica_model, mark_terminated_update_map) - return - compute = backend.compute() - assert isinstance(compute, ComputeWithGatewaySupport) - replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) - if replica_model.instance_id is None: - logger.warning( - "%s: instance_id is None, skipping gateway replica termination", fmt(replica_model) - ) - await _commit_update(item, replica_model, mark_terminated_update_map) - return - if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model): - await _commit_update(item, replica_model, update_map={}) - return + async with backends_services.lease_project_backend_with_model_by_id( + project=gateway_model.project, + backend_id=replica_model.backend_id, + ) as backend_with_model: + if backend_with_model is None: + logger.error( + "%s: backend not available, cannot terminate." + " Marking TERMINATED without termination", + fmt(replica_model), + ) + await _commit_update(item, replica_model, mark_terminated_update_map) + return + _, backend = backend_with_model + compute = backend.compute() + assert isinstance(compute, ComputeWithGatewaySupport) + replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) + if replica_model.instance_id is None: + logger.warning( + "%s: instance_id is None, skipping gateway replica termination", fmt(replica_model) + ) + await _commit_update(item, replica_model, mark_terminated_update_map) + return + if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model): + await _commit_update(item, replica_model, update_map={}) + return - if gateway_model.hostname is not None: - await _deregister_gateway_replica_from_load_balancer(compute, gateway_model, replica_model) + if gateway_model.hostname is not None: + await _deregister_gateway_replica_from_load_balancer( + compute, gateway_model, replica_model + ) - logger.debug("%s: terminating gateway replica", fmt(replica_model)) - try: - await run_async( - compute.terminate_gateway_replica, - replica_model.instance_id, - replica_configuration, - replica_model.backend_data, - ) - except Exception: - logger.exception("%s: error when terminating gateway replica", fmt(replica_model)) - await _commit_update(item, replica_model, update_map={}) - return + logger.debug("%s: terminating gateway replica", fmt(replica_model)) + try: + await run_async( + compute.terminate_gateway_replica, + replica_model.instance_id, + replica_configuration, + replica_model.backend_data, + ) + except Exception: + logger.exception("%s: error when terminating gateway replica", fmt(replica_model)) + await _commit_update(item, replica_model, update_map={}) + return if replica_model.ip_address is not None: await gateway_connections_pool.remove(replica_model.ip_address) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py index 377f82dd7c..a4290c485d 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import joinedload, load_only, selectinload from dstack._internal.core.backends.base.compute import ComputeWithGatewayLoadBalancerSupport -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.gateways import ( GATEWAY_REPLICAS_DEFAULT, GatewayReplicaStatus, @@ -351,60 +351,58 @@ async def _process_submitted_gateway(gateway_model: GatewayModel) -> _SubmittedR configuration = gateways_services.get_gateway_configuration(gateway_model) update_map: _GatewayUpdateMap = {} if get_effective_load_balancer(configuration) is not None: - try: - ( - _, - backend, - ) = await backends_services.get_project_backend_with_model_by_type_or_error( - project=gateway_model.project, backend_type=configuration.backend - ) - except BackendNotAvailable: - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Backend not available", - } - ) - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - logger.error( - "%s: backend does not support load balancer operations", fmt(gateway_model) - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Backend does not support load balancer operations", - } - ) - lb_configuration = get_gateway_lb_configuration(gateway_model) - logger.info("%s: creating gateway load balancer...", fmt(gateway_model)) - try: - backend_data = await run_async(compute.create_gateway_load_balancer, lb_configuration) - except BackendError as e: - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - logger.warning( - "%s: failed to create gateway load balancer: %s", - fmt(gateway_model), - status_message, - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": status_message, - } - ) - except Exception: - logger.exception( - "%s: unexpected error when creating gateway load balancer", fmt(gateway_model) - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Unexpected error when creating load balancer", - } - ) + async with backends_services.lease_project_backend_by_type( + project=gateway_model.project, backend_type=configuration.backend + ) as backend: + if backend is None: + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Backend not available", + } + ) + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + logger.error( + "%s: backend does not support load balancer operations", fmt(gateway_model) + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Backend does not support load balancer operations", + } + ) + lb_configuration = get_gateway_lb_configuration(gateway_model) + logger.info("%s: creating gateway load balancer...", fmt(gateway_model)) + try: + backend_data = await run_async( + compute.create_gateway_load_balancer, lb_configuration + ) + except BackendError as e: + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + logger.warning( + "%s: failed to create gateway load balancer: %s", + fmt(gateway_model), + status_message, + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": status_message, + } + ) + except Exception: + logger.exception( + "%s: unexpected error when creating gateway load balancer", fmt(gateway_model) + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Unexpected error when creating load balancer", + } + ) logger.info("%s: gateway load balancer created.", fmt(gateway_model)) update_map["hostname"] = backend_data.hostname update_map["backend_data"] = backend_data.backend_data @@ -682,28 +680,33 @@ async def _process_to_be_deleted_gateway(gateway_model: GatewayModel) -> _Proces async def _terminate_gateway_load_balancer(gateway_model: GatewayModel) -> bool: """Terminates the gateway's load balancer. Returns True on success, False on failure.""" configuration = gateways_services.get_gateway_configuration(gateway_model) - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_type_or_error( - project=gateway_model.project, backend_type=configuration.backend - ) - except BackendNotAvailable: - logger.error( - "%s: backend not available, cannot terminate load balancer", fmt(gateway_model) - ) - return False - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - logger.error("%s: backend does not support load balancer operations", fmt(gateway_model)) - return False - lb_configuration = get_gateway_lb_configuration(gateway_model) - logger.info("%s: terminating gateway load balancer...", fmt(gateway_model)) - try: - await run_async( - compute.terminate_gateway_load_balancer, lb_configuration, gateway_model.backend_data - ) - except Exception: - logger.exception("%s: error when terminating gateway load balancer", fmt(gateway_model)) - return False + async with backends_services.lease_project_backend_by_type( + project=gateway_model.project, backend_type=configuration.backend + ) as backend: + if backend is None: + logger.error( + "%s: backend not available, cannot terminate load balancer", fmt(gateway_model) + ) + return False + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + logger.error( + "%s: backend does not support load balancer operations", fmt(gateway_model) + ) + return False + lb_configuration = get_gateway_lb_configuration(gateway_model) + logger.info("%s: terminating gateway load balancer...", fmt(gateway_model)) + try: + await run_async( + compute.terminate_gateway_load_balancer, + lb_configuration, + gateway_model.backend_data, + ) + except Exception: + logger.exception( + "%s: error when terminating gateway load balancer", fmt(gateway_model) + ) + return False logger.info("%s: gateway load balancer terminated.", fmt(gateway_model)) return True diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index 0bd31d06c8..ef363ae9d3 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -1,6 +1,7 @@ import logging import uuid -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager from datetime import timedelta from typing import Optional @@ -334,59 +335,60 @@ async def _process_wait_for_instance_provisioning_data( ) return result - backend = await _get_backend_for_provisioning_wait( + async with _lease_backend_for_provisioning_wait( project_id=instance_model.project_id, backend_type=job_provisioning_data.backend, - ) - if backend is None: - logger.warning( - "Instance %s failed because instance's backend is not available", - instance_model.name, - ) - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATING, - termination_reason=InstanceTerminationReason.ERROR, - termination_reason_message="Backend not available", - ) - return result - - try: - await run_async( - backend.compute().update_provisioning_data, - job_provisioning_data, - instance_model.project.ssh_public_key, - instance_model.project.ssh_private_key, - ) - result.instance_update_map["job_provisioning_data"] = ( - job_provisioning_data.model_dump_json() - ) - except ProvisioningError as exc: - logger.warning( - "Error while waiting for instance %s to become running: %s", - instance_model.name, - repr(exc), - ) - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATING, - termination_reason=InstanceTerminationReason.ERROR, - termination_reason_message="Error while waiting for instance to become running", - ) - except Exception: - logger.exception( - "Got exception when updating instance %s provisioning data", - instance_model.name, - ) + ) as backend: + if backend is None: + logger.warning( + "Instance %s failed because instance's backend is not available", + instance_model.name, + ) + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.TERMINATING, + termination_reason=InstanceTerminationReason.ERROR, + termination_reason_message="Backend not available", + ) + return result + + try: + await run_async( + backend.compute().update_provisioning_data, + job_provisioning_data, + instance_model.project.ssh_public_key, + instance_model.project.ssh_private_key, + ) + result.instance_update_map["job_provisioning_data"] = ( + job_provisioning_data.model_dump_json() + ) + except ProvisioningError as exc: + logger.warning( + "Error while waiting for instance %s to become running: %s", + instance_model.name, + repr(exc), + ) + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.TERMINATING, + termination_reason=InstanceTerminationReason.ERROR, + termination_reason_message="Error while waiting for instance to become running", + ) + except Exception: + logger.exception( + "Got exception when updating instance %s provisioning data", + instance_model.name, + ) return result -async def _get_backend_for_provisioning_wait( +@asynccontextmanager +async def _lease_backend_for_provisioning_wait( project_id: uuid.UUID, backend_type: BackendType, -) -> Optional[Backend]: +) -> AsyncIterator[Optional[Backend]]: async with get_session_ctx() as session: res = await session.execute( select(ProjectModel) @@ -395,11 +397,13 @@ async def _get_backend_for_provisioning_wait( ) project_model = res.unique().scalar_one_or_none() if project_model is None: - return None - return await backends_services.get_project_backend_by_type( + yield None + return + async with backends_services.lease_project_backend_by_type( project=project_model, backend_type=backend_type, - ) + ) as backend: + yield backend @runner_ssh_tunnel diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py index bd869b8325..cf2a7abf59 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py @@ -37,6 +37,7 @@ ) from dstack._internal.server.db import get_session_ctx from dstack._internal.server.models import FleetModel, InstanceModel, PlacementGroupModel +from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services.fleets import get_fleet_offers, is_cloud_cluster from dstack._internal.server.services.instances import ( get_instance_configuration, @@ -103,148 +104,151 @@ async def create_cloud_instance(instance_model: InstanceModel) -> ProcessResult: ) master_job_provisioning_data = cluster_context.master_job_provisioning_data - offers = await get_fleet_offers( - project=instance_model.project, - profile=profile, - requirements=requirements, - fleet_model=instance_model.fleet, - placement_group=placement_group_model_to_placement_group_optional(placement_group_model), - blocks="auto" if instance_model.total_blocks is None else instance_model.total_blocks, - exclude_not_available=True, - master_job_provisioning_data=master_job_provisioning_data, - infer_master_job_provisioning_data_from_fleet_instances=False, - include_only_create_instance_supported_backends=True, - ) + async with backends_services.lease_project_backends(instance_model.project) as backends: + offers = await get_fleet_offers( + backends=backends, + profile=profile, + requirements=requirements, + fleet_model=instance_model.fleet, + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), + blocks="auto" if instance_model.total_blocks is None else instance_model.total_blocks, + exclude_not_available=True, + master_job_provisioning_data=master_job_provisioning_data, + infer_master_job_provisioning_data_from_fleet_instances=False, + include_only_create_instance_supported_backends=True, + ) - offers_iter = iter(offers) - offers_tried = 0 - # Limit number of offers tried to prevent long-running processing in case all offers fail. - while offers_tried < server_settings.MAX_OFFERS_TRIED: - backend_with_instance_offer = next(offers_iter, None) - if backend_with_instance_offer is None: - break - backend, instance_offer = backend_with_instance_offer - - if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT: - continue - compute = backend.compute() - assert isinstance(compute, ComputeWithCreateInstanceSupport) - if master_job_provisioning_data is not None: - # `get_fleet_offers()` already restricts backend and region from the master. - # Availability zone still has to be narrowed per offer. - instance_offer = get_instance_offer_with_restricted_az( - instance_offer=instance_offer, - master_job_provisioning_data=master_job_provisioning_data, - ) - if ( - cluster_context is not None - and cluster_context.is_current_instance_master - and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT - and isinstance(compute, ComputeWithPlacementGroupSupport) - and await can_use_placement_groups( - compute=compute, - instance_offer=instance_offer, - reservation=instance_configuration.reservation, - ) - ): - ( - placement_group_model, - created_placement_group_model, - ) = await _find_or_create_suitable_placement_group_model( - instance_model=instance_model, - placement_group_models=placement_group_models, - instance_offer=instance_offer, - compute=compute, - ) - if placement_group_model is None: + offers_iter = iter(offers) + offers_tried = 0 + # Limit number of offers tried to prevent long-running processing in case all offers fail. + while offers_tried < server_settings.MAX_OFFERS_TRIED: + backend_with_instance_offer = next(offers_iter, None) + if backend_with_instance_offer is None: + break + backend, instance_offer = backend_with_instance_offer + + if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT: continue - if created_placement_group_model: - placement_group_models.append(placement_group_model) - result.new_placement_group_models.append(placement_group_model) + compute = backend.compute() + assert isinstance(compute, ComputeWithCreateInstanceSupport) + if master_job_provisioning_data is not None: + # `get_fleet_offers()` already restricts backend and region from the master. + # Availability zone still has to be narrowed per offer. + instance_offer = get_instance_offer_with_restricted_az( + instance_offer=instance_offer, + master_job_provisioning_data=master_job_provisioning_data, + ) + if ( + cluster_context is not None + and cluster_context.is_current_instance_master + and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT + and isinstance(compute, ComputeWithPlacementGroupSupport) + and await can_use_placement_groups( + compute=compute, + instance_offer=instance_offer, + reservation=instance_configuration.reservation, + ) + ): + ( + placement_group_model, + created_placement_group_model, + ) = await _find_or_create_suitable_placement_group_model( + instance_model=instance_model, + placement_group_models=placement_group_models, + instance_offer=instance_offer, + compute=compute, + ) + if placement_group_model is None: + continue + if created_placement_group_model: + placement_group_models.append(placement_group_model) + result.new_placement_group_models.append(placement_group_model) - logger.debug( - "Trying %s in %s/%s for $%0.4f per hour", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, - instance_offer.price, - ) - offers_tried += 1 - try: - job_provisioning_data = await run_async( - compute.create_instance, - instance_offer, - instance_configuration, - placement_group_model_to_placement_group_optional(placement_group_model), - ) - except SkipOffer as exc: - offers_tried -= 1 - logger.info( - "%s launch in %s/%s skipped: %s", + logger.debug( + "Trying %s in %s/%s for $%0.4f per hour", instance_offer.instance.name, instance_offer.backend.value, instance_offer.region, - exc, - extra={"instance_name": instance_model.name}, + instance_offer.price, ) - continue - except BackendError as exc: - logger.warning( - "%s launch in %s/%s failed: %s", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, - repr(exc), - extra={"instance_name": instance_model.name}, + offers_tried += 1 + try: + job_provisioning_data = await run_async( + compute.create_instance, + instance_offer, + instance_configuration, + placement_group_model_to_placement_group_optional(placement_group_model), + ) + except SkipOffer as exc: + offers_tried -= 1 + logger.info( + "%s launch in %s/%s skipped: %s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + exc, + extra={"instance_name": instance_model.name}, + ) + continue + except BackendError as exc: + logger.warning( + "%s launch in %s/%s failed: %s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + repr(exc), + extra={"instance_name": instance_model.name}, + ) + continue + except Exception: + logger.exception( + "Got exception when launching %s in %s/%s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + ) + continue + + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.PROVISIONING, ) - continue - except Exception: - logger.exception( - "Got exception when launching %s in %s/%s", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, + result.instance_update_map["backend"] = backend.TYPE + result.instance_update_map["region"] = instance_offer.region + result.instance_update_map["price"] = instance_offer.price + result.instance_update_map["instance_configuration"] = ( + instance_configuration.model_dump_json() + ) + result.instance_update_map["job_provisioning_data"] = ( + job_provisioning_data.model_dump_json() ) - continue + result.instance_update_map["offer"] = instance_offer.model_dump_json() + result.instance_update_map["total_blocks"] = instance_offer.total_blocks + result.instance_update_map["started_at"] = NOW_PLACEHOLDER + + if ( + instance_model.fleet_id is not None + and cluster_context is not None + and cluster_context.is_current_instance_master + ): + # Clean up placement groups that did not end up being used. + result.schedule_pg_deletion_fleet_id = instance_model.fleet_id + if placement_group_model is not None: + result.schedule_pg_deletion_except_id = placement_group_model.id + return result set_status_update( update_map=result.instance_update_map, instance_model=instance_model, - new_status=InstanceStatus.PROVISIONING, - ) - result.instance_update_map["backend"] = backend.TYPE - result.instance_update_map["region"] = instance_offer.region - result.instance_update_map["price"] = instance_offer.price - result.instance_update_map["instance_configuration"] = ( - instance_configuration.model_dump_json() - ) - result.instance_update_map["job_provisioning_data"] = ( - job_provisioning_data.model_dump_json() + new_status=InstanceStatus.TERMINATED, + termination_reason=InstanceTerminationReason.NO_OFFERS, + termination_reason_message="All offers failed" if offers else "No offers found", ) - result.instance_update_map["offer"] = instance_offer.model_dump_json() - result.instance_update_map["total_blocks"] = instance_offer.total_blocks - result.instance_update_map["started_at"] = NOW_PLACEHOLDER - - if ( - instance_model.fleet_id is not None - and cluster_context is not None - and cluster_context.is_current_instance_master - ): - # Clean up placement groups that did not end up being used. - result.schedule_pg_deletion_fleet_id = instance_model.fleet_id - if placement_group_model is not None: - result.schedule_pg_deletion_except_id = placement_group_model.id return result - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATED, - termination_reason=InstanceTerminationReason.NO_OFFERS, - termination_reason_message="All offers failed" if offers else "No offers found", - ) - return result - async def _get_cluster_master_context( instance_model: InstanceModel, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py index 7b05d7294b..70bf8b9395 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py @@ -35,56 +35,58 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is not None and job_provisioning_data.backend != BackendType.REMOTE: - backend = await backends_services.get_project_backend_by_type( + async with backends_services.lease_project_backend_by_type( project=instance_model.project, backend_type=job_provisioning_data.backend, - ) - if backend is None: - logger.error( - "Failed to terminate instance %s. Backend %s not available.", - instance_model.name, - job_provisioning_data.backend, - ) - else: - logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) - try: - await run_async( - backend.compute().terminate_instance, - job_provisioning_data.instance_id, - job_provisioning_data.region, - job_provisioning_data.backend_data, - ) - except Exception as exc: - first_retry_at = instance_model.first_termination_retry_at - if first_retry_at is None: - first_retry_at = now - result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER - result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER - if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): - if isinstance(exc, NotYetTerminated): - logger.debug( - "Instance %s termination in progress: %s", - instance_model.name, - exc, - ) - else: - logger.warning( - "Failed to terminate instance %s. Will retry. Error: %r", - instance_model.name, - exc, - exc_info=not isinstance(exc, BackendError), - ) - return result - next_step = "Please terminate the instance manually to avoid unexpected charges." - if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): - next_step = "Orphan cleanup will retry deleting the instance." + ) as backend: + if backend is None: logger.error( - "Failed all attempts to terminate instance %s. %s Error: %r", + "Failed to terminate instance %s. Backend %s not available.", instance_model.name, - next_step, - exc, - exc_info=not isinstance(exc, BackendError), + job_provisioning_data.backend, ) + else: + logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) + try: + await run_async( + backend.compute().terminate_instance, + job_provisioning_data.instance_id, + job_provisioning_data.region, + job_provisioning_data.backend_data, + ) + except Exception as exc: + first_retry_at = instance_model.first_termination_retry_at + if first_retry_at is None: + first_retry_at = now + result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER + result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER + if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): + if isinstance(exc, NotYetTerminated): + logger.debug( + "Instance %s termination in progress: %s", + instance_model.name, + exc, + ) + else: + logger.warning( + "Failed to terminate instance %s. Will retry. Error: %r", + instance_model.name, + exc, + exc_info=not isinstance(exc, BackendError), + ) + return result + next_step = ( + "Please terminate the instance manually to avoid unexpected charges." + ) + if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): + next_step = "Orphan cleanup will retry deleting the instance." + logger.error( + "Failed all attempts to terminate instance %s. %s Error: %r", + instance_model.name, + next_step, + exc, + exc_info=not isinstance(exc, BackendError), + ) if job_provisioning_data is not None: instance_connection_pool.drop_by_jpd(job_provisioning_data) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index fb340d4ddc..3b845f7627 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -83,8 +83,8 @@ VolumeAttachmentModel, VolumeModel, ) +from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events -from dstack._internal.server.services.backends import get_project_backend_by_type_or_error from dstack._internal.server.services.docker import apply_server_docker_defaults from dstack._internal.server.services.fleets import ( can_create_new_cloud_instance_in_fleet, @@ -1886,90 +1886,89 @@ async def _process_volume_attachments( if len(prepared_job_volumes.volume_model_ids) == 0: return _VolumeAttachmentResult(attachments=[], locked_volume_ids=[]) - backend = await get_project_backend_by_type_or_error( - project=project, - backend_type=job_provisioning_data.backend, - ) - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) + async with backends_services.lease_project_backend_by_type_or_error( + project=project, backend_type=job_provisioning_data.backend + ) as backend: + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) - volume_models = await _lock_related_volume_models( - item=item, volume_model_ids=prepared_job_volumes.volume_model_ids - ) - if volume_models is None: - return _VolumeAttachmentResult( - attachments=[], - locked_volume_ids=[], - termination_message="Failed to attach volume: Cannot attach a volume locked for processing", + volume_models = await _lock_related_volume_models( + item=item, volume_model_ids=prepared_job_volumes.volume_model_ids ) + if volume_models is None: + return _VolumeAttachmentResult( + attachments=[], + locked_volume_ids=[], + termination_message="Failed to attach volume: Cannot attach a volume locked for processing", + ) - locked_volume_ids = sorted( - { - volume_model.id - for mount_point_volume_models in volume_models - for volume_model in mount_point_volume_models - } - ) - attachments: list[_VolumeAttachmentPayload] = [] - related_volume_lock_owner = _get_related_volume_lock_owner(item.id) - for mount_point_volume_models in volume_models: - for volume_model in mount_point_volume_models: - volume = volume_model_to_volume(volume_model) - try: - if volume_model.deleted: - raise ServerClientError("Cannot attach a deleted volume") - if volume_model.to_be_deleted: - raise ServerClientError("Cannot attach a volume marked for deletion") - if ( - volume_model.lock_expires_at is not None - and volume_model.lock_owner != related_volume_lock_owner - ): - raise ServerClientError("Cannot attach a volume locked for processing") - if ( - job_provisioning_data.get_base_backend() != volume.get_backend() - or job_provisioning_data.region.lower() != volume.get_region().lower() - ): - continue - if volume.provisioning_data is None or not volume.provisioning_data.attachable: - continue - attachment_data = await run_async( - compute.attach_volume, - volume=volume, - provisioning_data=job_provisioning_data, - ) - attachments.append( - _VolumeAttachmentPayload( - volume_id=volume_model.id, - attachment_data=attachment_data.model_dump_json(), - volume_name=volume.name, + locked_volume_ids = sorted( + { + volume_model.id + for mount_point_volume_models in volume_models + for volume_model in mount_point_volume_models + } + ) + attachments: list[_VolumeAttachmentPayload] = [] + related_volume_lock_owner = _get_related_volume_lock_owner(item.id) + for mount_point_volume_models in volume_models: + for volume_model in mount_point_volume_models: + volume = volume_model_to_volume(volume_model) + try: + if volume_model.deleted: + raise ServerClientError("Cannot attach a deleted volume") + if volume_model.to_be_deleted: + raise ServerClientError("Cannot attach a volume marked for deletion") + if ( + volume_model.lock_expires_at is not None + and volume_model.lock_owner != related_volume_lock_owner + ): + raise ServerClientError("Cannot attach a volume locked for processing") + if ( + job_provisioning_data.get_base_backend() != volume.get_backend() + or job_provisioning_data.region.lower() != volume.get_region().lower() + ): + continue + if volume.provisioning_data is None or not volume.provisioning_data.attachable: + continue + attachment_data = await run_async( + compute.attach_volume, + volume=volume, + provisioning_data=job_provisioning_data, ) - ) - break - except ServerClientError as e: - logger.info("%s: failed to attach volume: %s", fmt(job_model), repr(e)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message=f"Failed to attach volume: {e.msg}", - ) - except BackendError as e: - logger.warning("%s: failed to attach volume: %s", fmt(job_model), repr(e)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message=f"Failed to attach volume: {str(e)}", - ) - except Exception: - logger.exception("%s: got exception when attaching volume", fmt(job_model)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message="Failed to attach volume: unexpected error", - ) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - ) + attachments.append( + _VolumeAttachmentPayload( + volume_id=volume_model.id, + attachment_data=attachment_data.model_dump_json(), + volume_name=volume.name, + ) + ) + break + except ServerClientError as e: + logger.info("%s: failed to attach volume: %s", fmt(job_model), repr(e)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message=f"Failed to attach volume: {e.msg}", + ) + except BackendError as e: + logger.warning("%s: failed to attach volume: %s", fmt(job_model), repr(e)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message=f"Failed to attach volume: {str(e)}", + ) + except Exception: + logger.exception("%s: got exception when attaching volume", fmt(job_model)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message="Failed to attach volume: unexpected error", + ) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + ) async def _lock_related_volume_models( @@ -2430,98 +2429,134 @@ async def _provision_new_capacity( fleet_model=fleet_model, ) multinode = requirements.multinode or is_multinode_job(job) - offers = await get_offers_by_requirements( - project=project, - profile=profile, - requirements=requirements, - exclude_not_available=True, - multinode=multinode, - master_job_provisioning_data=master_job_provisioning_data, - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), - placement_group=placement_group_model_to_placement_group_optional(placement_group_model), - ) - extra_authorized_keys = get_extra_authorized_keys(run.run_spec) - offers_iter = iter(offers) - offers_tried = 0 - offers_taken = 0 - skip_reasons: list[str] = [] - offer_errors: list[_OfferAttemptError] = [] - while offers_tried < settings.MAX_OFFERS_TRIED: - backend_with_offer = next(offers_iter, None) - if backend_with_offer is None: - break - offers_taken += 1 - backend, offer = backend_with_offer - logger.debug( - "%s: trying %s in %s/%s for $%0.4f per hour", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - offer.price, + async with backends_services.lease_project_backends(project) as backends: + offers = await get_offers_by_requirements( + backends=backends, + profile=profile, + requirements=requirements, + exclude_not_available=True, + multinode=multinode, + master_job_provisioning_data=master_job_provisioning_data, + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), ) - offer_volumes = _get_offer_volumes(volumes, offer) - job_configurations = [ - JobConfiguration(job=job_to_run, volumes=offer_volumes) for job_to_run in jobs - ] - compute = backend.compute() - if master_job_provisioning_data is not None: - offer = get_instance_offer_with_restricted_az( - instance_offer=offer, - master_job_provisioning_data=master_job_provisioning_data, - ) - if ( - # The first real instance in an empty cluster fleet is responsible - # for creating/selecting the placement group. A placeholder alone - # must not suppress that path. - not filter_non_placeholder_instances(fleet_model.instances) - and is_cloud_cluster(fleet_model) - and offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT - and isinstance(compute, ComputeWithPlacementGroupSupport) - and await can_use_placement_groups( - compute=compute, - instance_offer=offer, - reservation=requirements.reservation, - ) - ): - placement_group_model = await find_or_create_suitable_placement_group( - fleet_model=fleet_model, - placement_groups=placement_group_models, - instance_offer=offer, - compute=compute, - ) - if placement_group_model is None: - skip_reasons.append("no compatible placement group") - continue - if placement_group_model.id not in known_placement_group_ids: - new_placement_group_models.append(placement_group_model) - placement_group_models.append(placement_group_model) - known_placement_group_ids.add(placement_group_model.id) - offers_tried += 1 - try: - # Use run_jobs only for a real batch (2+ jobs). A 1-job batch must - # stay on run_job: Slurm gets node_count=1 there, and RunPod's - # Instant Cluster API rejects pod_count=1 (only sizes like 2/4/8). - use_group_provisioning = ( - offer.backend in BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT and len(jobs) > 1 + extra_authorized_keys = get_extra_authorized_keys(run.run_spec) + offers_iter = iter(offers) + offers_tried = 0 + offers_taken = 0 + skip_reasons: list[str] = [] + offer_errors: list[_OfferAttemptError] = [] + while offers_tried < settings.MAX_OFFERS_TRIED: + backend_with_offer = next(offers_iter, None) + if backend_with_offer is None: + break + offers_taken += 1 + backend, offer = backend_with_offer + logger.debug( + "%s: trying %s in %s/%s for $%0.4f per hour", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + offer.price, ) - if use_group_provisioning: - assert isinstance(compute, ComputeWithGroupProvisioningSupport) - compute_group_provisioning_data = await run_async( - compute.run_jobs, + offer_volumes = _get_offer_volumes(volumes, offer) + job_configurations = [ + JobConfiguration(job=job_to_run, volumes=offer_volumes) for job_to_run in jobs + ] + compute = backend.compute() + if master_job_provisioning_data is not None: + offer = get_instance_offer_with_restricted_az( + instance_offer=offer, + master_job_provisioning_data=master_job_provisioning_data, + ) + if ( + # The first real instance in an empty cluster fleet is responsible + # for creating/selecting the placement group. A placeholder alone + # must not suppress that path. + not filter_non_placeholder_instances(fleet_model.instances) + and is_cloud_cluster(fleet_model) + and offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT + and isinstance(compute, ComputeWithPlacementGroupSupport) + and await can_use_placement_groups( + compute=compute, + instance_offer=offer, + reservation=requirements.reservation, + ) + ): + placement_group_model = await find_or_create_suitable_placement_group( + fleet_model=fleet_model, + placement_groups=placement_group_models, + instance_offer=offer, + compute=compute, + ) + if placement_group_model is None: + skip_reasons.append("no compatible placement group") + continue + if placement_group_model.id not in known_placement_group_ids: + new_placement_group_models.append(placement_group_model) + placement_group_models.append(placement_group_model) + known_placement_group_ids.add(placement_group_model.id) + offers_tried += 1 + try: + # Use run_jobs only for a real batch (2+ jobs). A 1-job batch must + # stay on run_job: Slurm gets node_count=1 there, and RunPod's + # Instant Cluster API rejects pod_count=1 (only sizes like 2/4/8). + use_group_provisioning = ( + offer.backend in BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT and len(jobs) > 1 + ) + if use_group_provisioning: + assert isinstance(compute, ComputeWithGroupProvisioningSupport) + compute_group_provisioning_data = await run_async( + compute.run_jobs, + run, + job_configurations, + offer, + project_ssh_public_key, + project_ssh_private_key, + placement_group_model_to_placement_group_optional(placement_group_model), + requirements, + extra_authorized_keys, + ) + return _ProvisionNewCapacityResult( + provisioning_data=compute_group_provisioning_data, + offer=offer, + effective_profile=profile, + placement_group_cleanup=_build_placement_group_cleanup( + fleet_model=fleet_model, + offers_tried=offers_tried, + selected_placement_group_id=( + None if placement_group_model is None else placement_group_model.id + ), + new_placement_group_models=new_placement_group_models, + ), + ) + placeholder_instance = job_model.instance + if ( + placeholder_instance is not None + and is_placeholder_instance(placeholder_instance) + and len(job.job_submissions) > 0 + ): + # `run_job()` sets the placeholder's `provider_ref` on the resources it creates + job.job_submissions[-1].provider_ref = placeholder_instance.provider_ref + job_provisioning_data = await run_async( + compute.run_job, run, - job_configurations, + job, offer, project_ssh_public_key, project_ssh_private_key, + offer_volumes, placement_group_model_to_placement_group_optional(placement_group_model), requirements, extra_authorized_keys, ) return _ProvisionNewCapacityResult( - provisioning_data=compute_group_provisioning_data, + provisioning_data=job_provisioning_data, offer=offer, effective_profile=profile, placement_group_cleanup=_build_placement_group_cleanup( @@ -2533,91 +2568,58 @@ async def _provision_new_capacity( new_placement_group_models=new_placement_group_models, ), ) - placeholder_instance = job_model.instance - if ( - placeholder_instance is not None - and is_placeholder_instance(placeholder_instance) - and len(job.job_submissions) > 0 - ): - # `run_job()` sets the placeholder's `provider_ref` on the resources it creates - job.job_submissions[-1].provider_ref = placeholder_instance.provider_ref - job_provisioning_data = await run_async( - compute.run_job, - run, - job, - offer, - project_ssh_public_key, - project_ssh_private_key, - offer_volumes, - placement_group_model_to_placement_group_optional(placement_group_model), - requirements, - extra_authorized_keys, - ) - return _ProvisionNewCapacityResult( - provisioning_data=job_provisioning_data, - offer=offer, - effective_profile=profile, - placement_group_cleanup=_build_placement_group_cleanup( - fleet_model=fleet_model, - offers_tried=offers_tried, - selected_placement_group_id=( - None if placement_group_model is None else placement_group_model.id - ), - new_placement_group_models=new_placement_group_models, + except SkipOffer as e: + offers_tried -= 1 + skip_reasons.append(str(e) or "offer skipped") + logger.info( + "%s: %s launch in %s/%s skipped: %s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + e, + ) + continue + except BackendError as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) + logger.warning( + "%s: %s launch in %s/%s failed: %s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + repr(e), + ) + continue + except Exception as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) + logger.exception( + "%s: got exception when launching %s in %s/%s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + ) + continue + return _FailedNewCapacityProvisioning( + message=_get_new_capacity_failure_message( + fleet_name=fleet_model.name, + attempts=_NewCapacityAttempts( + total=len(offers), + tried=offers_tried, + skip_reasons=skip_reasons, + errors=offer_errors, + # Offers are left only if the attempt limit, not the offer list, ended the loop. + limit_reached=offers_taken < len(offers), ), - ) - except SkipOffer as e: - offers_tried -= 1 - skip_reasons.append(str(e) or "offer skipped") - logger.info( - "%s: %s launch in %s/%s skipped: %s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - e, - ) - continue - except BackendError as e: - offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) - logger.warning( - "%s: %s launch in %s/%s failed: %s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - repr(e), - ) - continue - except Exception as e: - offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) - logger.exception( - "%s: got exception when launching %s in %s/%s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - ) - continue - return _FailedNewCapacityProvisioning( - message=_get_new_capacity_failure_message( - fleet_name=fleet_model.name, - attempts=_NewCapacityAttempts( - total=len(offers), - tried=offers_tried, - skip_reasons=skip_reasons, - errors=offer_errors, - # Offers are left only if the attempt limit, not the offer list, ended the loop. - limit_reached=offers_taken < len(offers), ), - ), - placement_group_cleanup=_build_placement_group_cleanup( - fleet_model=fleet_model, - offers_tried=offers_tried, - selected_placement_group_id=None, - new_placement_group_models=new_placement_group_models, - ), - ) + placement_group_cleanup=_build_placement_group_cleanup( + fleet_model=fleet_model, + offers_tried=offers_tried, + selected_placement_group_id=None, + new_placement_group_models=new_placement_group_models, + ), + ) _MAX_OFFER_ERROR_LEN = 200 diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index 269e2e10c5..ab9bbc5280 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -964,32 +964,32 @@ async def _detach_volumes_from_job_instance( run_termination_reason: Optional[RunTerminationReason], ) -> _VolumeDetachResult: job_spec = get_job_spec(job_model) - backend = await backends_services.get_project_backend_by_type( + async with backends_services.lease_project_backend_by_type( project=instance_model.project, backend_type=jpd.backend, - ) - if backend is None: - logger.error( - "Failed to detach volumes from %s. Backend not available.", instance_model.name - ) - return _VolumeDetachResult(all_detached=False) + ) as backend: + if backend is None: + logger.error( + "Failed to detach volumes from %s. Backend not available.", instance_model.name + ) + return _VolumeDetachResult(all_detached=False) - detached_volume_ids = set() - all_detached = True - for volume_model in volume_models: - detached = await _detach_volume_from_job_instance( - backend=backend, - job_model=job_model, - jpd=jpd, - job_spec=job_spec, - instance_model=instance_model, - volume_model=volume_model, - run_termination_reason=run_termination_reason, - ) - if detached: - detached_volume_ids.add(volume_model.id) - else: - all_detached = False + detached_volume_ids = set() + all_detached = True + for volume_model in volume_models: + detached = await _detach_volume_from_job_instance( + backend=backend, + job_model=job_model, + jpd=jpd, + job_spec=job_spec, + instance_model=instance_model, + volume_model=volume_model, + run_termination_reason=run_termination_reason, + ) + if detached: + detached_volume_ids.add(volume_model.id) + else: + all_detached = False return _VolumeDetachResult( all_detached=all_detached, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py index 8406b1a664..95d97fa76d 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py @@ -382,67 +382,69 @@ async def _process_orphan_cleanup(process_context: _ProcessContext) -> _ProcessR orphan_cleanup_model = process_context.orphan_cleanup_model if not process_context.needs_cleanup: return _ProcessResult(update_map=_OrphanCleanupUpdateMap()) - backend = await backends_services.get_project_backend_by_type( + async with backends_services.lease_project_backend_by_type( project=orphan_cleanup_model.backend.project, backend_type=orphan_cleanup_model.backend.type, - ) - if backend is None: - logger.warning( - "Failed to clean up orphaned resources in %s/%s: backend is not available", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - compute = backend.compute() - assert isinstance(compute, ComputeWithResourceListingSupport) - listed_at = get_current_datetime() - try: - resource_listing = await run_async(compute.list_resource_groups) - except BackendError as e: - logger.warning( - "Failed to list resource groups in %s/%s due to backend error: %s", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - repr(e), - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - except Exception: - logger.exception( - "Failed to list resource groups in %s/%s due to unexpected error", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - for failure in resource_listing.failures: - logger.warning( - "Failed to list resources of %s in %s for orphan cleanup in %s/%s: %s", - failure.backend.value, - failure.region if failure.region is not None else "all regions", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - failure.message, - ) - resource_groups = resource_listing.resource_groups - owner_records = await _load_owner_records( - project_id=orphan_cleanup_model.backend.project_id, - resource_groups=resource_groups, - ) - decisions = [ - _get_resource_group_decision( - orphan_cleanup_model, resource_group, owner_records, listed_at + ) as backend: + if backend is None: + logger.warning( + "Failed to clean up orphaned resources in %s/%s: backend is not available", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + compute = backend.compute() + assert isinstance(compute, ComputeWithResourceListingSupport) + listed_at = get_current_datetime() + try: + resource_listing = await run_async(compute.list_resource_groups) + except BackendError as e: + logger.warning( + "Failed to list resource groups in %s/%s due to backend error: %s", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + repr(e), + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + except Exception: + logger.exception( + "Failed to list resource groups in %s/%s due to unexpected error", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + for failure in resource_listing.failures: + logger.warning( + "Failed to list resources of %s in %s for orphan cleanup in %s/%s: %s", + failure.backend.value, + failure.region if failure.region is not None else "all regions", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + failure.message, + ) + resource_groups = resource_listing.resource_groups + owner_records = await _load_owner_records( + project_id=orphan_cleanup_model.backend.project_id, + resource_groups=resource_groups, ) - for resource_group in resource_groups - ] - final_decisions: list[_ResourceGroupDecision] = [] - orphan_events: list[_OrphanEvent] = [] - for resource_group, decision in zip(resource_groups, decisions): - if decision == _ResourceGroupDecision.ORPHAN: - decision = await _handle_orphaned_resource_group( - compute, orphan_cleanup_model, resource_group + decisions = [ + _get_resource_group_decision( + orphan_cleanup_model, resource_group, owner_records, listed_at ) - if decision in (_ResourceGroupDecision.ORPHAN, _ResourceGroupDecision.DELETING): - orphan_events.append(_get_orphan_event(resource_group, owner_records, decision)) - final_decisions.append(decision) + for resource_group in resource_groups + ] + final_decisions: list[_ResourceGroupDecision] = [] + orphan_events: list[_OrphanEvent] = [] + for resource_group, decision in zip(resource_groups, decisions): + if decision == _ResourceGroupDecision.ORPHAN: + decision = await _handle_orphaned_resource_group( + compute, orphan_cleanup_model, resource_group + ) + if decision in (_ResourceGroupDecision.ORPHAN, _ResourceGroupDecision.DELETING): + orphan_events.append( + _get_orphan_event(resource_group, owner_records, decision) + ) + final_decisions.append(decision) if any(d is _ResourceGroupDecision.DELETE_FAILED for d in final_decisions): result = _get_backoff_cleanup_process_result(orphan_cleanup_model) elif ( diff --git a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py index ff8439dadc..fa88fb1955 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py @@ -245,32 +245,33 @@ async def _delete_placement_group( "Failed to delete placement group %s. provisioning_data is None.", placement_group.name ) return _get_deleted_result() - backend = await backends_services.get_project_backend_by_type( + async with backends_services.lease_project_backend_by_type( project=placement_group_model.project, backend_type=placement_group.provisioning_data.backend, - ) - if backend is None: - # TODO: Retry deletion - logger.error( - "Failed to delete placement group %s. Backend not available. Please delete it manually.", - placement_group.name, - ) - return _get_deleted_result() - compute = backend.compute() - assert isinstance(compute, ComputeWithPlacementGroupSupport) - try: - await run_async(compute.delete_placement_group, placement_group) - except PlacementGroupInUseError: - logger.info( - "Placement group %s is still in use. Skipping deletion for now.", placement_group.name - ) - return _DeleteResult() - except Exception: - # TODO: Retry deletion - logger.exception( - "Got exception when deleting placement group %s. Please delete it manually.", - placement_group.name, - ) + ) as backend: + if backend is None: + # TODO: Retry deletion + logger.error( + "Failed to delete placement group %s. Backend not available. Please delete it manually.", + placement_group.name, + ) + return _get_deleted_result() + compute = backend.compute() + assert isinstance(compute, ComputeWithPlacementGroupSupport) + try: + await run_async(compute.delete_placement_group, placement_group) + except PlacementGroupInUseError: + logger.info( + "Placement group %s is still in use. Skipping deletion for now.", + placement_group.name, + ) + return _DeleteResult() + except Exception: + # TODO: Retry deletion + logger.exception( + "Got exception when deleting placement group %s. Please delete it manually.", + placement_group.name, + ) return _get_deleted_result() diff --git a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py index a5d08ac803..d9d82e2f70 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import joinedload, load_only from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.volumes import VolumeStatus from dstack._internal.server.background.pipeline_tasks.base import ( NOW_PLACEHOLDER, @@ -306,59 +306,57 @@ class _ProcessResult: async def _process_submitted_volume(volume_model: VolumeModel) -> _ProcessResult: volume = volume_model_to_volume(volume_model) - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.configuration.backend, - overrides=True, - ) - except BackendNotAvailable: - logger.error( - "Failed to process volume %s. Backend %s not available.", - volume.name, - volume.configuration.backend.value, - ) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": "Backend not available", - } - ) - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - try: - if volume.configuration.is_external: - logger.info("Registering external volume %s", volume_model.name) - vpd = await run_async( - compute.register_volume, - volume=volume, + async with backends_services.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.configuration.backend, + overrides=True, + ) as backend: + if backend is None: + logger.error( + "Failed to process volume %s. Backend %s not available.", + volume.name, + volume.configuration.backend.value, ) - else: - logger.info("Provisioning new volume %s", volume_model.name) - vpd = await run_async( - compute.create_volume, - volume=volume, + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": "Backend not available", + } + ) + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + try: + if volume.configuration.is_external: + logger.info("Registering external volume %s", volume_model.name) + vpd = await run_async( + compute.register_volume, + volume=volume, + ) + else: + logger.info("Provisioning new volume %s", volume_model.name) + vpd = await run_async( + compute.create_volume, + volume=volume, + ) + except BackendError as e: + logger.info("Failed to create volume %s: %s", volume_model.name, repr(e)) + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": status_message, + } + ) + except Exception as e: + logger.exception("Got exception when creating volume %s", volume_model.name) + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": f"Unexpected error: {repr(e)}", + } ) - except BackendError as e: - logger.info("Failed to create volume %s: %s", volume_model.name, repr(e)) - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": status_message, - } - ) - except Exception as e: - logger.exception("Got exception when creating volume %s", volume_model.name) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": f"Unexpected error: {repr(e)}", - } - ) logger.info("Added new volume %s", volume_model.name) # Provisioned volumes marked as active since they become available almost immediately in AWS @@ -383,32 +381,30 @@ async def _process_to_be_deleted_volume(volume_model: VolumeModel) -> _ProcessRe f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None." ) return _get_deleted_result() - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.provisioning_data.backend, - ) - except BackendNotAvailable: - # TODO: Retry deletion - logger.error( - f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." - " Please terminate it manually to avoid unexpected charges.", - ) - return _get_deleted_result() - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - try: - await run_async( - compute.delete_volume, - volume=volume, - ) - except Exception: - # TODO: Retry deletion - logger.exception( - "Got exception when deleting volume %s. Please terminate it manually to avoid unexpected charges.", - volume.name, - ) + async with backends_services.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.provisioning_data.backend, + ) as backend: + if backend is None: + # TODO: Retry deletion + logger.error( + f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + " Please terminate it manually to avoid unexpected charges.", + ) + return _get_deleted_result() + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + try: + await run_async( + compute.delete_volume, + volume=volume, + ) + except Exception: + # TODO: Retry deletion + logger.exception( + "Got exception when deleting volume %s. Please terminate it manually to avoid unexpected charges.", + volume.name, + ) return _get_deleted_result() diff --git a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py index 33aa278303..b1598b1a01 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py @@ -30,6 +30,7 @@ collect_prometheus_metrics, delete_prometheus_metrics, ) +from dstack._internal.server.services.backends import close_idle_backends _scheduler = AsyncIOScheduler() @@ -53,6 +54,7 @@ def start_scheduled_tasks() -> AsyncIOScheduler: _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1) _scheduler.add_job(delete_events, IntervalTrigger(minutes=7), max_instances=1) _scheduler.add_job(process_gateways_connections, IntervalTrigger(seconds=15)) + _scheduler.add_job(close_idle_backends, IntervalTrigger(minutes=1), max_instances=1) _scheduler.add_job( process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1 ) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 8472375b7a..35e94ae8cf 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,11 +1,12 @@ import asyncio import json import time -from collections.abc import Iterable, Iterator +from collections.abc import AsyncIterator, Iterable, Iterator +from contextlib import asynccontextmanager +from dataclasses import dataclass, field from typing import Callable, Coroutine, Dict, List, Optional, Tuple from uuid import UUID -from cachetools import TTLCache from pydantic import ValidationError from sqlalchemy import delete, update from sqlalchemy.ext.asyncio import AsyncSession @@ -65,8 +66,8 @@ async def create_backend( configurator = get_configurator(config.type) if configurator is None: raise BackendNotAvailable() - backend = await get_project_backend_by_type(project=project, backend_type=configurator.TYPE) - if backend is not None: + backend_exists = any(configurator.TYPE == b.type for b in project.backends) + if backend_exists: raise ResourceExistsError() backend = await validate_and_create_backend_model( project=project, configurator=configurator, config=config @@ -260,26 +261,249 @@ async def delete_backends( BackendTuple = Tuple[BackendModel, Backend] +@dataclass +class _CachedBackend: + model: BackendModel + backend: Backend + leases: int = 0 + last_used: float = field(default_factory=time.monotonic) + retired: bool = False + + +_BACKEND_IDLE_TTL = 300 _BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} -_BACKENDS_CACHE = TTLCache[UUID, Dict[BackendType, BackendTuple]](maxsize=1000, ttl=300) +_BACKENDS_CACHE: Dict[UUID, Dict[BackendType, _CachedBackend]] = {} +""" +A backend is cached per project and used through leases (`lease_project_backend*()`). +The cache retires a backend if its config changes, if it is deleted from the project, +or if it is not leased for `_BACKEND_IDLE_TTL`. A retired backend is closed +once its last lease is released, so callers must not use backends outside of leases. +""" def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: return _BACKENDS_CACHE_LOCKS.setdefault(project_id, asyncio.Lock()) -async def get_project_backends_with_models(project: ProjectModel) -> List[BackendTuple]: +@asynccontextmanager +async def lease_project_backends_with_models( + project: ProjectModel, +) -> AsyncIterator[List[BackendTuple]]: + cached_backends = await _acquire_project_backends(project) + try: + yield [(c.model, c.backend) for c in cached_backends] + finally: + await _release_backends(cached_backends) + + +@asynccontextmanager +async def lease_project_backends(project: ProjectModel) -> AsyncIterator[List[Backend]]: + async with lease_project_backends_with_models(project) as backends_with_models: + yield [b for _, b in backends_with_models] + + +_select_project_backend_with_model_by_type = None + + +def set_select_project_backend_with_model_by_type( + func: Callable[ + [ProjectModel, BackendType, List[BackendTuple]], + Coroutine[None, None, Optional[BackendTuple]], + ], +): + """ + Overrides how `lease_project_backend*_by_type()` select a backend from the project backends + if `overrides=True` is passed. This can be used if a backend needs to be replaced with + another backend. For example, DstackBackend in dstack Sky can be used in place of + other backends. + """ + global _select_project_backend_with_model_by_type + _select_project_backend_with_model_by_type = func + + +@asynccontextmanager +async def lease_project_backend_with_model_by_type( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncIterator[Optional[BackendTuple]]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + if overrides and _select_project_backend_with_model_by_type is not None: + return await _select_project_backend_with_model_by_type( + project, backend_type, backends_with_models + ) + for backend_model, backend in backends_with_models: + if backend.TYPE == backend_type: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend_with_model: + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_with_model_by_type_or_error( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncIterator[BackendTuple]: + async with lease_project_backend_with_model_by_type( + project=project, backend_type=backend_type, overrides=overrides + ) as backend_with_model: + if backend_with_model is None: + raise BackendNotAvailable() + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_by_type( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncIterator[Optional[Backend]]: + async with lease_project_backend_with_model_by_type( + project=project, backend_type=backend_type, overrides=overrides + ) as backend_with_model: + yield backend_with_model[1] if backend_with_model is not None else None + + +@asynccontextmanager +async def lease_project_backend_by_type_or_error( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncIterator[Backend]: + async with lease_project_backend_with_model_by_type_or_error( + project=project, backend_type=backend_type, overrides=overrides + ) as (_, backend): + yield backend + + +@asynccontextmanager +async def lease_project_backend_with_model_by_id( + project: ProjectModel, backend_id: UUID +) -> AsyncIterator[Optional[BackendTuple]]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + for backend_model, backend in backends_with_models: + if backend_model.id == backend_id: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend_with_model: + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_with_model_by_id_or_error( + project: ProjectModel, backend_id: UUID +) -> AsyncIterator[BackendTuple]: + async with lease_project_backend_with_model_by_id( + project=project, backend_id=backend_id + ) as backend_with_model: + if backend_with_model is None: + raise BackendNotAvailable() + yield backend_with_model + + +async def get_project_backend_model_by_type( + project: ProjectModel, backend_type: BackendType +) -> Optional[BackendModel]: + for backend in project.backends: + if backend.type == backend_type: + return backend + return None + + +async def get_project_backend_model_by_type_or_error( + project: ProjectModel, backend_type: BackendType +) -> BackendModel: + backend_model = await get_project_backend_model_by_type( + project=project, backend_type=backend_type + ) + if backend_model is None: + raise BackendNotAvailable() + return backend_model + + +async def close_idle_backends(): + """ + Retires and closes backends that are not leased for `_BACKEND_IDLE_TTL`. + """ + idle_since = time.monotonic() - _BACKEND_IDLE_TTL + to_close: List[_CachedBackend] = [] + for project_id in list(_BACKENDS_CACHE): + async with _get_project_cache_lock(project_id): + project_backends = _BACKENDS_CACHE.get(project_id, {}) + for backend_type, cached_backend in list(project_backends.items()): + if cached_backend.leases == 0 and cached_backend.last_used < idle_since: + del project_backends[backend_type] + to_close.extend(_retire_backend(cached_backend)) + if not project_backends: + _BACKENDS_CACHE.pop(project_id, None) + await _close_backends(to_close) + + +async def close_all_backends(): + """ + Retires all cached backends. Backends that are leased are closed once released. + """ + to_close: List[_CachedBackend] = [] + for project_id in list(_BACKENDS_CACHE): + async with _get_project_cache_lock(project_id): + for cached_backend in _BACKENDS_CACHE.pop(project_id, {}).values(): + to_close.extend(_retire_backend(cached_backend)) + await _close_backends(to_close) + + +@asynccontextmanager +async def _lease_selected_project_backend( + project: ProjectModel, + select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], +) -> AsyncIterator[Optional[BackendTuple]]: + # All project backends are leased while `select` runs outside of the cache lock, + # so that the selected backend cannot be closed in between. + cached_backends = await _acquire_project_backends(project, touch=False) + try: + backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) + except BaseException: + await _release_backends(cached_backends, touch=False) + raise + selected = [ + c + for c in cached_backends + if backend_with_model is not None and c.backend is backend_with_model[1] + ] + await _release_backends( + [c for c in cached_backends if c not in selected], + touch=False, + ) + for c in selected: + c.last_used = time.monotonic() + try: + yield backend_with_model + finally: + await _release_backends(selected) + + +async def _acquire_project_backends( + project: ProjectModel, touch: bool = True +) -> List[_CachedBackend]: + to_close: List[_CachedBackend] = [] async with _get_project_cache_lock(project.id): - key = project.id - project_backends = _BACKENDS_CACHE.get(key, {}) - to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] - for backend_model in project.backends: - cached_backend = project_backends.get(backend_model.type) + project_backends = _BACKENDS_CACHE.setdefault(project.id, {}) + backend_models = {backend_model.type: backend_model for backend_model in project.backends} + for backend_type, cached_backend in list(project_backends.items()): + backend_model = backend_models.get(backend_type) if ( - cached_backend is not None - and cached_backend[0].config == backend_model.config - and cached_backend[0].auth == backend_model.auth + backend_model is None + or cached_backend.model.config != backend_model.config + or cached_backend.model.auth != backend_model.auth ): + del project_backends[backend_type] + to_close.extend(_retire_backend(cached_backend)) + to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] + for backend_model in project.backends: + if backend_model.type in project_backends: continue configurator = get_configurator(backend_model.type) if configurator is None: @@ -320,7 +544,9 @@ async def get_project_backends_with_models(project: ProjectModel) -> List[Backen ) else: backend, duration = result - project_backends[backend_model.type] = (backend_model, backend) + project_backends[backend_model.type] = _CachedBackend( + model=backend_model, backend=backend + ) initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") logger.debug( "Initialized %d backends in %.1fs: %s", @@ -329,12 +555,51 @@ async def get_project_backends_with_models(project: ProjectModel) -> List[Backen ", ".join(initialized_results), ) - # `__setitem__()` will also expire the cache. - # Note that there is no global cache lock so a race condition is possible: - # one coroutine updates/re-assigns backends expired by another coroutine. - # This is ok since the only effect is that project's cache gets restored. - _BACKENDS_CACHE[key] = project_backends - return list(project_backends.values()) + cached_backends = list(project_backends.values()) + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases += 1 + if touch: + cached_backend.last_used = now + await _close_backends(to_close) + return cached_backends + + +async def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): + to_close: List[_CachedBackend] = [] + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases -= 1 + if touch: + cached_backend.last_used = now + if cached_backend.retired and cached_backend.leases == 0: + to_close.append(cached_backend) + await _close_backends(to_close) + + +def _retire_backend(cached_backend: _CachedBackend) -> List[_CachedBackend]: + """ + Marks the backend as retired and returns it if it can be closed now. + """ + cached_backend.retired = True + if cached_backend.leases == 0: + return [cached_backend] + return [] + + +async def _close_backends(cached_backends: List[_CachedBackend]): + for cached_backend, result in zip( + cached_backends, + await asyncio.gather( + *(run_async(c.backend.close) for c in cached_backends), return_exceptions=True + ), + ): + if isinstance(result, BaseException): + logger.error( + "Failed to close %s backend", + cached_backend.model.type.value, + exc_info=result, + ) async def _get_backend_tracked( @@ -345,128 +610,6 @@ async def _get_backend_tracked( return backend, time.time() - t -_get_project_backend_with_model_by_type = None - - -def set_get_project_backend_with_model_by_type( - func: Callable[[ProjectModel, BackendType], Coroutine[None, None, Optional[BackendTuple]]], -): - """ - Overrides `get_project_effective_backend_with_model_by_type` with `func`. - Then get_project_backend_* functions can pass overrides=True to call `func` - This can be used if a backend needs to be replaced with another backend. - For example, DstackBackend in dstack Sky can be used in place of other backends. - """ - global _get_project_backend_with_model_by_type - _get_project_backend_with_model_by_type = func - - -async def get_project_backend_with_model_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Optional[BackendTuple]: - if overrides and _get_project_backend_with_model_by_type is not None: - return await _get_project_backend_with_model_by_type(project, backend_type) - backends_with_models = await get_project_backends_with_models(project=project) - for backend_model, backend in backends_with_models: - if backend.TYPE == backend_type: - return backend_model, backend - return None - - -async def get_project_backend_with_model_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> BackendTuple: - backend_with_model = await get_project_backend_with_model_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend_with_model is None: - raise BackendNotAvailable() - return backend_with_model - - -async def get_project_backends(project: ProjectModel) -> List[Backend]: - backends_with_models = await get_project_backends_with_models(project) - backends = [b for _, b in backends_with_models] - return backends - - -async def get_project_backend_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Optional[Backend]: - backend_with_model = await get_project_backend_with_model_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend_with_model is None: - return None - return backend_with_model[1] - - -async def get_project_backend_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Backend: - backend = await get_project_backend_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend is None: - raise BackendNotAvailable() - return backend - - -async def get_project_backend_model_by_type( - project: ProjectModel, backend_type: BackendType -) -> Optional[BackendModel]: - for backend in project.backends: - if backend.type == backend_type: - return backend - return None - - -async def get_project_backend_model_by_type_or_error( - project: ProjectModel, backend_type: BackendType -) -> BackendModel: - backend_model = await get_project_backend_model_by_type( - project=project, backend_type=backend_type - ) - if backend_model is None: - raise BackendNotAvailable() - return backend_model - - -async def get_project_backend_with_model_by_id( - project: ProjectModel, backend_id: UUID -) -> Optional[BackendTuple]: - backends_with_models = await get_project_backends_with_models(project=project) - for backend_model, backend in backends_with_models: - if backend_model.id == backend_id: - return backend_model, backend - return None - - -async def get_project_backend_with_model_by_id_or_error( - project: ProjectModel, backend_id: UUID -) -> BackendTuple: - backend_with_model = await get_project_backend_with_model_by_id( - project=project, backend_id=backend_id - ) - if backend_with_model is None: - raise BackendNotAvailable() - return backend_with_model - - async def get_backend_offers( backends: List[Backend], requirements: Requirements, diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index ac677b4948..5ee3a48784 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -66,6 +66,7 @@ RunModel, UserModel, ) +from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import instances as instances_services from dstack._internal.server.services import offers as offers_services @@ -460,16 +461,17 @@ async def get_plan( include_only_create_instance_supported_backends = True if nodes is not None: include_only_create_instance_supported_backends = nodes.target != 0 - offers_with_backends = await get_fleet_offers( - project=project, - profile=effective_spec.merged_profile, - requirements=requirements, - fleet_spec=effective_spec, - blocks=effective_spec.configuration.blocks, - include_only_create_instance_supported_backends=( - include_only_create_instance_supported_backends - ), - ) + async with backends_services.lease_project_backends(project) as backends: + offers_with_backends = await get_fleet_offers( + backends=backends, + profile=effective_spec.merged_profile, + requirements=requirements, + fleet_spec=effective_spec, + blocks=effective_spec.configuration.blocks, + include_only_create_instance_supported_backends=( + include_only_create_instance_supported_backends + ), + ) offers = [offer for _, offer in offers_with_backends] _remove_fleet_spec_sensitive_info(effective_spec) @@ -490,7 +492,7 @@ async def get_plan( async def get_fleet_offers( - project: ProjectModel, + backends: List[Backend], profile: Profile, requirements: Requirements, placement_group: Optional[PlacementGroup] = None, @@ -528,7 +530,7 @@ async def get_fleet_offers( break offers = await offers_services.get_offers_by_requirements( - project=project, + backends=backends, profile=profile, requirements=requirements, exclude_not_available=exclude_not_available, diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index cb0edb0712..fd8946b1e3 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -66,7 +66,7 @@ from dstack._internal.server.services import events from dstack._internal.server.services.backends import ( check_backend_type_available, - get_project_backend_with_model_by_type_or_error, + lease_project_backend_with_model_by_type_or_error, ) from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.server.services.gateways.pool import gateway_connections_pool @@ -253,9 +253,11 @@ async def create_gateway( _validate_gateway_configuration(effective_configuration) configuration = effective_configuration - backend_model, _ = await get_project_backend_with_model_by_type_or_error( + # Leasing the backend also checks that it can be initialized, not just that it's configured. + async with lease_project_backend_with_model_by_type_or_error( project=project, backend_type=configuration.backend - ) + ) as (backend_model, _): + pass lock_namespace = f"gateway_names_{project.name}" if is_db_sqlite(): diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index eb826e03ee..6f6a575b9b 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -23,13 +23,12 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.runs import JobProvisioningData, Requirements from dstack._internal.core.models.volumes import Volume, VolumeConfigurationWithRegion -from dstack._internal.server.models import ProjectModel from dstack._internal.server.services import backends as backends_services from dstack._internal.utils.common import run_async async def get_offers_by_requirements( - project: ProjectModel, + backends: List[Backend], profile: Profile, requirements: Requirements, exclude_not_available=False, @@ -44,8 +43,11 @@ async def get_offers_by_requirements( full_offers: bool = False, unallocated_resources: bool = False, ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]: - backends: List[Backend] = await backends_services.get_project_backends(project=project) - + """ + Returns offers of `backends` matching the requirements. The caller must lease `backends` + (see `backends_services.lease_project_backends()`) and must not use the returned backends + after the lease is released. + """ backend_types: Optional[list[BackendType]] = profile.backends regions: Optional[list[str]] = profile.regions availability_zones: Optional[list[str]] = profile.availability_zones diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index 41869b9538..02aeada741 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -36,6 +36,7 @@ ProjectModel, RunModel, ) +from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services.fleets import ( check_can_create_new_cloud_instance_in_fleet, get_fleet_master_instance_provisioning_data, @@ -797,19 +798,20 @@ async def _get_backend_offers_in_fleet( # Handle multinode for old jobs that don't have requirements.multinode set. # TODO: Drop multinode param. multinode = requirements.multinode or is_multinode_job(job) - backend_offers = await get_offers_by_requirements( - project=project, - profile=profile, - requirements=requirements, - multinode=multinode, - master_job_provisioning_data=master_instance_provisioning_data, - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run_spec), - max_offers=max_offers, - full_offers=full_offers, - unallocated_resources=unallocated_resources, - ) + async with backends_services.lease_project_backends(project) as backends: + backend_offers = await get_offers_by_requirements( + backends=backends, + profile=profile, + requirements=requirements, + multinode=multinode, + master_job_provisioning_data=master_instance_provisioning_data, + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run_spec), + max_offers=max_offers, + full_offers=full_offers, + unallocated_resources=unallocated_resources, + ) return backend_offers @@ -877,18 +879,19 @@ async def get_non_fleet_offers( if skip_backend_offers: backend_offers = [] else: - backend_offers = await get_offers_by_requirements( - project=project, - profile=run_spec.merged_profile, - requirements=job.job_spec.requirements, - exclude_not_available=False, - multinode=is_multinode_job(job), - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run_spec), - full_offers=full_offers, - unallocated_resources=unallocated_resources, - ) + async with backends_services.lease_project_backends(project) as backends: + backend_offers = await get_offers_by_requirements( + backends=backends, + profile=run_spec.merged_profile, + requirements=job.job_spec.requirements, + exclude_not_available=False, + multinode=is_multinode_job(job), + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run_spec), + full_offers=full_offers, + unallocated_resources=unallocated_resources, + ) return instance_offers, backend_offers diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index 9d648eee46..262775a459 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -10,7 +10,6 @@ from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport from dstack._internal.core.backends.features import BACKENDS_WITH_VOLUMES_SUPPORT from dstack._internal.core.errors import ( - BackendNotAvailable, ResourceExistsError, ServerClientError, ) @@ -518,23 +517,21 @@ async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_mo ) return - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.provisioning_data.backend, - ) - except BackendNotAvailable: - logger.error( - f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + async with backends_services.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.provisioning_data.backend, + ) as backend: + if backend is None: + logger.error( + f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + ) + return + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + await common.run_async( + compute.delete_volume, + volume=volume, ) - return - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - await common.run_async( - compute.delete_volume, - volume=volume, - ) # Clouds charge volumes assuming 30-day months, e.g. https://aws.amazon.com/ebs/pricing/ diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 29f2eac87b..451d6dfc67 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -1,10 +1,11 @@ import json import uuid from collections.abc import Callable -from contextlib import contextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager from datetime import datetime, timezone from http import HTTPStatus from typing import Any, Dict, List, Literal, Optional, Union +from unittest.mock import Mock from uuid import UUID import gpuhunt @@ -147,6 +148,27 @@ from dstack._internal.server.services.users import get_token_hash +def lease_yielding(value: Any) -> Callable[..., AbstractAsyncContextManager[Any]]: + """ + Returns a replacement for a `lease_*()` function from `services.backends` + that yields `value`, e.g. to use as a mock's `side_effect`. + """ + + @asynccontextmanager + async def lease(*args, **kwargs): + yield value + + return lease + + +def mock_lease(value: Any) -> Mock: + """ + Returns a mock to patch a `lease_*()` function from `services.backends` with. + The lease yields `value`. + """ + return Mock(side_effect=lease_yielding(value)) + + def get_auth_headers(token: Union[DecryptedString, str]) -> Dict: if isinstance(token, DecryptedString): token = token.get_plaintext_or_error() diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py index a1b61979bf..c79bdccc55 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py @@ -1,7 +1,7 @@ import datetime as dt import logging from typing import ClassVar, Optional -from unittest.mock import AsyncMock, Mock +from unittest.mock import Mock import pytest import pytest_asyncio @@ -54,6 +54,7 @@ get_job_provisioning_data, get_remote_connection_info, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime from tests._internal.server.background.pipeline_tasks.instances.helpers import ( @@ -166,8 +167,8 @@ async def test_cloudrift_waits_for_backend_within_timeout( backend = Mock() monkeypatch.setattr( instances_check, - "_get_backend_for_provisioning_wait", - AsyncMock(return_value=backend), + "_lease_backend_for_provisioning_wait", + mock_lease(backend), ) await process_instance(session, worker, instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py index 804f81e0fe..695df5a20e 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py @@ -35,6 +35,7 @@ get_instance_offer_with_availability, get_job_provisioning_data, get_placement_group_provisioning_data, + lease_yielding, ) from tests._internal.server.background.pipeline_tasks.instances.helpers import ( instance_to_pipeline_item, @@ -86,9 +87,9 @@ async def test_creates_instance( total_blocks=requested_blocks, busy_blocks=0, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: backend_mock = Mock() - m.return_value = [backend_mock] + m.side_effect = lease_yielding([backend_mock]) backend_mock.TYPE = BackendType.AWS gpu = Gpu(name="T4", memory_mib=16384, vendor=gpuhunt.AcceleratorVendor.NVIDIA) offer = InstanceOfferWithAvailability( @@ -161,8 +162,8 @@ async def test_tries_second_offer_if_first_fails( region=offer.region, price=offer.price, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [aws_mock, gcp_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([aws_mock, gcp_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -190,8 +191,8 @@ async def test_fails_if_all_offers_fail( aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.get_offers.return_value = [offer] aws_mock.compute.return_value.create_instance.side_effect = err - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [aws_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([aws_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -210,8 +211,8 @@ async def test_fails_if_no_offers( project=project, status=InstanceStatus.PENDING, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -248,8 +249,8 @@ async def test_waits_when_fleet_has_no_current_master( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -296,8 +297,8 @@ async def test_waits_for_current_master_to_determine_cluster_placement( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) await session.refresh(master_instance) @@ -349,8 +350,8 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi master_item = instance_to_pipeline_item(master_instance) sibling_item = instance_to_pipeline_item(sibling_instance) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([]) await worker.process(master_item) await session.refresh(master_instance) @@ -383,8 +384,8 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [gcp_mock, aws_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([gcp_mock, aws_mock]) await worker.process(sibling_item) await session.refresh(sibling_instance) @@ -409,8 +410,8 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [promoted_backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([promoted_backend_mock]) await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -478,8 +479,8 @@ async def test_follows_current_master_backend_and_region_constraints( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [gcp_mock, aws_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([gcp_mock, aws_mock]) await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -540,8 +541,8 @@ async def test_non_master_does_not_create_new_placement_group_without_master_pg( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -605,8 +606,8 @@ async def test_non_master_reuses_existing_current_master_placement_group( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -677,8 +678,8 @@ async def test_allows_parallel_processing_after_master_is_provisioned( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, later_instance) assert backend_mock.compute.return_value.create_instance.call_count == 1 await process_instance(session, worker, earlier_instance) @@ -736,8 +737,8 @@ async def test_create_placement_group_if_placement_cluster( backend_mock.compute.return_value.create_placement_group.return_value = ( get_placement_group_provisioning_data() ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -790,8 +791,8 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.create_instance.return_value = get_job_provisioning_data() compute_mock.create_placement_group.return_value = get_placement_group_provisioning_data() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -859,8 +860,8 @@ def create_instance_method( get_placement_group_provisioning_data() ) backend_mock.compute.return_value.is_suitable_placement_group.return_value = can_reuse - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) await session.refresh(instance) @@ -918,8 +919,8 @@ async def test_master_reuses_existing_placement_group( backend_mock.compute.return_value.create_instance.return_value = ( get_job_provisioning_data() ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, master_instance) await session.refresh(master_instance) @@ -979,8 +980,8 @@ def create_placement_group_method( raise err backend_mock.compute.return_value.create_placement_group = create_placement_group_method - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) await session.refresh(instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py index 414518d0d1..2e5ed45f37 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py @@ -1,7 +1,7 @@ import datetime as dt from contextlib import contextmanager from typing import Optional -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import Mock, patch import pytest import time_machine @@ -21,6 +21,7 @@ create_instance, create_project, get_job_provisioning_data, + mock_lease, ) from tests._internal.server.background.pipeline_tasks.instances.helpers import ( instance_to_pipeline_item, @@ -42,8 +43,8 @@ def mock_terminate_in_backend(error: Optional[Exception] = None): terminate_instance.side_effect = error with patch.object( instances_termination.backends_services, - "get_project_backend_by_type", - AsyncMock(return_value=backend), + "lease_project_backend_by_type", + mock_lease(backend), ): yield terminate_instance diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py index 66296ba844..441e5df909 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py @@ -21,6 +21,7 @@ create_compute_group, create_fleet, create_project, + lease_yielding, ) from dstack._internal.utils.common import get_current_datetime @@ -164,11 +165,11 @@ async def test_terminates_compute_group( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD await worker.process(_compute_group_to_pipeline_item(compute_group)) compute_mock.terminate_compute_group.assert_called_once() @@ -190,11 +191,11 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) @@ -210,11 +211,11 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index 1c1ca8bb4f..d2b8e55252 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -48,6 +48,7 @@ get_gateway_replica_configuration, get_job_provisioning_data, get_run_spec, + lease_yielding, list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -413,10 +414,10 @@ async def test_submitted_to_provisioning( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: aws = Mock() - m.return_value = [(backend, aws)] + m.side_effect = lease_yielding((backend, aws)) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.return_value = ( GatewayReplicaProvisioningData( @@ -459,10 +460,10 @@ async def test_submitted_backend_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: aws = Mock() - m.return_value = [(backend, aws)] + m.side_effect = lease_yielding((backend, aws)) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = BackendError( "Some error" @@ -499,9 +500,9 @@ async def test_submitted_backend_not_available_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: - m.return_value = [] + m.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -535,7 +536,7 @@ async def test_submitted_skips_provisioning_if_gateway_to_be_deleted( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -570,7 +571,7 @@ async def test_submitted_skips_provisioning_if_gateway_failed( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -605,10 +606,10 @@ async def test_submitted_unexpected_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: aws = Mock() - m.return_value = [(backend, aws)] + m.side_effect = lease_yielding((backend, aws)) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = RuntimeError( "Unexpected!" @@ -647,7 +648,7 @@ async def test_submitted_to_terminated_when_scaled_in( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -2213,14 +2214,14 @@ async def test_provisioning_to_running_registers_with_load_balancer( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2262,7 +2263,7 @@ async def test_provisioning_skips_load_balancer_registration_without_hostname( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2304,7 +2305,7 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2314,7 +2315,7 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails backend_mock.compute.return_value.register_gateway_replica_with_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2351,14 +2352,14 @@ async def test_provisioning_to_terminating_when_backend_does_not_support_load_ba "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2590,7 +2591,7 @@ async def test_terminating_to_terminated( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2598,7 +2599,7 @@ async def test_terminating_to_terminated( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2668,7 +2669,7 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2676,7 +2677,7 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica_to_terminate)) @@ -2719,7 +2720,7 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2727,7 +2728,7 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2772,7 +2773,7 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2783,7 +2784,7 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( backend_mock.compute.return_value.deregister_gateway_replica_from_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2823,7 +2824,7 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2831,7 +2832,7 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2867,11 +2868,11 @@ async def test_terminating_waits_for_pending_acm_gateway_migration( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -2915,9 +2916,9 @@ async def test_terminating_to_terminated_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock: - get_backends_mock.return_value = [] + get_backends_mock.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -2960,7 +2961,7 @@ async def test_terminating_to_terminated_with_no_instance_id( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2968,7 +2969,7 @@ async def test_terminating_to_terminated_with_no_instance_id( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) @@ -3014,7 +3015,7 @@ async def test_terminating_retries_if_terminate_fails( with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -3025,7 +3026,7 @@ async def test_terminating_retries_if_terminate_fails( backend_mock.compute.return_value.terminate_gateway_replica.side_effect = Exception( "Terminate failed" ) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) await worker.process(_replica_to_pipeline_item(replica)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py index 2953744be4..9105fcb5a0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py @@ -35,6 +35,7 @@ create_gateway_replica, create_project, get_gateway_replica_configuration, + lease_yielding, list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -535,7 +536,7 @@ async def test_submitted_to_provisioning_creates_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -545,7 +546,7 @@ async def test_submitted_to_provisioning_creates_load_balancer( hostname="gateway-lb.example.com", backend_data="lb-backend-data" ) ) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -575,7 +576,7 @@ async def test_submitted_skips_load_balancer_creation_for_non_acm_gateway( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -602,12 +603,12 @@ async def test_submitted_to_failed_when_backend_does_not_support_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -643,13 +644,13 @@ async def test_submitted_to_failed_when_load_balancer_creation_raises( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_gateway_load_balancer.side_effect = exception - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -1698,12 +1699,12 @@ async def test_deletes_gateway_and_terminates_load_balancer_when_hostname_set( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1742,7 +1743,7 @@ async def test_delete_skips_load_balancer_termination_when_hostname_not_set( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -1778,7 +1779,7 @@ async def test_delete_deferred_when_load_balancer_termination_fails( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -1786,7 +1787,7 @@ async def test_delete_deferred_when_load_balancer_termination_fails( backend_mock.compute.return_value.terminate_gateway_load_balancer.side_effect = ( Exception("boom") ) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1827,12 +1828,12 @@ async def test_delete_deferred_when_backend_does_not_support_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1867,7 +1868,7 @@ async def test_delete_migrates_hostname_before_evaluating_termination( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) # Migration happens before the load balancer would be evaluated for termination, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py index 8294e86238..a42244bae2 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py @@ -88,6 +88,7 @@ get_run_spec, get_ssh_fleet_configuration, get_volume_provisioning_data, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime, get_or_error @@ -428,9 +429,11 @@ async def test_provisions_new_capacity_for_assigned_job_with_placeholder( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -511,12 +514,14 @@ async def test_multinode_master_reuses_placeholder_when_provisioning_falls_back_ ) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( dockerized=True, @@ -608,9 +613,11 @@ async def test_provisioning_master_job_respects_cluster_placement_in_non_empty_f fleet.lock_owner = "OtherPipeline:cluster-master" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS offer_1 = get_instance_offer_with_availability( backend=BackendType.AWS, @@ -674,7 +681,7 @@ async def test_defers_new_capacity_provisioning_until_fleet_master_is_elected( ) previous_last_processed_at = job.last_processed_at - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -732,9 +739,11 @@ async def test_provisioning_non_master_job_ignores_cluster_master_fleet_lock( fleet.lock_owner = "OtherPipeline:cluster-master" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS offer_1 = get_instance_offer_with_availability( backend=BackendType.AWS, @@ -783,12 +792,14 @@ async def test_creates_placement_group_for_cluster_fleet( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( backend=BackendType.AWS, @@ -836,12 +847,14 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.run_job.return_value = get_job_provisioning_data( @@ -905,12 +918,14 @@ async def test_marks_unused_existing_placement_groups_for_cleanup( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.is_suitable_placement_group.side_effect = ( lambda placement_group, _: placement_group.name == selected_pg.name @@ -963,12 +978,14 @@ async def test_marks_new_and_existing_placement_groups_for_cleanup_on_failed_pro job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.is_suitable_placement_group.return_value = False compute_mock.create_placement_group.return_value = ( @@ -1019,7 +1036,7 @@ async def test_resets_lock_for_retry_when_cluster_master_fleet_lock_is_unavailab previous_last_processed_at = job.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -1058,9 +1075,11 @@ async def test_reclaims_stale_related_cluster_master_fleet_lock( fleet.lock_owner = f"{JobSubmittedPipeline.__name__}:{job.id}" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.AWS) @@ -1854,9 +1873,11 @@ async def test_assignment_creates_placeholder_instance_for_new_capacity( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] @@ -1932,7 +1953,7 @@ async def test_terminates_job_when_no_reusable_instances_with_reuse_creation_pol ) job = await create_job(session=session, run=run) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) # Backend offers must not be requested with the reuse policy. @@ -1974,9 +1995,11 @@ async def test_job_fails_when_fleet_is_full( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] @@ -2027,12 +2050,14 @@ async def test_leaves_placeholder_for_terminating_pipeline_on_failed_new_capacit await session.commit() offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.side_effect = BackendError("boom") @@ -2058,12 +2083,14 @@ async def test_reports_tried_offers_when_new_capacity_provisioning_fails( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us-east-1") - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.side_effect = BackendError("InsufficientInstanceCapacity") @@ -2111,11 +2138,13 @@ async def test_provisions_compute_group( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_jobs.return_value = get_compute_group_provisioning_data( @@ -2181,11 +2210,13 @@ async def test_provisions_one_job_node_group_via_run_job( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( @@ -2457,9 +2488,11 @@ async def test_attaches_volume_on_existing_instance( instance_assigned=True, ) - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData() @@ -2528,9 +2561,11 @@ async def test_terminates_job_when_volume_is_locked_for_processing( ) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -2588,9 +2623,11 @@ async def test_reclaims_stale_related_volume_lock( volume.lock_owner = f"{JobSubmittedPipeline.__name__}:{job.id}" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData() @@ -2639,9 +2676,11 @@ async def test_run_job_uses_server_default_registry( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -2704,11 +2743,13 @@ async def test_run_jobs_uses_server_default_registry( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_jobs.return_value = get_compute_group_provisioning_data( @@ -2771,9 +2812,11 @@ async def test_interpolates_secrets_when_provisioning_new_capacity( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -2829,9 +2872,11 @@ async def test_terminates_job_when_secret_is_missing( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index 978a2720a1..d0360c1913 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -45,6 +45,7 @@ get_runner_job_submission_mismatch_error, get_volume_configuration, get_volume_provisioning_data, + lease_yielding, list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -810,14 +811,14 @@ async def test_detaches_job_volumes( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" ) as m: backend_mock = Mock() - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) - m.assert_awaited_once() + m.assert_called_once() backend_mock.compute.return_value.detach_volume.assert_called_once() backend_mock.compute.return_value.is_volume_detached.assert_called_once() @@ -861,10 +862,10 @@ async def test_force_detaches_job_volumes( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" ) as m: backend_mock = Mock() - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -880,7 +881,7 @@ async def test_force_detaches_job_volumes( await session.commit() with ( patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" ) as m, patch( "dstack._internal.server.background.pipeline_tasks.jobs_terminating.get_current_datetime" @@ -890,7 +891,7 @@ async def test_force_detaches_job_volumes( tzinfo=timezone.utc ) + timedelta(minutes=30) backend_mock = Mock() - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -909,10 +910,10 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" ) as m: backend_mock = Mock() - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) @@ -1017,10 +1018,10 @@ async def test_detaches_job_volumes_on_shared_instance( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" ) as m: backend_mock = Mock() - m.return_value = backend_mock + m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py index ccd2513230..72a05d8cc9 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime, timedelta, timezone from typing import Optional -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import Mock, patch import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -50,6 +50,7 @@ create_user, get_job_provisioning_data, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -404,8 +405,8 @@ def _patch_compute(compute: _ListingCompute): backend.compute.return_value = compute return patch.object( orphan_cleanup_pipeline.backends_services, - "get_project_backend_by_type", - AsyncMock(return_value=backend), + "lease_project_backend_by_type", + mock_lease(backend), ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py index 9b3f7fbef1..85deeb37c0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py @@ -19,6 +19,7 @@ create_fleet, create_placement_group, create_project, + lease_yielding, ) from dstack._internal.utils.common import get_current_datetime @@ -196,9 +197,9 @@ async def test_deletes_placement_group( placement_group.lock_token = uuid.uuid4() placement_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: aws_mock = Mock() - m.return_value = aws_mock + m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await worker.process(_placement_group_to_pipeline_item(placement_group)) aws_mock.compute.return_value.delete_placement_group.assert_called_once() @@ -225,9 +226,9 @@ async def test_retries_placement_group_deletion_if_still_in_use( placement_group.lock_owner = "PlacementGroupPipeline" original_last_processed_at = placement_group.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: aws_mock = Mock() - m.return_value = aws_mock + m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.delete_placement_group.side_effect = ( PlacementGroupInUseError() diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py index 2487dc4cfe..1dc511c4b3 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py @@ -6,7 +6,7 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.volumes import VolumeProvisioningData, VolumeStatus from dstack._internal.server.background.pipeline_tasks import volumes as volumes_pipeline @@ -24,6 +24,7 @@ create_volume, get_volume_configuration, get_volume_provisioning_data, + lease_yielding, list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -215,7 +216,7 @@ async def test_submitted_to_active(self, test_db, session: AsyncSession, worker: await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -224,7 +225,7 @@ async def test_submitted_to_active(self, test_db, session: AsyncSession, worker: volume_id="vol-1234", size_gb=100, ) - get_backend_mock.return_value = backend_mock + get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -256,7 +257,7 @@ async def test_registers_external_volume( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -267,7 +268,7 @@ async def test_registers_external_volume( size_gb=100, ) ) - get_backend_mock.return_value = backend_mock + get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -297,9 +298,9 @@ async def test_marks_volume_failed_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: - get_backend_mock.side_effect = BackendNotAvailable() + get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -329,14 +330,14 @@ async def test_marks_volume_failed_if_backend_returns_error( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_volume.side_effect = BackendError( "Some error" ) - get_backend_mock.return_value = backend_mock + get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -438,11 +439,11 @@ async def test_marks_volume_deleted( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backend_mock.return_value = backend_mock + get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -474,7 +475,7 @@ async def test_marks_external_volume_deleted_without_backend_call( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_not_called() @@ -504,9 +505,9 @@ async def test_marks_volume_deleted_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: - get_backend_mock.side_effect = BackendNotAvailable() + get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -535,14 +536,14 @@ async def test_marks_volume_deleted_if_backend_delete_errors( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.delete_volume.side_effect = BackendError( "Delete failed" ) - get_backend_mock.return_value = backend_mock + get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py index 3b6659729a..78ec75d8c4 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py @@ -20,6 +20,7 @@ create_volume, get_volume_configuration, get_volume_provisioning_data, + lease_yielding, list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -64,11 +65,9 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): ) await session.commit() - with patch( - "dstack._internal.server.services.backends.get_project_backend_by_type_or_error" - ) as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: aws_mock = Mock() - m.return_value = aws_mock + m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() @@ -110,11 +109,9 @@ async def test_deletes_idle_volume_with_null_auto_cleanup_enabled( volume.auto_cleanup_enabled = None await session.commit() - with patch( - "dstack._internal.server.services.backends.get_project_backend_by_type_or_error" - ) as m: + with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: aws_mock = Mock() - m.return_value = aws_mock + m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index 90c1c9bd3f..2d702e808c 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -54,6 +54,7 @@ get_private_key_string, get_remote_connection_info, get_ssh_fleet_configuration, + mock_lease, ) from dstack._internal.server.testing.matchers import SomeUUID4Str @@ -2248,9 +2249,11 @@ async def test_returns_create_plan_for_new_fleet( ) ] spec = get_fleet_spec() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = offers response = await client.post( @@ -2291,9 +2294,11 @@ async def test_returns_offers_for_elastic_container_backend_fleet( spec = get_fleet_spec( conf=get_fleet_configuration(nodes=FleetNodesSpec(min=0, target=0, max=1)) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] response = await client.post( @@ -2327,9 +2332,11 @@ async def test_returns_no_offers_for_non_elastic_container_backend_fleet( spec = get_fleet_spec( conf=get_fleet_configuration(nodes=FleetNodesSpec(min=0, target=1, max=1)) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] response = await client.post( diff --git a/src/tests/_internal/server/routers/test_gpus.py b/src/tests/_internal/server/routers/test_gpus.py index 638f133101..70f4305bb0 100644 --- a/src/tests/_internal/server/routers/test_gpus.py +++ b/src/tests/_internal/server/routers/test_gpus.py @@ -33,6 +33,7 @@ get_fleet_spec, get_instance_offer_with_availability, get_run_spec, + mock_lease, ) pytestmark = pytest.mark.usefixtures("image_config_mock") @@ -482,8 +483,10 @@ async def test_price_order( BackendType.AWS, "T4", 16384, 100, availability=InstanceAvailability.NO_QUOTA ), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = create_mock_backends_with_offers({BackendType.AWS: offers}) + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(create_mock_backends_with_offers({BackendType.AWS: offers})), + ): response = await call_gpus_api( client, project.name, user.token, run_spec, group_by=group_by, reverse=reverse ) @@ -550,10 +553,13 @@ async def test_cpu_groups( availability=InstanceAvailability.NO_QUOTA, ) unavailable.instance.resources.cpu_arch = gpuhunt.CPUArchitecture.X86 - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = create_mock_backends_with_offers( - {BackendType.AWS: aws, BackendType.AZURE: [unavailable]} - ) + mocked_backends = create_mock_backends_with_offers( + {BackendType.AWS: aws, BackendType.AZURE: [unavailable]} + ) + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -625,8 +631,10 @@ async def test_returns_gpus_without_group_by( offers_by_backend = {BackendType.AWS: [offer_aws], BackendType.RUNPOD: [offer_runpod]} mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api(client, project.name, user.token, run_spec) assert response.status_code == 200 @@ -657,8 +665,10 @@ async def test_forwards_full_offers_to_compute_get_offers( offer = create_gpu_offer(BackendType.AWS, "T4", 16384, 0.50) mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, user.token, run_spec, full_offers=body_full_offers ) @@ -694,8 +704,10 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( offer = create_gpu_offer(BackendType.AWS, "T4", 16384, 0.50) mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -763,8 +775,10 @@ async def test_filters_gpus_by_multiple_specified_fleets( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -791,8 +805,10 @@ async def test_includes_backend_offers_when_creation_policy_reuse_or_create( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) response = await call_gpus_api(client, project.name, user.token, run_spec) @@ -813,8 +829,10 @@ async def test_skips_backend_offers_when_creation_policy_reuse( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): # reuse skips backend offers, keeping only the existing instance. run_spec = get_run_spec( run_name="test-run", @@ -850,8 +868,10 @@ async def test_uses_targeted_instance_offers_when_instances_specified( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api(client, project.name, user.token, run_spec) assert response.status_code == 200 @@ -871,11 +891,13 @@ async def test_returns_empty_gpus_when_no_offers( repo = await create_repo(session=session, project_id=project.id) run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [] - m.return_value = [backend_mock_aws] response = await client.post( f"/api/project/{project.name}/gpus/list", @@ -1020,8 +1042,12 @@ async def test_exact_aggregation_values( BackendType.RUNPOD, "T4", 16384, 0.25, spot=True, region="us-east-1" ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ offer_t4_spot, @@ -1030,7 +1056,6 @@ async def test_exact_aggregation_values( offer_t4_multi, ] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [ offer_runpod_rtx_east, @@ -1038,8 +1063,6 @@ async def test_exact_aggregation_values( offer_runpod_t4_east, ] - m.return_value = [backend_mock_aws, backend_mock_runpod] - response = await client.post( f"/api/project/{project.name}/gpus/list", headers=get_auth_headers(user.token), diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 70cd675155..5ffb2587d2 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -75,6 +75,7 @@ get_run_spec, get_ssh_fleet_configuration, list_events, + mock_lease, ) from dstack._internal.server.testing.matchers import SomeUUID4Str from dstack._internal.utils.common import render_datetime_as_api @@ -1581,8 +1582,10 @@ async def test_sorts_offers_before_limit( } if reverse_offers is not None: body["reverse_offers"] = reverse_offers - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = [backend] + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1655,14 +1658,16 @@ async def test_returns_run_plan_privileged_false( privileged=False, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1706,12 +1711,14 @@ async def test_forwards_full_offers_to_compute_get_offers( if body_full_offers is not None: body["full_offers"] = body_full_offers - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS get_offers_mock = backend_mock.compute.return_value.get_offers get_offers_mock.return_value = [] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1761,12 +1768,14 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( if body_unallocated_resources is not None: body["unallocated_resources"] = body_unallocated_resources - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS get_offers_mock = backend_mock.compute.return_value.get_offers get_offers_mock.return_value = [] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1828,14 +1837,16 @@ async def test_returns_run_plan_privileged_true( privileged=True, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1874,11 +1885,13 @@ async def test_task_with_two_nodes_returns_two_job_plans( configuration=TaskConfiguration(commands=["echo hi"], nodes=2), ) body = {"run_spec": json.loads(run_spec.model_dump_json())} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1965,11 +1978,13 @@ def offers_by_requirements( return [gpu_offer] return [cpu_offer] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.side_effect = offers_by_requirements - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2015,18 +2030,20 @@ async def test_service_reservation_group_filters_backends_by_reservation_support ) body = {"run_spec": json.loads(run_spec.model_dump_json())} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() + aws_backend_mock = Mock() + verda_backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([aws_backend_mock, verda_backend_mock]), + ): aws_backend_mock.TYPE = BackendType.AWS aws_backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.AWS, price=2) ] - verda_backend_mock = Mock() verda_backend_mock.TYPE = BackendType.VERDA verda_backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.VERDA, price=1) ] - m.return_value = [aws_backend_mock, verda_backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", @@ -2091,14 +2108,16 @@ async def test_returns_run_plan_docker_true( docker=True, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2154,14 +2173,16 @@ async def test_returns_run_plan_instance_volumes( volumes=[InstanceMountPoint.parse("/data:/data")], ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2482,8 +2503,10 @@ async def test_preserves_backend_specific_offer_order( get_instance_offer_with_availability(backend=BackendType.VASTAI, price=2.0), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock_aws, backend_mock_vastai] + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_vastai]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2553,8 +2576,10 @@ async def test_offer_cli_preserves_backend_specific_offer_order_across_fleets( get_instance_offer_with_availability(backend=BackendType.VASTAI, price=2.0), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock_aws, backend_mock_vastai] + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_vastai]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2687,8 +2712,11 @@ async def test_offer_cli_deduplicates_identical_backend_offers_across_specified_ ) body = {"run_spec": run_spec.model_dump(), "for_offers_only": True} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2704,7 +2732,6 @@ async def test_offer_cli_deduplicates_identical_backend_offers_across_specified_ availability_zones=["us-a"], ) ] - m.return_value = [backend_mock_aws] response = await client.post( f"/api/project/{project.name}/runs/get_plan", @@ -2808,8 +2835,12 @@ async def test_offer_cli_without_fleet_keeps_global_offers( ), ) body = {"run_spec": run_spec.model_dump(), "for_offers_only": True} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2823,7 +2854,6 @@ async def test_offer_cli_without_fleet_keeps_global_offers( availability=InstanceAvailability.AVAILABLE, ) ] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2837,7 +2867,6 @@ async def test_offer_cli_without_fleet_keeps_global_offers( availability=InstanceAvailability.AVAILABLE, ) ] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py new file mode 100644 index 0000000000..b250d21d93 --- /dev/null +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -0,0 +1,165 @@ +import uuid +from types import SimpleNamespace +from typing import cast +from unittest.mock import Mock + +import pytest + +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.errors import BackendNotAvailable +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.server.services import backends as backends_services + + +def make_backend_model(backend_type: BackendType, config: str = "{}") -> BackendModel: + backend_model = SimpleNamespace( + id=uuid.uuid4(), + type=backend_type, + config=config, + auth=SimpleNamespace(decrypted=True), + ) + return cast(BackendModel, backend_model) + + +def make_project(*backend_models: BackendModel) -> ProjectModel: + project = SimpleNamespace(id=uuid.uuid4(), backends=list(backend_models)) + return cast(ProjectModel, project) + + +def close_mock(backend: Backend) -> Mock: + return cast(Mock, backend.close) + + +class TestBackendsCache: + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + monkeypatch.setattr(backends_services, "_BACKENDS_CACHE", {}) + monkeypatch.setattr(backends_services, "_BACKENDS_CACHE_LOCKS", {}) + monkeypatch.setattr(backends_services, "get_stored_backend_record", lambda m: m) + self.backends: list[Mock] = [] + + def get_backend(backend_model): + backend = Mock(TYPE=backend_model.type) + self.backends.append(backend) + return backend + + monkeypatch.setattr( + backends_services, + "get_configurator", + lambda backend_type: Mock(get_backend=Mock(side_effect=get_backend)), + ) + + @pytest.mark.asyncio + async def test_reuses_backends_between_leases(self): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_services.lease_project_backends(project) as first: + pass + async with backends_services.lease_project_backends(project) as second: + pass + assert first == second == self.backends + self.backends[0].close.assert_not_called() + + @pytest.mark.asyncio + async def test_closes_backend_with_changed_config_after_release(self): + project = make_project(make_backend_model(BackendType.AWS, config="old")) + async with backends_services.lease_project_backends(project) as [old_backend]: + project.backends = [make_backend_model(BackendType.AWS, config="new")] + async with backends_services.lease_project_backends(project) as [new_backend]: + assert new_backend is not old_backend + close_mock(old_backend).assert_not_called() + close_mock(old_backend).assert_called_once() + close_mock(new_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_closes_deleted_backend(self): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_services.lease_project_backends(project): + pass + aws_backend, gcp_backend = self.backends + project.backends = project.backends[:1] + async with backends_services.lease_project_backends(project) as backends: + assert backends == [aws_backend] + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_closes_idle_backends(self, monkeypatch): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_services.lease_project_backends(project): + pass + aws_backend, gcp_backend = self.backends + async with backends_services.lease_project_backend_by_type( + project, BackendType.AWS + ) as backend: + assert backend is aws_backend + # Only GCP is idle: AWS is leased and was used just now + monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) + await backends_services.close_idle_backends() + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypatch): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_services.lease_project_backends(project): + pass + aws_backend, gcp_backend = self.backends + monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) + async with backends_services.lease_project_backend_by_type(project, BackendType.AWS): + await backends_services.close_idle_backends() + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_close_all_backends_waits_for_leases(self): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_services.lease_project_backends(project) as [backend]: + await backends_services.close_all_backends() + close_mock(backend).assert_not_called() + close_mock(backend).assert_called_once() + + @pytest.mark.asyncio + async def test_does_not_raise_if_backend_fails_to_close(self): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_services.lease_project_backends(project) as [backend]: + close_mock(backend).side_effect = RuntimeError("close failed") + await backends_services.close_all_backends() + close_mock(backend).assert_called_once() + + @pytest.mark.asyncio + async def test_or_error_raises_and_releases_leases(self): + project = make_project(make_backend_model(BackendType.AWS)) + with pytest.raises(BackendNotAvailable): + async with backends_services.lease_project_backend_by_type_or_error( + project, BackendType.GCP + ): + pass + [aws_backend] = self.backends + await backends_services.close_all_backends() + close_mock(aws_backend).assert_called_once() + + @pytest.mark.asyncio + async def test_uses_selector_with_overrides(self, monkeypatch): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + return backends_with_models[0] + + monkeypatch.setattr( + backends_services, "_select_project_backend_with_model_by_type", select + ) + async with backends_services.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ) as backend: + assert backend is self.backends[0] + async with backends_services.lease_project_backend_by_type( + project, BackendType.AWS + ) as backend: + assert backend is None diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index a4c9cf2621..3222bddb1b 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -46,6 +46,7 @@ get_job_provisioning_data, get_remote_connection_info, get_run_spec, + mock_lease, ) pytestmark = pytest.mark.usefixtures("image_config_mock") @@ -839,6 +840,10 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el ) offer = get_instance_offer_with_availability() backend = AsyncMock() + monkeypatch.setattr( + "dstack._internal.server.services.backends.lease_project_backends", + mock_lease([backend]), + ) get_offers_by_requirements_mock.return_value = [(backend, offer)] offers = await _get_backend_offers_in_fleet( @@ -853,4 +858,5 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el get_offers_by_requirements_mock.assert_awaited_once() await_args = get_offers_by_requirements_mock.await_args assert await_args is not None + assert await_args.kwargs["backends"] == [backend] assert await_args.kwargs["master_job_provisioning_data"] is None diff --git a/src/tests/_internal/server/services/test_fleets.py b/src/tests/_internal/server/services/test_fleets.py index 19ae011d65..1663a68964 100644 --- a/src/tests/_internal/server/services/test_fleets.py +++ b/src/tests/_internal/server/services/test_fleets.py @@ -4,7 +4,6 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.fleets import ( @@ -17,7 +16,6 @@ ) from dstack._internal.core.models.instances import RemoteConnectionInfo from dstack._internal.server.models import FleetModel, ProjectModel -from dstack._internal.server.services.backends import get_project_backends from dstack._internal.server.services.fleets import ( get_fleet_master_instance_provisioning_data, get_plan, @@ -30,14 +28,17 @@ get_fleet_spec, get_job_provisioning_data, get_ssh_key, + mock_lease, ) class TestGetPlanSSHFleetHostsValidation: @pytest.fixture - def get_project_backends_mock(self, monkeypatch: pytest.MonkeyPatch) -> list[Backend]: - mock = Mock(spec_set=get_project_backends, return_value=[]) - monkeypatch.setattr("dstack._internal.server.services.backends.get_project_backends", mock) + def lease_project_backends_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = mock_lease([]) + monkeypatch.setattr( + "dstack._internal.server.services.backends.lease_project_backends", mock + ) return mock def get_ssh_fleet_spec( @@ -74,7 +75,7 @@ async def create_fleet( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_same_fleet_update(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -88,7 +89,7 @@ async def test_ok_same_fleet_update(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_deleted_instances_ignored(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -106,7 +107,7 @@ async def test_ok_deleted_instances_ignored(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_no_common_hosts_with_another_fleet(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -120,7 +121,7 @@ async def test_ok_no_common_hosts_with_another_fleet(self, session: AsyncSession @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_another_fleet_same_project(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -138,7 +139,7 @@ async def test_error_another_fleet_same_project(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_another_fleet_another_project(self, session: AsyncSession): another_user = await create_user(session=session, name="another-user") another_project = await create_project( @@ -160,7 +161,7 @@ async def test_error_another_fleet_another_project(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_fleet_spec_without_name(self, session: AsyncSession): # Even if the user apply the same configuration again, we cannot be sure if it is the same # fleet or a brand new fleet, as we identify fleets by name. diff --git a/src/tests/_internal/server/services/test_offers.py b/src/tests/_internal/server/services/test_offers.py index 7ff0b4fe4a..02658c50e4 100644 --- a/src/tests/_internal/server/services/test_offers.py +++ b/src/tests/_internal/server/services/test_offers.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest @@ -21,85 +21,76 @@ class TestGetOffersByRequirements: async def test_returns_all_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer), (runpod_backend_mock, runpod_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) + runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, aws_offer), (runpod_backend_mock, runpod_offer)] @pytest.mark.asyncio async def test_returns_multinode_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - vastai_backend_mock = Mock() - vastai_backend_mock.TYPE = BackendType.VASTAI - vastai_offer = get_instance_offer_with_availability(backend=BackendType.VASTAI) - vastai_backend_mock.compute.return_value.get_offers.return_value = [vastai_offer] - m.return_value = [aws_backend_mock, vastai_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - multinode=True, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + vastai_backend_mock = Mock() + vastai_backend_mock.TYPE = BackendType.VASTAI + vastai_offer = get_instance_offer_with_availability(backend=BackendType.VASTAI) + vastai_backend_mock.compute.return_value.get_offers.return_value = [vastai_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, vastai_backend_mock], + profile=profile, + requirements=requirements, + multinode=True, + ) + assert res == [(aws_backend_mock, aws_offer)] @pytest.mark.asyncio async def test_returns_volume_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer1 = get_instance_offer_with_availability( - backend=BackendType.RUNPOD, region="eu" - ) - runpod_offer2 = get_instance_offer_with_availability( - backend=BackendType.RUNPOD, region="us" - ) - runpod_backend_mock.compute.return_value.get_offers.return_value = [ - runpod_offer1, - runpod_offer2, - ] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - volumes=[ - [ - get_volume( - configuration=get_volume_configuration( - backend=BackendType.RUNPOD, region="us" - ) + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer1 = get_instance_offer_with_availability( + backend=BackendType.RUNPOD, region="eu" + ) + runpod_offer2 = get_instance_offer_with_availability( + backend=BackendType.RUNPOD, region="us" + ) + runpod_backend_mock.compute.return_value.get_offers.return_value = [ + runpod_offer1, + runpod_offer2, + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + volumes=[ + [ + get_volume( + configuration=get_volume_configuration( + backend=BackendType.RUNPOD, region="us" ) - ] - ], - ) - m.assert_awaited_once() - assert res == [(runpod_backend_mock, runpod_offer2)] + ) + ] + ], + ) + assert res == [(runpod_backend_mock, runpod_offer2)] @pytest.mark.asyncio async def test_global_volume_matches_cpu_and_gpu_regions(self): @@ -113,16 +104,12 @@ async def test_global_volume_matches_cpu_and_gpu_regions(self): aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us") aws_backend.compute.return_value.get_offers.return_value = [aws_offer] - with patch( - "dstack._internal.server.services.backends.get_project_backends", - return_value=[daytona_backend, aws_backend], - ): - offers = await get_offers_by_requirements( - project=Mock(), - profile=Profile(), - requirements=Requirements(resources=ResourcesSpec()), - volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], - ) + offers = await get_offers_by_requirements( + backends=[daytona_backend, aws_backend], + profile=Profile(), + requirements=Requirements(resources=ResourcesSpec()), + volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], + ) assert offers == [(daytona_backend, cpu_offer), (daytona_backend, gpu_offer)] @@ -130,113 +117,102 @@ async def test_global_volume_matches_cpu_and_gpu_regions(self): async def test_returns_volume_offers_without_region(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - kubernetes_backend_mock = Mock() - kubernetes_backend_mock.TYPE = BackendType.KUBERNETES - kubernetes_offer = get_instance_offer_with_availability( - backend=BackendType.KUBERNETES, - region="", - availability_zones=None, - ) - kubernetes_backend_mock.compute.return_value.get_offers.return_value = [ - kubernetes_offer, - get_instance_offer_with_availability( - backend=BackendType.KUBERNETES, region="other-cluster" - ), - ] - m.return_value = [aws_backend_mock, kubernetes_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - volumes=[[get_volume(configuration=get_kubernetes_volume_configuration())]], - ) - m.assert_awaited_once() - assert res == [(kubernetes_backend_mock, kubernetes_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + kubernetes_backend_mock = Mock() + kubernetes_backend_mock.TYPE = BackendType.KUBERNETES + kubernetes_offer = get_instance_offer_with_availability( + backend=BackendType.KUBERNETES, + region="", + availability_zones=None, + ) + kubernetes_backend_mock.compute.return_value.get_offers.return_value = [ + kubernetes_offer, + get_instance_offer_with_availability( + backend=BackendType.KUBERNETES, region="other-cluster" + ), + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, kubernetes_backend_mock], + profile=profile, + requirements=requirements, + volumes=[[get_volume(configuration=get_kubernetes_volume_configuration())]], + ) + assert res == [(kubernetes_backend_mock, kubernetes_offer)] @pytest.mark.asyncio async def test_returns_az_offers(self): profile = Profile(name="test", availability_zones=["az1", "az3"]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer1 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az1"] - ) - aws_offer2 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az2"] - ) - aws_offer3 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az2", "az3"] - ) - expected_aws_offer3 = aws_offer3.model_copy() - expected_aws_offer3.availability_zones = ["az3"] - aws_offer4 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=None - ) - aws_backend_mock.compute.return_value.get_offers.return_value = [ - aws_offer1, - aws_offer2, - aws_offer3, - aws_offer4, - ] - m.return_value = [aws_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer1), (aws_backend_mock, expected_aws_offer3)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer1 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az1"] + ) + aws_offer2 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az2"] + ) + aws_offer3 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az2", "az3"] + ) + expected_aws_offer3 = aws_offer3.model_copy() + expected_aws_offer3.availability_zones = ["az3"] + aws_offer4 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=None + ) + aws_backend_mock.compute.return_value.get_offers.return_value = [ + aws_offer1, + aws_offer2, + aws_offer3, + aws_offer4, + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, aws_offer1), (aws_backend_mock, expected_aws_offer3)] @pytest.mark.asyncio async def test_returns_az_offers_ignoring_case(self): profile = Profile(name="test", availability_zones=["AZ1"]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az1", "az2"] - ) - # The offer keeps the zone spelling reported by the backend. - expected_aws_offer = aws_offer.model_copy() - expected_aws_offer.availability_zones = ["az1"] - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - m.return_value = [aws_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - assert res == [(aws_backend_mock, expected_aws_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az1", "az2"] + ) + # The offer keeps the zone spelling reported by the backend. + expected_aws_offer = aws_offer.model_copy() + expected_aws_offer.availability_zones = ["az1"] + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, expected_aws_offer)] @pytest.mark.asyncio async def test_returns_no_offers_for_multinode_instance_mounts_and_non_multinode_backend(self): # Regression test for https://github.com/dstackai/dstack/issues/2211 profile = Profile(name="test", backends=[BackendType.RUNPOD]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - multinode=True, - instance_mounts=True, - ) - m.assert_awaited_once() - assert res == [] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) + runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + multinode=True, + instance_mounts=True, + ) + assert res == [] From 764fdac568b166bf18dfbc28206234eddfe555db Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:46:12 +0500 Subject: [PATCH 04/19] Close retired backends in the background Leases no longer close backends, so closing does not delay callers and cannot be skipped if a lease is cancelled. Retired backends stay in the cache until close_unused_backends() closes them. --- .../background/scheduled_tasks/__init__.py | 4 +- .../server/services/backends/__init__.py | 97 +++++++++---------- .../server/services/backends/test_cache.py | 32 +++++- 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py index b1598b1a01..ba332c90d3 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py @@ -30,7 +30,7 @@ collect_prometheus_metrics, delete_prometheus_metrics, ) -from dstack._internal.server.services.backends import close_idle_backends +from dstack._internal.server.services.backends import close_unused_backends _scheduler = AsyncIOScheduler() @@ -54,7 +54,7 @@ def start_scheduled_tasks() -> AsyncIOScheduler: _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1) _scheduler.add_job(delete_events, IntervalTrigger(minutes=7), max_instances=1) _scheduler.add_job(process_gateways_connections, IntervalTrigger(seconds=15)) - _scheduler.add_job(close_idle_backends, IntervalTrigger(minutes=1), max_instances=1) + _scheduler.add_job(close_unused_backends, IntervalTrigger(minutes=1), max_instances=1) _scheduler.add_job( process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1 ) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 35e94ae8cf..de59016a5a 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -267,17 +267,23 @@ class _CachedBackend: backend: Backend leases: int = 0 last_used: float = field(default_factory=time.monotonic) - retired: bool = False + + +@dataclass +class _ProjectBackends: + active: Dict[BackendType, _CachedBackend] = field(default_factory=dict) + retired: List[_CachedBackend] = field(default_factory=list) _BACKEND_IDLE_TTL = 300 _BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} -_BACKENDS_CACHE: Dict[UUID, Dict[BackendType, _CachedBackend]] = {} +_BACKENDS_CACHE: Dict[UUID, _ProjectBackends] = {} """ A backend is cached per project and used through leases (`lease_project_backend*()`). The cache retires a backend if its config changes, if it is deleted from the project, -or if it is not leased for `_BACKEND_IDLE_TTL`. A retired backend is closed -once its last lease is released, so callers must not use backends outside of leases. +or if it is not leased for `_BACKEND_IDLE_TTL`. New leases use only active backends. +`close_unused_backends()` closes retired backends once they are not leased, +so callers must not use backends outside of leases. """ @@ -293,7 +299,7 @@ async def lease_project_backends_with_models( try: yield [(c.model, c.backend) for c in cached_backends] finally: - await _release_backends(cached_backends) + _release_backends(cached_backends) @asynccontextmanager @@ -425,33 +431,39 @@ async def get_project_backend_model_by_type_or_error( return backend_model -async def close_idle_backends(): +async def close_unused_backends(): """ - Retires and closes backends that are not leased for `_BACKEND_IDLE_TTL`. + Retires backends that are not leased for `_BACKEND_IDLE_TTL` + and closes retired backends that are not leased. """ idle_since = time.monotonic() - _BACKEND_IDLE_TTL - to_close: List[_CachedBackend] = [] - for project_id in list(_BACKENDS_CACHE): - async with _get_project_cache_lock(project_id): - project_backends = _BACKENDS_CACHE.get(project_id, {}) - for backend_type, cached_backend in list(project_backends.items()): - if cached_backend.leases == 0 and cached_backend.last_used < idle_since: - del project_backends[backend_type] - to_close.extend(_retire_backend(cached_backend)) - if not project_backends: - _BACKENDS_CACHE.pop(project_id, None) - await _close_backends(to_close) + await _close_retired_backends( + should_retire=lambda c: c.leases == 0 and c.last_used < idle_since + ) async def close_all_backends(): """ - Retires all cached backends. Backends that are leased are closed once released. + Retires all backends and closes those that are not leased. """ + await _close_retired_backends(should_retire=lambda c: True) + + +async def _close_retired_backends(should_retire: Callable[[_CachedBackend], bool]): to_close: List[_CachedBackend] = [] for project_id in list(_BACKENDS_CACHE): async with _get_project_cache_lock(project_id): - for cached_backend in _BACKENDS_CACHE.pop(project_id, {}).values(): - to_close.extend(_retire_backend(cached_backend)) + project_backends = _BACKENDS_CACHE.get(project_id) + if project_backends is None: + continue + for backend_type, cached_backend in list(project_backends.active.items()): + if should_retire(cached_backend): + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) + to_close.extend(c for c in project_backends.retired if c.leases == 0) + project_backends.retired = [c for c in project_backends.retired if c.leases > 0] + if not project_backends.active and not project_backends.retired: + del _BACKENDS_CACHE[project_id] await _close_backends(to_close) @@ -462,48 +474,45 @@ async def _lease_selected_project_backend( ) -> AsyncIterator[Optional[BackendTuple]]: # All project backends are leased while `select` runs outside of the cache lock, # so that the selected backend cannot be closed in between. + # Only the selected backend counts as used. cached_backends = await _acquire_project_backends(project, touch=False) try: backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) except BaseException: - await _release_backends(cached_backends, touch=False) + _release_backends(cached_backends, touch=False) raise selected = [ c for c in cached_backends if backend_with_model is not None and c.backend is backend_with_model[1] ] - await _release_backends( - [c for c in cached_backends if c not in selected], - touch=False, - ) + _release_backends([c for c in cached_backends if c not in selected], touch=False) for c in selected: c.last_used = time.monotonic() try: yield backend_with_model finally: - await _release_backends(selected) + _release_backends(selected) async def _acquire_project_backends( project: ProjectModel, touch: bool = True ) -> List[_CachedBackend]: - to_close: List[_CachedBackend] = [] async with _get_project_cache_lock(project.id): - project_backends = _BACKENDS_CACHE.setdefault(project.id, {}) + project_backends = _BACKENDS_CACHE.setdefault(project.id, _ProjectBackends()) backend_models = {backend_model.type: backend_model for backend_model in project.backends} - for backend_type, cached_backend in list(project_backends.items()): + for backend_type, cached_backend in list(project_backends.active.items()): backend_model = backend_models.get(backend_type) if ( backend_model is None or cached_backend.model.config != backend_model.config or cached_backend.model.auth != backend_model.auth ): - del project_backends[backend_type] - to_close.extend(_retire_backend(cached_backend)) + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] for backend_model in project.backends: - if backend_model.type in project_backends: + if backend_model.type in project_backends.active: continue configurator = get_configurator(backend_model.type) if configurator is None: @@ -544,7 +553,7 @@ async def _acquire_project_backends( ) else: backend, duration = result - project_backends[backend_model.type] = _CachedBackend( + project_backends.active[backend_model.type] = _CachedBackend( model=backend_model, backend=backend ) initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") @@ -555,36 +564,22 @@ async def _acquire_project_backends( ", ".join(initialized_results), ) - cached_backends = list(project_backends.values()) + cached_backends = list(project_backends.active.values()) now = time.monotonic() for cached_backend in cached_backends: cached_backend.leases += 1 if touch: cached_backend.last_used = now - await _close_backends(to_close) return cached_backends -async def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): - to_close: List[_CachedBackend] = [] +def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): + # Releasing does not await, so it cannot be interrupted by cancellation now = time.monotonic() for cached_backend in cached_backends: cached_backend.leases -= 1 if touch: cached_backend.last_used = now - if cached_backend.retired and cached_backend.leases == 0: - to_close.append(cached_backend) - await _close_backends(to_close) - - -def _retire_backend(cached_backend: _CachedBackend) -> List[_CachedBackend]: - """ - Marks the backend as retired and returns it if it can be closed now. - """ - cached_backend.retired = True - if cached_backend.leases == 0: - return [cached_backend] - return [] async def _close_backends(cached_backends: List[_CachedBackend]): diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index b250d21d93..4718792129 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -1,3 +1,4 @@ +import asyncio import uuid from types import SimpleNamespace from typing import cast @@ -67,7 +68,9 @@ async def test_closes_backend_with_changed_config_after_release(self): project.backends = [make_backend_model(BackendType.AWS, config="new")] async with backends_services.lease_project_backends(project) as [new_backend]: assert new_backend is not old_backend + await backends_services.close_unused_backends() close_mock(old_backend).assert_not_called() + await backends_services.close_unused_backends() close_mock(old_backend).assert_called_once() close_mock(new_backend).assert_not_called() @@ -82,6 +85,7 @@ async def test_closes_deleted_backend(self): project.backends = project.backends[:1] async with backends_services.lease_project_backends(project) as backends: assert backends == [aws_backend] + await backends_services.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @@ -99,7 +103,7 @@ async def test_closes_idle_backends(self, monkeypatch): assert backend is aws_backend # Only GCP is idle: AWS is leased and was used just now monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) - await backends_services.close_idle_backends() + await backends_services.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @@ -113,17 +117,39 @@ async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypat aws_backend, gcp_backend = self.backends monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) async with backends_services.lease_project_backend_by_type(project, BackendType.AWS): - await backends_services.close_idle_backends() + await backends_services.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @pytest.mark.asyncio - async def test_close_all_backends_waits_for_leases(self): + async def test_close_all_backends_does_not_close_leased_backends(self): project = make_project(make_backend_model(BackendType.AWS)) async with backends_services.lease_project_backends(project) as [backend]: await backends_services.close_all_backends() close_mock(backend).assert_not_called() + async with backends_services.lease_project_backends(project) as [new_backend]: + assert new_backend is not backend + await backends_services.close_all_backends() close_mock(backend).assert_called_once() + close_mock(new_backend).assert_called_once() + + @pytest.mark.asyncio + async def test_releases_lease_if_cancelled(self): + project = make_project(make_backend_model(BackendType.AWS)) + leased = asyncio.Event() + + async def hold_lease(): + async with backends_services.lease_project_backends(project): + leased.set() + await asyncio.Event().wait() + + task = asyncio.create_task(hold_lease()) + await leased.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await backends_services.close_all_backends() + close_mock(self.backends[0]).assert_called_once() @pytest.mark.asyncio async def test_does_not_raise_if_backend_fails_to_close(self): From f8ffe03e9d7b8668d1c0cf85ee0cb5cc832d8cc8 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:54:03 +0500 Subject: [PATCH 05/19] Annotate lease context managers with AsyncGenerator --- .../pipeline_tasks/instances/check.py | 4 ++-- .../server/services/backends/__init__.py | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index ef363ae9d3..c6c24b9573 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -1,6 +1,6 @@ import logging import uuid -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager from datetime import timedelta from typing import Optional @@ -388,7 +388,7 @@ async def _process_wait_for_instance_provisioning_data( async def _lease_backend_for_provisioning_wait( project_id: uuid.UUID, backend_type: BackendType, -) -> AsyncIterator[Optional[Backend]]: +) -> AsyncGenerator[Optional[Backend], None]: async with get_session_ctx() as session: res = await session.execute( select(ProjectModel) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index de59016a5a..c9fa4f76b9 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,7 +1,7 @@ import asyncio import json import time -from collections.abc import AsyncIterator, Iterable, Iterator +from collections.abc import AsyncGenerator, Iterable, Iterator from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import Callable, Coroutine, Dict, List, Optional, Tuple @@ -294,7 +294,7 @@ def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: @asynccontextmanager async def lease_project_backends_with_models( project: ProjectModel, -) -> AsyncIterator[List[BackendTuple]]: +) -> AsyncGenerator[List[BackendTuple], None]: cached_backends = await _acquire_project_backends(project) try: yield [(c.model, c.backend) for c in cached_backends] @@ -303,7 +303,7 @@ async def lease_project_backends_with_models( @asynccontextmanager -async def lease_project_backends(project: ProjectModel) -> AsyncIterator[List[Backend]]: +async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: async with lease_project_backends_with_models(project) as backends_with_models: yield [b for _, b in backends_with_models] @@ -332,7 +332,7 @@ async def lease_project_backend_with_model_by_type( project: ProjectModel, backend_type: BackendType, overrides: bool = False, -) -> AsyncIterator[Optional[BackendTuple]]: +) -> AsyncGenerator[Optional[BackendTuple], None]: async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: if overrides and _select_project_backend_with_model_by_type is not None: return await _select_project_backend_with_model_by_type( @@ -352,7 +352,7 @@ async def lease_project_backend_with_model_by_type_or_error( project: ProjectModel, backend_type: BackendType, overrides: bool = False, -) -> AsyncIterator[BackendTuple]: +) -> AsyncGenerator[BackendTuple, None]: async with lease_project_backend_with_model_by_type( project=project, backend_type=backend_type, overrides=overrides ) as backend_with_model: @@ -366,7 +366,7 @@ async def lease_project_backend_by_type( project: ProjectModel, backend_type: BackendType, overrides: bool = False, -) -> AsyncIterator[Optional[Backend]]: +) -> AsyncGenerator[Optional[Backend], None]: async with lease_project_backend_with_model_by_type( project=project, backend_type=backend_type, overrides=overrides ) as backend_with_model: @@ -378,7 +378,7 @@ async def lease_project_backend_by_type_or_error( project: ProjectModel, backend_type: BackendType, overrides: bool = False, -) -> AsyncIterator[Backend]: +) -> AsyncGenerator[Backend, None]: async with lease_project_backend_with_model_by_type_or_error( project=project, backend_type=backend_type, overrides=overrides ) as (_, backend): @@ -388,7 +388,7 @@ async def lease_project_backend_by_type_or_error( @asynccontextmanager async def lease_project_backend_with_model_by_id( project: ProjectModel, backend_id: UUID -) -> AsyncIterator[Optional[BackendTuple]]: +) -> AsyncGenerator[Optional[BackendTuple], None]: async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: for backend_model, backend in backends_with_models: if backend_model.id == backend_id: @@ -402,7 +402,7 @@ async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTu @asynccontextmanager async def lease_project_backend_with_model_by_id_or_error( project: ProjectModel, backend_id: UUID -) -> AsyncIterator[BackendTuple]: +) -> AsyncGenerator[BackendTuple, None]: async with lease_project_backend_with_model_by_id( project=project, backend_id=backend_id ) as backend_with_model: @@ -471,7 +471,7 @@ async def _close_retired_backends(should_retire: Callable[[_CachedBackend], bool async def _lease_selected_project_backend( project: ProjectModel, select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], -) -> AsyncIterator[Optional[BackendTuple]]: +) -> AsyncGenerator[Optional[BackendTuple], None]: # All project backends are leased while `select` runs outside of the cache lock, # so that the selected backend cannot be closed in between. # Only the selected backend counts as used. From 2e920795f9a8a02bf75004db2bcf031d8ef48406 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 15:58:11 +0500 Subject: [PATCH 06/19] Simplify selecting a leased backend --- .../server/services/backends/__init__.py | 39 ++++++++++++------- .../server/services/backends/test_cache.py | 38 ++++++++++++++++++ 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index c9fa4f76b9..c8d7798043 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -472,27 +472,36 @@ async def _lease_selected_project_backend( project: ProjectModel, select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], ) -> AsyncGenerator[Optional[BackendTuple], None]: - # All project backends are leased while `select` runs outside of the cache lock, - # so that the selected backend cannot be closed in between. - # Only the selected backend counts as used. + """ + Leases the backend chosen by `select` from the project backends. + + `select` may do I/O, so it runs outside of the cache lock: otherwise it would block + other leases of the project and could deadlock if it leased backends itself. Then the + selected backend could be retired and closed before it is leased, so all project backends + are leased while `select` runs, and the ones not selected are released after it. + They are leased without updating `last_used` so that only the selected backend + counts as used for idle retirement. + """ cached_backends = await _acquire_project_backends(project, touch=False) + selected: Optional[_CachedBackend] = None try: backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) - except BaseException: - _release_backends(cached_backends, touch=False) - raise - selected = [ - c - for c in cached_backends - if backend_with_model is not None and c.backend is backend_with_model[1] - ] - _release_backends([c for c in cached_backends if c not in selected], touch=False) - for c in selected: - c.last_used = time.monotonic() + if backend_with_model is not None: + selected = next( + (c for c in cached_backends if c.backend is backend_with_model[1]), None + ) + if selected is None: + raise ValueError("`select` must return one of the given backends") + finally: + _release_backends([c for c in cached_backends if c is not selected], touch=False) + if selected is None: + yield None + return + selected.last_used = time.monotonic() try: yield backend_with_model finally: - _release_backends(selected) + _release_backends([selected]) async def _acquire_project_backends( diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index 4718792129..dbbfc3eb31 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -189,3 +189,41 @@ async def select(project, backend_type, backends_with_models): project, BackendType.AWS ) as backend: assert backend is None + + @pytest.mark.asyncio + async def test_releases_leases_if_selector_fails(self, monkeypatch): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + raise RuntimeError("select failed") + + monkeypatch.setattr( + backends_services, "_select_project_backend_with_model_by_type", select + ) + with pytest.raises(RuntimeError): + async with backends_services.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ): + pass + await backends_services.close_all_backends() + close_mock(self.backends[0]).assert_called_once() + + @pytest.mark.asyncio + async def test_raises_and_releases_leases_if_selector_returns_unknown_backend( + self, monkeypatch + ): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + return backends_with_models[0][0], Mock() + + monkeypatch.setattr( + backends_services, "_select_project_backend_with_model_by_type", select + ) + with pytest.raises(ValueError): + async with backends_services.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ): + pass + await backends_services.close_all_backends() + close_mock(self.backends[0]).assert_called_once() From c0bd7de174187c05b68a92576c17a5b3312bf434 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:06:44 +0500 Subject: [PATCH 07/19] Move the backends cache to cache.py and backend model helpers to store.py --- src/dstack/_internal/server/app.py | 4 +- .../pipeline_tasks/compute_groups.py | 4 +- .../pipeline_tasks/gateway_replicas.py | 8 +- .../background/pipeline_tasks/gateways.py | 6 +- .../pipeline_tasks/instances/check.py | 4 +- .../instances/cloud_provisioning.py | 4 +- .../pipeline_tasks/instances/termination.py | 4 +- .../pipeline_tasks/jobs_submitted.py | 6 +- .../pipeline_tasks/jobs_terminating.py | 4 +- .../pipeline_tasks/orphan_cleanup.py | 4 +- .../pipeline_tasks/placement_groups.py | 4 +- .../background/pipeline_tasks/volumes.py | 6 +- .../background/scheduled_tasks/__init__.py | 2 +- .../server/services/backends/__init__.py | 463 +----------------- .../server/services/backends/cache.py | 364 ++++++++++++++ .../server/services/backends/store.py | 121 +++++ .../_internal/server/services/fleets.py | 4 +- .../server/services/gateways/__init__.py | 4 +- .../_internal/server/services/offers.py | 2 +- .../_internal/server/services/projects.py | 2 +- .../_internal/server/services/runs/plan.py | 6 +- .../_internal/server/services/volumes.py | 3 +- .../instances/test_cloud_provisioning.py | 36 +- .../instances/test_termination.py | 2 +- .../pipeline_tasks/test_compute_groups.py | 12 +- .../pipeline_tasks/test_gateway_replicas.py | 40 +- .../pipeline_tasks/test_gateways.py | 18 +- .../pipeline_tasks/test_jobs_submitted.py | 50 +- .../pipeline_tasks/test_jobs_terminating.py | 10 +- .../pipeline_tasks/test_orphan_cleanup.py | 2 +- .../pipeline_tasks/test_placement_groups.py | 8 +- .../background/pipeline_tasks/test_volumes.py | 16 +- .../scheduled_tasks/test_idle_volumes.py | 8 +- .../_internal/server/routers/test_fleets.py | 6 +- .../_internal/server/routers/test_gpus.py | 22 +- .../_internal/server/routers/test_runs.py | 28 +- .../server/services/backends/test_cache.py | 88 ++-- .../server/services/runs/test_plan.py | 2 +- .../_internal/server/services/test_fleets.py | 2 +- 39 files changed, 715 insertions(+), 664 deletions(-) create mode 100644 src/dstack/_internal/server/services/backends/cache.py create mode 100644 src/dstack/_internal/server/services/backends/store.py diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index ad7fbff443..76b6f03db6 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -48,8 +48,8 @@ users, volumes, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import prometheus as prometheus_service +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.config import ServerConfigManager from dstack._internal.server.services.gateways import gateway_connections_pool from dstack._internal.server.services.jobs.server_connection import job_server_connections_pool @@ -225,7 +225,7 @@ async def lifespan(app: FastAPI): scheduler.shutdown() if pipeline_manager is not None: await pipeline_manager.drain() - await backends_services.close_all_backends() + await backends_cache.close_all_backends() await gateway_connections_pool.remove_all() await job_server_connections_pool.remove_all() service_conn_pool = await get_injector_from_app(app).get_service_connection_pool() diff --git a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py index 11fbf33f00..af8c0510f0 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py @@ -28,7 +28,7 @@ ) from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ComputeGroupModel, InstanceModel, ProjectModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.compute_groups import compute_group_model_to_compute_group from dstack._internal.server.services.instances import emit_instance_status_change_event from dstack._internal.server.services.locking import get_locker @@ -289,7 +289,7 @@ async def _terminate_compute_group(compute_group_model: ComputeGroupModel) -> _T return result compute_group = compute_group_model_to_compute_group(compute_group_model) cgpd = compute_group.provisioning_data - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=compute_group_model.project, backend_type=cgpd.backend, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index a989e59eb1..457935fdf5 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -45,9 +45,9 @@ ServiceRegistrationModel, ServiceReplicaRegistrationModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import gateways as gateways_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.gateways import ( get_gateway_configuration, get_gateway_lb_configuration, @@ -454,7 +454,7 @@ async def _provision_gateway_replica( gateway_model: GatewayModel, replica_model: GatewayReplicaModel, ) -> _GatewayReplicaUpdateMap: - async with backends_services.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_with_model_by_id( project=gateway_model.project, backend_id=replica_model.backend_id ) as backend_with_model: if backend_with_model is None: @@ -581,7 +581,7 @@ async def _register_replica_with_load_balancer( """ if replica_model.instance_id is None: return "instance_id is None, cannot register with load balancer" - async with backends_services.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_with_model_by_id( project=gateway_model.project, backend_id=replica_model.backend_id ) as backend_with_model: if backend_with_model is None: @@ -1468,7 +1468,7 @@ async def _process_terminating_item(item: GatewayReplicaPipelineItem): mark_terminated_update_map = _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.TERMINATED, active=False, deleted=True ) - async with backends_services.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_with_model_by_id( project=gateway_model.project, backend_id=replica_model.backend_id, ) as backend_with_model: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py index a4290c485d..300c6a1c66 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py @@ -39,9 +39,9 @@ GatewayReplicaModel, ProjectModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import gateways as gateways_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.gateways import ( emit_gateway_status_change_event, get_gateway_lb_configuration, @@ -351,7 +351,7 @@ async def _process_submitted_gateway(gateway_model: GatewayModel) -> _SubmittedR configuration = gateways_services.get_gateway_configuration(gateway_model) update_map: _GatewayUpdateMap = {} if get_effective_load_balancer(configuration) is not None: - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=gateway_model.project, backend_type=configuration.backend ) as backend: if backend is None: @@ -680,7 +680,7 @@ async def _process_to_be_deleted_gateway(gateway_model: GatewayModel) -> _Proces async def _terminate_gateway_load_balancer(gateway_model: GatewayModel) -> bool: """Terminates the gateway's load balancer. Returns True on success, False on failure.""" configuration = gateways_services.get_gateway_configuration(gateway_model) - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=gateway_model.project, backend_type=configuration.backend ) as backend: if backend is None: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index c6c24b9573..c3bab21783 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -53,7 +53,7 @@ ComponentStatus, InstanceHealthResponse, ) -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( get_instance_provisioning_data, get_instance_ssh_private_keys, @@ -399,7 +399,7 @@ async def _lease_backend_for_provisioning_wait( if project_model is None: yield None return - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=project_model, backend_type=backend_type, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py index cf2a7abf59..45e7384b0e 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py @@ -37,7 +37,7 @@ ) from dstack._internal.server.db import get_session_ctx from dstack._internal.server.models import FleetModel, InstanceModel, PlacementGroupModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.fleets import get_fleet_offers, is_cloud_cluster from dstack._internal.server.services.instances import ( get_instance_configuration, @@ -104,7 +104,7 @@ async def create_cloud_instance(instance_model: InstanceModel) -> ProcessResult: ) master_job_provisioning_data = cluster_context.master_job_provisioning_data - async with backends_services.lease_project_backends(instance_model.project) as backends: + async with backends_cache.lease_project_backends(instance_model.project) as backends: offers = await get_fleet_offers( backends=backends, profile=profile, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py index 70bf8b9395..470c26e167 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py @@ -13,7 +13,7 @@ set_status_update, ) from dstack._internal.server.models import InstanceModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.runner.pool import ( instance_connection_pool, @@ -35,7 +35,7 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is not None and job_provisioning_data.backend != BackendType.REMOTE: - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=instance_model.project, backend_type=job_provisioning_data.backend, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 3b845f7627..606f51609b 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -83,8 +83,8 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.docker import apply_server_docker_defaults from dstack._internal.server.services.fleets import ( can_create_new_cloud_instance_in_fleet, @@ -1886,7 +1886,7 @@ async def _process_volume_attachments( if len(prepared_job_volumes.volume_model_ids) == 0: return _VolumeAttachmentResult(attachments=[], locked_volume_ids=[]) - async with backends_services.lease_project_backend_by_type_or_error( + async with backends_cache.lease_project_backend_by_type_or_error( project=project, backend_type=job_provisioning_data.backend ) as backend: compute = backend.compute() @@ -2429,7 +2429,7 @@ async def _provision_new_capacity( fleet_model=fleet_model, ) multinode = requirements.multinode or is_multinode_job(job) - async with backends_services.lease_project_backends(project) as backends: + async with backends_cache.lease_project_backends(project) as backends: offers = await get_offers_by_requirements( backends=backends, profile=profile, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index ab9bbc5280..0f7bab7992 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -48,9 +48,9 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import logs as logs_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( emit_instance_status_change_event, get_instance_ssh_private_keys, @@ -964,7 +964,7 @@ async def _detach_volumes_from_job_instance( run_termination_reason: Optional[RunTerminationReason], ) -> _VolumeDetachResult: job_spec = get_job_spec(job_model) - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=instance_model.project, backend_type=jpd.backend, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py index 95d97fa76d..d47d99c982 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py @@ -45,8 +45,8 @@ ProjectModel, ProviderRefModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol @@ -382,7 +382,7 @@ async def _process_orphan_cleanup(process_context: _ProcessContext) -> _ProcessR orphan_cleanup_model = process_context.orphan_cleanup_model if not process_context.needs_cleanup: return _ProcessResult(update_map=_OrphanCleanupUpdateMap()) - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=orphan_cleanup_model.backend.project, backend_type=orphan_cleanup_model.backend.type, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py index fa88fb1955..53c10f8694 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py @@ -29,7 +29,7 @@ PlacementGroupModel, ProjectModel, ) -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.placement import placement_group_model_to_placement_group @@ -245,7 +245,7 @@ async def _delete_placement_group( "Failed to delete placement group %s. provisioning_data is None.", placement_group.name ) return _get_deleted_result() - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=placement_group_model.project, backend_type=placement_group.provisioning_data.backend, ) as backend: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py index d9d82e2f70..23189fd79d 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py @@ -34,8 +34,8 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.volumes import ( @@ -306,7 +306,7 @@ class _ProcessResult: async def _process_submitted_volume(volume_model: VolumeModel) -> _ProcessResult: volume = volume_model_to_volume(volume_model) - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=volume_model.project, backend_type=volume.configuration.backend, overrides=True, @@ -381,7 +381,7 @@ async def _process_to_be_deleted_volume(volume_model: VolumeModel) -> _ProcessRe f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None." ) return _get_deleted_result() - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=volume_model.project, backend_type=volume.provisioning_data.backend, ) as backend: diff --git a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py index ba332c90d3..697133a1c8 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py @@ -30,7 +30,7 @@ collect_prometheus_metrics, delete_prometheus_metrics, ) -from dstack._internal.server.services.backends import close_unused_backends +from dstack._internal.server.services.backends.cache import close_unused_backends _scheduler = AsyncIOScheduler() diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index c8d7798043..25fb58b4b7 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,34 +1,20 @@ import asyncio -import json import time -from collections.abc import AsyncGenerator, Iterable, Iterator -from contextlib import asynccontextmanager -from dataclasses import dataclass, field -from typing import Callable, Coroutine, Dict, List, Optional, Tuple -from uuid import UUID +from collections.abc import Iterable, Iterator +from typing import List, Optional, Tuple -from pydantic import ValidationError from sqlalchemy import delete, update from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.backends.base.backend import Backend -from dstack._internal.core.backends.base.configurator import ( - Configurator, - StoredBackendRecord, -) +from dstack._internal.core.backends.base.configurator import Configurator from dstack._internal.core.backends.configurators import ( get_configurator, list_available_backend_types, ) -from dstack._internal.core.backends.models import ( - AnyBackendConfigWithCreds, - AnyBackendConfigWithoutCreds, - BackendConfigWithCreds, -) +from dstack._internal.core.backends.models import AnyBackendConfigWithCreds from dstack._internal.core.errors import ( - BackendAuthError, BackendError, - BackendInvalidCredentialsError, BackendNotAvailable, ResourceExistsError, ResourceNotExistsError, @@ -41,6 +27,12 @@ from dstack._internal.core.models.runs import Requirements from dstack._internal.server import settings from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel +from dstack._internal.server.services.backends.store import ( + get_backend_config_with_creds_from_backend_model, + get_project_backend_model_by_type, + get_source_backend_config_from_backend_model, + serialize_source_backend_config, +) from dstack._internal.server.services.offers import merge_offer_iterables from dstack._internal.utils.common import run_async from dstack._internal.utils.logging import get_logger @@ -48,16 +40,6 @@ logger = get_logger(__name__) -def serialize_source_backend_config( - config: AnyBackendConfigWithCreds, -) -> Tuple[str, Optional[str]]: - """Split user-intent backend config into non-sensitive and sensitive JSON blobs.""" - source_config_dict = config.model_dump() - source_auth = source_config_dict.pop("creds", None) - source_auth_json = None if source_auth is None else json.dumps(source_auth) - return json.dumps(source_config_dict), source_auth_json - - async def create_backend( session: AsyncSession, project: ProjectModel, @@ -163,75 +145,6 @@ async def get_source_backend_config( return get_source_backend_config_from_backend_model(backend_model) -def get_backend_config_with_creds_from_backend_model( - configurator: Configurator, - backend_model: BackendModel, -) -> AnyBackendConfigWithCreds: - backend_record = get_stored_backend_record(backend_model) - backend_config = configurator.get_backend_config_with_creds(backend_record) - return backend_config - - -def get_backend_config_without_creds_from_backend_model( - configurator: Configurator, - backend_model: BackendModel, -) -> AnyBackendConfigWithoutCreds: - backend_record = get_stored_backend_record(backend_model) - backend_config = configurator.get_backend_config_without_creds(backend_record) - return backend_config - - -def get_source_backend_config_from_backend_model( - backend_model: BackendModel, -) -> Optional[AnyBackendConfigWithCreds]: - """Reconstruct user-intent backend config from `source_config`/`source_auth`.""" - - if backend_model.source_config is None: - return None - try: - source_config_dict = json.loads(backend_model.source_config) - except ValueError: - logger.warning( - "Failed to parse source config for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - if backend_model.source_auth is not None: - if not backend_model.source_auth.decrypted: - logger.warning( - "Failed to decrypt source creds for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - try: - source_config_dict["creds"] = json.loads( - backend_model.source_auth.get_plaintext_or_error() - ) - except ValueError: - logger.warning( - "Failed to parse source creds for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - try: - return BackendConfigWithCreds.model_validate(source_config_dict).root - except ValidationError: - logger.warning( - "Failed to validate source config for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - - -def get_stored_backend_record(backend_model: BackendModel) -> StoredBackendRecord: - return StoredBackendRecord( - config=backend_model.config, - auth=backend_model.auth.get_plaintext_or_error(), - project_id=backend_model.project_id, - backend_id=backend_model.id, - ) - - async def delete_backends( session: AsyncSession, project: ProjectModel, @@ -258,362 +171,6 @@ async def delete_backends( ) -BackendTuple = Tuple[BackendModel, Backend] - - -@dataclass -class _CachedBackend: - model: BackendModel - backend: Backend - leases: int = 0 - last_used: float = field(default_factory=time.monotonic) - - -@dataclass -class _ProjectBackends: - active: Dict[BackendType, _CachedBackend] = field(default_factory=dict) - retired: List[_CachedBackend] = field(default_factory=list) - - -_BACKEND_IDLE_TTL = 300 -_BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} -_BACKENDS_CACHE: Dict[UUID, _ProjectBackends] = {} -""" -A backend is cached per project and used through leases (`lease_project_backend*()`). -The cache retires a backend if its config changes, if it is deleted from the project, -or if it is not leased for `_BACKEND_IDLE_TTL`. New leases use only active backends. -`close_unused_backends()` closes retired backends once they are not leased, -so callers must not use backends outside of leases. -""" - - -def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: - return _BACKENDS_CACHE_LOCKS.setdefault(project_id, asyncio.Lock()) - - -@asynccontextmanager -async def lease_project_backends_with_models( - project: ProjectModel, -) -> AsyncGenerator[List[BackendTuple], None]: - cached_backends = await _acquire_project_backends(project) - try: - yield [(c.model, c.backend) for c in cached_backends] - finally: - _release_backends(cached_backends) - - -@asynccontextmanager -async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: - async with lease_project_backends_with_models(project) as backends_with_models: - yield [b for _, b in backends_with_models] - - -_select_project_backend_with_model_by_type = None - - -def set_select_project_backend_with_model_by_type( - func: Callable[ - [ProjectModel, BackendType, List[BackendTuple]], - Coroutine[None, None, Optional[BackendTuple]], - ], -): - """ - Overrides how `lease_project_backend*_by_type()` select a backend from the project backends - if `overrides=True` is passed. This can be used if a backend needs to be replaced with - another backend. For example, DstackBackend in dstack Sky can be used in place of - other backends. - """ - global _select_project_backend_with_model_by_type - _select_project_backend_with_model_by_type = func - - -@asynccontextmanager -async def lease_project_backend_with_model_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[Optional[BackendTuple], None]: - async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: - if overrides and _select_project_backend_with_model_by_type is not None: - return await _select_project_backend_with_model_by_type( - project, backend_type, backends_with_models - ) - for backend_model, backend in backends_with_models: - if backend.TYPE == backend_type: - return backend_model, backend - return None - - async with _lease_selected_project_backend(project, select) as backend_with_model: - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_with_model_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[BackendTuple, None]: - async with lease_project_backend_with_model_by_type( - project=project, backend_type=backend_type, overrides=overrides - ) as backend_with_model: - if backend_with_model is None: - raise BackendNotAvailable() - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[Optional[Backend], None]: - async with lease_project_backend_with_model_by_type( - project=project, backend_type=backend_type, overrides=overrides - ) as backend_with_model: - yield backend_with_model[1] if backend_with_model is not None else None - - -@asynccontextmanager -async def lease_project_backend_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[Backend, None]: - async with lease_project_backend_with_model_by_type_or_error( - project=project, backend_type=backend_type, overrides=overrides - ) as (_, backend): - yield backend - - -@asynccontextmanager -async def lease_project_backend_with_model_by_id( - project: ProjectModel, backend_id: UUID -) -> AsyncGenerator[Optional[BackendTuple], None]: - async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: - for backend_model, backend in backends_with_models: - if backend_model.id == backend_id: - return backend_model, backend - return None - - async with _lease_selected_project_backend(project, select) as backend_with_model: - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_with_model_by_id_or_error( - project: ProjectModel, backend_id: UUID -) -> AsyncGenerator[BackendTuple, None]: - async with lease_project_backend_with_model_by_id( - project=project, backend_id=backend_id - ) as backend_with_model: - if backend_with_model is None: - raise BackendNotAvailable() - yield backend_with_model - - -async def get_project_backend_model_by_type( - project: ProjectModel, backend_type: BackendType -) -> Optional[BackendModel]: - for backend in project.backends: - if backend.type == backend_type: - return backend - return None - - -async def get_project_backend_model_by_type_or_error( - project: ProjectModel, backend_type: BackendType -) -> BackendModel: - backend_model = await get_project_backend_model_by_type( - project=project, backend_type=backend_type - ) - if backend_model is None: - raise BackendNotAvailable() - return backend_model - - -async def close_unused_backends(): - """ - Retires backends that are not leased for `_BACKEND_IDLE_TTL` - and closes retired backends that are not leased. - """ - idle_since = time.monotonic() - _BACKEND_IDLE_TTL - await _close_retired_backends( - should_retire=lambda c: c.leases == 0 and c.last_used < idle_since - ) - - -async def close_all_backends(): - """ - Retires all backends and closes those that are not leased. - """ - await _close_retired_backends(should_retire=lambda c: True) - - -async def _close_retired_backends(should_retire: Callable[[_CachedBackend], bool]): - to_close: List[_CachedBackend] = [] - for project_id in list(_BACKENDS_CACHE): - async with _get_project_cache_lock(project_id): - project_backends = _BACKENDS_CACHE.get(project_id) - if project_backends is None: - continue - for backend_type, cached_backend in list(project_backends.active.items()): - if should_retire(cached_backend): - del project_backends.active[backend_type] - project_backends.retired.append(cached_backend) - to_close.extend(c for c in project_backends.retired if c.leases == 0) - project_backends.retired = [c for c in project_backends.retired if c.leases > 0] - if not project_backends.active and not project_backends.retired: - del _BACKENDS_CACHE[project_id] - await _close_backends(to_close) - - -@asynccontextmanager -async def _lease_selected_project_backend( - project: ProjectModel, - select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], -) -> AsyncGenerator[Optional[BackendTuple], None]: - """ - Leases the backend chosen by `select` from the project backends. - - `select` may do I/O, so it runs outside of the cache lock: otherwise it would block - other leases of the project and could deadlock if it leased backends itself. Then the - selected backend could be retired and closed before it is leased, so all project backends - are leased while `select` runs, and the ones not selected are released after it. - They are leased without updating `last_used` so that only the selected backend - counts as used for idle retirement. - """ - cached_backends = await _acquire_project_backends(project, touch=False) - selected: Optional[_CachedBackend] = None - try: - backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) - if backend_with_model is not None: - selected = next( - (c for c in cached_backends if c.backend is backend_with_model[1]), None - ) - if selected is None: - raise ValueError("`select` must return one of the given backends") - finally: - _release_backends([c for c in cached_backends if c is not selected], touch=False) - if selected is None: - yield None - return - selected.last_used = time.monotonic() - try: - yield backend_with_model - finally: - _release_backends([selected]) - - -async def _acquire_project_backends( - project: ProjectModel, touch: bool = True -) -> List[_CachedBackend]: - async with _get_project_cache_lock(project.id): - project_backends = _BACKENDS_CACHE.setdefault(project.id, _ProjectBackends()) - backend_models = {backend_model.type: backend_model for backend_model in project.backends} - for backend_type, cached_backend in list(project_backends.active.items()): - backend_model = backend_models.get(backend_type) - if ( - backend_model is None - or cached_backend.model.config != backend_model.config - or cached_backend.model.auth != backend_model.auth - ): - del project_backends.active[backend_type] - project_backends.retired.append(cached_backend) - to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] - for backend_model in project.backends: - if backend_model.type in project_backends.active: - continue - configurator = get_configurator(backend_model.type) - if configurator is None: - logger.warning( - "Missing dependencies for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - if not backend_model.auth.decrypted: - logger.warning( - "Failed to decrypt creds for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - backend_record = get_stored_backend_record(backend_model) - to_init.append((backend_model, configurator, backend_record)) - - if to_init: - t0 = time.time() - tasks = [ - _get_backend_tracked(configurator, backend_record) - for _, configurator, backend_record in to_init - ] - results = await asyncio.gather(*tasks, return_exceptions=True) - initialized_results = [] - for (backend_model, _, _), result in zip(to_init, results): - if isinstance(result, BaseException): - if isinstance(result, (BackendInvalidCredentialsError, BackendAuthError)): - logger.warning( - "Credentials for %s backend are invalid. Backend will be ignored.", - backend_model.type.value, - ) - else: - logger.error( - "Failed to initialize %s backend. Backend will be ignored.", - backend_model.type.value, - exc_info=result, - ) - else: - backend, duration = result - project_backends.active[backend_model.type] = _CachedBackend( - model=backend_model, backend=backend - ) - initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") - logger.debug( - "Initialized %d backends in %.1fs: %s", - len(initialized_results), - time.time() - t0, - ", ".join(initialized_results), - ) - - cached_backends = list(project_backends.active.values()) - now = time.monotonic() - for cached_backend in cached_backends: - cached_backend.leases += 1 - if touch: - cached_backend.last_used = now - return cached_backends - - -def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): - # Releasing does not await, so it cannot be interrupted by cancellation - now = time.monotonic() - for cached_backend in cached_backends: - cached_backend.leases -= 1 - if touch: - cached_backend.last_used = now - - -async def _close_backends(cached_backends: List[_CachedBackend]): - for cached_backend, result in zip( - cached_backends, - await asyncio.gather( - *(run_async(c.backend.close) for c in cached_backends), return_exceptions=True - ), - ): - if isinstance(result, BaseException): - logger.error( - "Failed to close %s backend", - cached_backend.model.type.value, - exc_info=result, - ) - - -async def _get_backend_tracked( - configurator: Configurator, backend_record: StoredBackendRecord -) -> Tuple[Backend, float]: - t = time.time() - backend = await run_async(configurator.get_backend, backend_record) - return backend, time.time() - t - - async def get_backend_offers( backends: List[Backend], requirements: Requirements, diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py new file mode 100644 index 0000000000..6dd77df312 --- /dev/null +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -0,0 +1,364 @@ +import asyncio +import time +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import Callable, Coroutine, Dict, List, Optional, Tuple +from uuid import UUID + +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.backends.base.configurator import ( + Configurator, + StoredBackendRecord, +) +from dstack._internal.core.backends.configurators import ( + get_configurator, +) +from dstack._internal.core.errors import ( + BackendAuthError, + BackendInvalidCredentialsError, + BackendNotAvailable, +) +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.server.services.backends.store import get_stored_backend_record +from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +BackendTuple = Tuple[BackendModel, Backend] + + +@dataclass +class _CachedBackend: + model: BackendModel + backend: Backend + leases: int = 0 + last_used: float = field(default_factory=time.monotonic) + + +@dataclass +class _ProjectBackends: + active: Dict[BackendType, _CachedBackend] = field(default_factory=dict) + retired: List[_CachedBackend] = field(default_factory=list) + + +_BACKEND_IDLE_TTL = 300 +_BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} +_BACKENDS_CACHE: Dict[UUID, _ProjectBackends] = {} +""" +A backend is cached per project and used through leases (`lease_project_backend*()`). +The cache retires a backend if its config changes, if it is deleted from the project, +or if it is not leased for `_BACKEND_IDLE_TTL`. New leases use only active backends. +`close_unused_backends()` closes retired backends once they are not leased, +so callers must not use backends outside of leases. +""" + + +def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: + return _BACKENDS_CACHE_LOCKS.setdefault(project_id, asyncio.Lock()) + + +@asynccontextmanager +async def lease_project_backends_with_models( + project: ProjectModel, +) -> AsyncGenerator[List[BackendTuple], None]: + cached_backends = await _acquire_project_backends(project) + try: + yield [(c.model, c.backend) for c in cached_backends] + finally: + _release_backends(cached_backends) + + +@asynccontextmanager +async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: + async with lease_project_backends_with_models(project) as backends_with_models: + yield [b for _, b in backends_with_models] + + +_select_project_backend_with_model_by_type = None + + +def set_select_project_backend_with_model_by_type( + func: Callable[ + [ProjectModel, BackendType, List[BackendTuple]], + Coroutine[None, None, Optional[BackendTuple]], + ], +): + """ + Overrides how `lease_project_backend*_by_type()` select a backend from the project backends + if `overrides=True` is passed. This can be used if a backend needs to be replaced with + another backend. For example, DstackBackend in dstack Sky can be used in place of + other backends. + """ + global _select_project_backend_with_model_by_type + _select_project_backend_with_model_by_type = func + + +@asynccontextmanager +async def lease_project_backend_with_model_by_type( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncGenerator[Optional[BackendTuple], None]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + if overrides and _select_project_backend_with_model_by_type is not None: + return await _select_project_backend_with_model_by_type( + project, backend_type, backends_with_models + ) + for backend_model, backend in backends_with_models: + if backend.TYPE == backend_type: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend_with_model: + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_with_model_by_type_or_error( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncGenerator[BackendTuple, None]: + async with lease_project_backend_with_model_by_type( + project=project, backend_type=backend_type, overrides=overrides + ) as backend_with_model: + if backend_with_model is None: + raise BackendNotAvailable() + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_by_type( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncGenerator[Optional[Backend], None]: + async with lease_project_backend_with_model_by_type( + project=project, backend_type=backend_type, overrides=overrides + ) as backend_with_model: + yield backend_with_model[1] if backend_with_model is not None else None + + +@asynccontextmanager +async def lease_project_backend_by_type_or_error( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncGenerator[Backend, None]: + async with lease_project_backend_with_model_by_type_or_error( + project=project, backend_type=backend_type, overrides=overrides + ) as (_, backend): + yield backend + + +@asynccontextmanager +async def lease_project_backend_with_model_by_id( + project: ProjectModel, backend_id: UUID +) -> AsyncGenerator[Optional[BackendTuple], None]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + for backend_model, backend in backends_with_models: + if backend_model.id == backend_id: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend_with_model: + yield backend_with_model + + +@asynccontextmanager +async def lease_project_backend_with_model_by_id_or_error( + project: ProjectModel, backend_id: UUID +) -> AsyncGenerator[BackendTuple, None]: + async with lease_project_backend_with_model_by_id( + project=project, backend_id=backend_id + ) as backend_with_model: + if backend_with_model is None: + raise BackendNotAvailable() + yield backend_with_model + + +async def close_unused_backends(): + """ + Retires backends that are not leased for `_BACKEND_IDLE_TTL` + and closes retired backends that are not leased. + """ + idle_since = time.monotonic() - _BACKEND_IDLE_TTL + await _close_retired_backends( + should_retire=lambda c: c.leases == 0 and c.last_used < idle_since + ) + + +async def close_all_backends(): + """ + Retires all backends and closes those that are not leased. + """ + await _close_retired_backends(should_retire=lambda c: True) + + +async def _close_retired_backends(should_retire: Callable[[_CachedBackend], bool]): + to_close: List[_CachedBackend] = [] + for project_id in list(_BACKENDS_CACHE): + async with _get_project_cache_lock(project_id): + project_backends = _BACKENDS_CACHE.get(project_id) + if project_backends is None: + continue + for backend_type, cached_backend in list(project_backends.active.items()): + if should_retire(cached_backend): + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) + to_close.extend(c for c in project_backends.retired if c.leases == 0) + project_backends.retired = [c for c in project_backends.retired if c.leases > 0] + if not project_backends.active and not project_backends.retired: + del _BACKENDS_CACHE[project_id] + await _close_backends(to_close) + + +@asynccontextmanager +async def _lease_selected_project_backend( + project: ProjectModel, + select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], +) -> AsyncGenerator[Optional[BackendTuple], None]: + """ + Leases the backend chosen by `select` from the project backends. + + `select` may do I/O, so it runs outside of the cache lock: otherwise it would block + other leases of the project and could deadlock if it leased backends itself. Then the + selected backend could be retired and closed before it is leased, so all project backends + are leased while `select` runs, and the ones not selected are released after it. + They are leased without updating `last_used` so that only the selected backend + counts as used for idle retirement. + """ + cached_backends = await _acquire_project_backends(project, touch=False) + selected: Optional[_CachedBackend] = None + try: + backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) + if backend_with_model is not None: + selected = next( + (c for c in cached_backends if c.backend is backend_with_model[1]), None + ) + if selected is None: + raise ValueError("`select` must return one of the given backends") + finally: + _release_backends([c for c in cached_backends if c is not selected], touch=False) + if selected is None: + yield None + return + selected.last_used = time.monotonic() + try: + yield backend_with_model + finally: + _release_backends([selected]) + + +async def _acquire_project_backends( + project: ProjectModel, touch: bool = True +) -> List[_CachedBackend]: + async with _get_project_cache_lock(project.id): + project_backends = _BACKENDS_CACHE.setdefault(project.id, _ProjectBackends()) + backend_models = {backend_model.type: backend_model for backend_model in project.backends} + for backend_type, cached_backend in list(project_backends.active.items()): + backend_model = backend_models.get(backend_type) + if ( + backend_model is None + or cached_backend.model.config != backend_model.config + or cached_backend.model.auth != backend_model.auth + ): + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) + to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] + for backend_model in project.backends: + if backend_model.type in project_backends.active: + continue + configurator = get_configurator(backend_model.type) + if configurator is None: + logger.warning( + "Missing dependencies for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + if not backend_model.auth.decrypted: + logger.warning( + "Failed to decrypt creds for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + backend_record = get_stored_backend_record(backend_model) + to_init.append((backend_model, configurator, backend_record)) + + if to_init: + t0 = time.time() + tasks = [ + _get_backend_tracked(configurator, backend_record) + for _, configurator, backend_record in to_init + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + initialized_results = [] + for (backend_model, _, _), result in zip(to_init, results): + if isinstance(result, BaseException): + if isinstance(result, (BackendInvalidCredentialsError, BackendAuthError)): + logger.warning( + "Credentials for %s backend are invalid. Backend will be ignored.", + backend_model.type.value, + ) + else: + logger.error( + "Failed to initialize %s backend. Backend will be ignored.", + backend_model.type.value, + exc_info=result, + ) + else: + backend, duration = result + project_backends.active[backend_model.type] = _CachedBackend( + model=backend_model, backend=backend + ) + initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") + logger.debug( + "Initialized %d backends in %.1fs: %s", + len(initialized_results), + time.time() - t0, + ", ".join(initialized_results), + ) + + cached_backends = list(project_backends.active.values()) + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases += 1 + if touch: + cached_backend.last_used = now + return cached_backends + + +def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): + # Releasing does not await, so it cannot be interrupted by cancellation + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases -= 1 + if touch: + cached_backend.last_used = now + + +async def _close_backends(cached_backends: List[_CachedBackend]): + for cached_backend, result in zip( + cached_backends, + await asyncio.gather( + *(run_async(c.backend.close) for c in cached_backends), return_exceptions=True + ), + ): + if isinstance(result, BaseException): + logger.error( + "Failed to close %s backend", + cached_backend.model.type.value, + exc_info=result, + ) + + +async def _get_backend_tracked( + configurator: Configurator, backend_record: StoredBackendRecord +) -> Tuple[Backend, float]: + t = time.time() + backend = await run_async(configurator.get_backend, backend_record) + return backend, time.time() - t diff --git a/src/dstack/_internal/server/services/backends/store.py b/src/dstack/_internal/server/services/backends/store.py new file mode 100644 index 0000000000..ba0d3a9039 --- /dev/null +++ b/src/dstack/_internal/server/services/backends/store.py @@ -0,0 +1,121 @@ +import json +from typing import Optional, Tuple + +from pydantic import ValidationError + +from dstack._internal.core.backends.base.configurator import ( + Configurator, + StoredBackendRecord, +) +from dstack._internal.core.backends.models import ( + AnyBackendConfigWithCreds, + AnyBackendConfigWithoutCreds, + BackendConfigWithCreds, +) +from dstack._internal.core.errors import ( + BackendNotAvailable, +) +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +def serialize_source_backend_config( + config: AnyBackendConfigWithCreds, +) -> Tuple[str, Optional[str]]: + """Split user-intent backend config into non-sensitive and sensitive JSON blobs.""" + source_config_dict = config.model_dump() + source_auth = source_config_dict.pop("creds", None) + source_auth_json = None if source_auth is None else json.dumps(source_auth) + return json.dumps(source_config_dict), source_auth_json + + +def get_backend_config_with_creds_from_backend_model( + configurator: Configurator, + backend_model: BackendModel, +) -> AnyBackendConfigWithCreds: + backend_record = get_stored_backend_record(backend_model) + backend_config = configurator.get_backend_config_with_creds(backend_record) + return backend_config + + +def get_backend_config_without_creds_from_backend_model( + configurator: Configurator, + backend_model: BackendModel, +) -> AnyBackendConfigWithoutCreds: + backend_record = get_stored_backend_record(backend_model) + backend_config = configurator.get_backend_config_without_creds(backend_record) + return backend_config + + +def get_source_backend_config_from_backend_model( + backend_model: BackendModel, +) -> Optional[AnyBackendConfigWithCreds]: + """Reconstruct user-intent backend config from `source_config`/`source_auth`.""" + + if backend_model.source_config is None: + return None + try: + source_config_dict = json.loads(backend_model.source_config) + except ValueError: + logger.warning( + "Failed to parse source config for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + if backend_model.source_auth is not None: + if not backend_model.source_auth.decrypted: + logger.warning( + "Failed to decrypt source creds for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + try: + source_config_dict["creds"] = json.loads( + backend_model.source_auth.get_plaintext_or_error() + ) + except ValueError: + logger.warning( + "Failed to parse source creds for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + try: + return BackendConfigWithCreds.model_validate(source_config_dict).root + except ValidationError: + logger.warning( + "Failed to validate source config for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + + +def get_stored_backend_record(backend_model: BackendModel) -> StoredBackendRecord: + return StoredBackendRecord( + config=backend_model.config, + auth=backend_model.auth.get_plaintext_or_error(), + project_id=backend_model.project_id, + backend_id=backend_model.id, + ) + + +async def get_project_backend_model_by_type( + project: ProjectModel, backend_type: BackendType +) -> Optional[BackendModel]: + for backend in project.backends: + if backend.type == backend_type: + return backend + return None + + +async def get_project_backend_model_by_type_or_error( + project: ProjectModel, backend_type: BackendType +) -> BackendModel: + backend_model = await get_project_backend_model_by_type( + project=project, backend_type=backend_type + ) + if backend_model is None: + raise BackendNotAvailable() + return backend_model diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 5ee3a48784..7be4bded4f 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -66,10 +66,10 @@ RunModel, UserModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import instances as instances_services from dstack._internal.server.services import offers as offers_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( get_instance_remote_connection_info, is_placeholder_instance, @@ -461,7 +461,7 @@ async def get_plan( include_only_create_instance_supported_backends = True if nodes is not None: include_only_create_instance_supported_backends = nodes.target != 0 - async with backends_services.lease_project_backends(project) as backends: + async with backends_cache.lease_project_backends(project) as backends: offers_with_backends = await get_fleet_offers( backends=backends, profile=effective_spec.merged_profile, diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index fd8946b1e3..d188f212df 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -64,8 +64,8 @@ UserModel, ) from dstack._internal.server.services import events -from dstack._internal.server.services.backends import ( - check_backend_type_available, +from dstack._internal.server.services.backends import check_backend_type_available +from dstack._internal.server.services.backends.cache import ( lease_project_backend_with_model_by_type_or_error, ) from dstack._internal.server.services.gateways.connection import GatewayConnection diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index 6f6a575b9b..2f078f07fc 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -45,7 +45,7 @@ async def get_offers_by_requirements( ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]: """ Returns offers of `backends` matching the requirements. The caller must lease `backends` - (see `backends_services.lease_project_backends()`) and must not use the returned backends + (see `backends_cache.lease_project_backends()`) and must not use the returned backends after the lease is released. """ backend_types: Optional[list[BackendType]] = profile.backends diff --git a/src/dstack/_internal/server/services/projects.py b/src/dstack/_internal/server/services/projects.py index afee5796a6..452ff93c2c 100644 --- a/src/dstack/_internal/server/services/projects.py +++ b/src/dstack/_internal/server/services/projects.py @@ -41,7 +41,7 @@ from dstack._internal.server.schemas.projects import MemberSetting from dstack._internal.server.services import events, users from dstack._internal.server.services import templates as templates_service -from dstack._internal.server.services.backends import ( +from dstack._internal.server.services.backends.store import ( get_backend_config_without_creds_from_backend_model, ) from dstack._internal.server.services.locking import ( diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index 02aeada741..f79a37213e 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -36,7 +36,7 @@ ProjectModel, RunModel, ) -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.fleets import ( check_can_create_new_cloud_instance_in_fleet, get_fleet_master_instance_provisioning_data, @@ -798,7 +798,7 @@ async def _get_backend_offers_in_fleet( # Handle multinode for old jobs that don't have requirements.multinode set. # TODO: Drop multinode param. multinode = requirements.multinode or is_multinode_job(job) - async with backends_services.lease_project_backends(project) as backends: + async with backends_cache.lease_project_backends(project) as backends: backend_offers = await get_offers_by_requirements( backends=backends, profile=profile, @@ -879,7 +879,7 @@ async def get_non_fleet_offers( if skip_backend_offers: backend_offers = [] else: - async with backends_services.lease_project_backends(project) as backends: + async with backends_cache.lease_project_backends(project) as backends: backend_offers = await get_offers_by_requirements( backends=backends, profile=run_spec.merged_profile, diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index 262775a459..20af3ded26 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -38,6 +38,7 @@ ) from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.locking import ( get_locker, @@ -517,7 +518,7 @@ async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_mo ) return - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=volume_model.project, backend_type=volume.provisioning_data.backend, ) as backend: diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py index 695df5a20e..e89b0c07af 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py @@ -87,7 +87,7 @@ async def test_creates_instance( total_blocks=requested_blocks, busy_blocks=0, ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: backend_mock = Mock() m.side_effect = lease_yielding([backend_mock]) backend_mock.TYPE = BackendType.AWS @@ -162,7 +162,7 @@ async def test_tries_second_offer_if_first_fails( region=offer.region, price=offer.price, ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([aws_mock, gcp_mock]) await process_instance(session, worker, instance) @@ -191,7 +191,7 @@ async def test_fails_if_all_offers_fail( aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.get_offers.return_value = [offer] aws_mock.compute.return_value.create_instance.side_effect = err - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([aws_mock]) await process_instance(session, worker, instance) @@ -211,7 +211,7 @@ async def test_fails_if_no_offers( project=project, status=InstanceStatus.PENDING, ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([]) await process_instance(session, worker, instance) @@ -249,7 +249,7 @@ async def test_waits_when_fleet_has_no_current_master( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) @@ -297,7 +297,7 @@ async def test_waits_for_current_master_to_determine_cluster_placement( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) @@ -350,7 +350,7 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi master_item = instance_to_pipeline_item(master_instance) sibling_item = instance_to_pipeline_item(sibling_instance) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([]) await worker.process(master_item) @@ -384,7 +384,7 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([gcp_mock, aws_mock]) await worker.process(sibling_item) @@ -410,7 +410,7 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([promoted_backend_mock]) await process_instance(session, worker, sibling_instance) @@ -479,7 +479,7 @@ async def test_follows_current_master_backend_and_region_constraints( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([gcp_mock, aws_mock]) await process_instance(session, worker, sibling_instance) @@ -541,7 +541,7 @@ async def test_non_master_does_not_create_new_placement_group_without_master_pg( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) @@ -606,7 +606,7 @@ async def test_non_master_reuses_existing_current_master_placement_group( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, sibling_instance) @@ -678,7 +678,7 @@ async def test_allows_parallel_processing_after_master_is_provisioned( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, later_instance) assert backend_mock.compute.return_value.create_instance.call_count == 1 @@ -737,7 +737,7 @@ async def test_create_placement_group_if_placement_cluster( backend_mock.compute.return_value.create_placement_group.return_value = ( get_placement_group_provisioning_data() ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) @@ -791,7 +791,7 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.create_instance.return_value = get_job_provisioning_data() compute_mock.create_placement_group.return_value = get_placement_group_provisioning_data() - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) @@ -860,7 +860,7 @@ def create_instance_method( get_placement_group_provisioning_data() ) backend_mock.compute.return_value.is_suitable_placement_group.return_value = can_reuse - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) @@ -919,7 +919,7 @@ async def test_master_reuses_existing_placement_group( backend_mock.compute.return_value.create_instance.return_value = ( get_job_provisioning_data() ) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, master_instance) @@ -980,7 +980,7 @@ def create_placement_group_method( raise err backend_mock.compute.return_value.create_placement_group = create_placement_group_method - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: m.side_effect = lease_yielding([backend_mock]) await process_instance(session, worker, instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py index 2e5ed45f37..17394f3806 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py @@ -42,7 +42,7 @@ def mock_terminate_in_backend(error: Optional[Exception] = None): if error is not None: terminate_instance.side_effect = error with patch.object( - instances_termination.backends_services, + instances_termination.backends_cache, "lease_project_backend_by_type", mock_lease(backend), ): diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py index 441e5df909..38f318022c 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py @@ -165,7 +165,9 @@ async def test_terminates_compute_group( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock @@ -191,7 +193,9 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock @@ -211,7 +215,9 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: backend_mock = Mock() compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index d2b8e55252..4df3565949 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -414,7 +414,7 @@ async def test_submitted_to_provisioning( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: aws = Mock() m.side_effect = lease_yielding((backend, aws)) @@ -460,7 +460,7 @@ async def test_submitted_backend_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: aws = Mock() m.side_effect = lease_yielding((backend, aws)) @@ -500,7 +500,7 @@ async def test_submitted_backend_not_available_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: m.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) @@ -536,7 +536,7 @@ async def test_submitted_skips_provisioning_if_gateway_to_be_deleted( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -571,7 +571,7 @@ async def test_submitted_skips_provisioning_if_gateway_failed( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -606,7 +606,7 @@ async def test_submitted_unexpected_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: aws = Mock() m.side_effect = lease_yielding((backend, aws)) @@ -648,7 +648,7 @@ async def test_submitted_to_terminated_when_scaled_in( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -2214,7 +2214,7 @@ async def test_provisioning_to_running_registers_with_load_balancer( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2263,7 +2263,7 @@ async def test_provisioning_skips_load_balancer_registration_without_hostname( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2305,7 +2305,7 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2352,7 +2352,7 @@ async def test_provisioning_to_terminating_when_backend_does_not_support_load_ba "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2591,7 +2591,7 @@ async def test_terminating_to_terminated( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2669,7 +2669,7 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2720,7 +2720,7 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2773,7 +2773,7 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2824,7 +2824,7 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2868,7 +2868,7 @@ async def test_terminating_waits_for_pending_acm_gateway_migration( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -2916,7 +2916,7 @@ async def test_terminating_to_terminated_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock: get_backends_mock.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) @@ -2961,7 +2961,7 @@ async def test_terminating_to_terminated_with_no_instance_id( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -3015,7 +3015,7 @@ async def test_terminating_retries_if_terminate_fails( with ( patch( - "dstack._internal.server.services.backends.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py index 9105fcb5a0..6fddad65f4 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py @@ -536,7 +536,7 @@ async def test_submitted_to_provisioning_creates_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -576,7 +576,7 @@ async def test_submitted_skips_load_balancer_creation_for_non_acm_gateway( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -603,7 +603,7 @@ async def test_submitted_to_failed_when_backend_does_not_support_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -644,7 +644,7 @@ async def test_submitted_to_failed_when_load_balancer_creation_raises( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -1699,7 +1699,7 @@ async def test_deletes_gateway_and_terminates_load_balancer_when_hostname_set( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -1743,7 +1743,7 @@ async def test_delete_skips_load_balancer_termination_when_hostname_not_set( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -1779,7 +1779,7 @@ async def test_delete_deferred_when_load_balancer_termination_fails( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -1828,7 +1828,7 @@ async def test_delete_deferred_when_backend_does_not_support_load_balancer( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: backend_mock = Mock() backend_mock.TYPE = BackendType.AWS @@ -1868,7 +1868,7 @@ async def test_delete_migrates_hostname_before_evaluating_termination( await session.commit() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) # Migration happens before the load balancer would be evaluated for termination, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py index a42244bae2..f618280989 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py @@ -431,7 +431,7 @@ async def test_provisions_new_capacity_for_assigned_job_with_placeholder( offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -516,7 +516,7 @@ async def test_multinode_master_reuses_placeholder_when_provisioning_falls_back_ offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -615,7 +615,7 @@ async def test_provisioning_master_job_respects_cluster_placement_in_non_empty_f backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -681,7 +681,7 @@ async def test_defers_new_capacity_provisioning_until_fleet_master_is_elected( ) previous_last_processed_at = job.last_processed_at - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -741,7 +741,7 @@ async def test_provisioning_non_master_job_ignores_cluster_master_fleet_lock( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -794,7 +794,7 @@ async def test_creates_placement_group_for_cluster_fleet( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -849,7 +849,7 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -920,7 +920,7 @@ async def test_marks_unused_existing_placement_groups_for_cleanup( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -980,7 +980,7 @@ async def test_marks_new_and_existing_placement_groups_for_cleanup_on_failed_pro backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -1036,7 +1036,7 @@ async def test_resets_lock_for_retry_when_cluster_master_fleet_lock_is_unavailab previous_last_processed_at = job.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -1077,7 +1077,7 @@ async def test_reclaims_stale_related_cluster_master_fleet_lock( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -1875,7 +1875,7 @@ async def test_assignment_creates_placeholder_instance_for_new_capacity( offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -1953,7 +1953,7 @@ async def test_terminates_job_when_no_reusable_instances_with_reuse_creation_pol ) job = await create_job(session=session, run=run) - with patch("dstack._internal.server.services.backends.lease_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) # Backend offers must not be requested with the reuse policy. @@ -1997,7 +1997,7 @@ async def test_job_fails_when_fleet_is_full( offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -2052,7 +2052,7 @@ async def test_leaves_placeholder_for_terminating_pipeline_on_failed_new_capacit offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -2085,7 +2085,7 @@ async def test_reports_tried_offers_when_new_capacity_provisioning_fails( offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us-east-1") backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -2140,7 +2140,7 @@ async def test_provisions_compute_group( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -2212,7 +2212,7 @@ async def test_provisions_one_job_node_group_via_run_job( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -2490,7 +2490,7 @@ async def test_attaches_volume_on_existing_instance( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS @@ -2563,7 +2563,7 @@ async def test_terminates_job_when_volume_is_locked_for_processing( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS @@ -2625,7 +2625,7 @@ async def test_reclaims_stale_related_volume_lock( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS @@ -2678,7 +2678,7 @@ async def test_run_job_uses_server_default_registry( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.RUNPOD @@ -2745,7 +2745,7 @@ async def test_run_jobs_uses_server_default_registry( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): compute_mock = Mock(spec=ComputeMockSpec) @@ -2814,7 +2814,7 @@ async def test_interpolates_secrets_when_provisioning_new_capacity( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.RUNPOD @@ -2874,7 +2874,7 @@ async def test_terminates_job_when_secret_is_missing( offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.RUNPOD diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index d0360c1913..6874d523ca 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -811,7 +811,7 @@ async def test_detaches_job_volumes( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" ) as m: backend_mock = Mock() m.side_effect = lease_yielding(backend_mock) @@ -862,7 +862,7 @@ async def test_force_detaches_job_volumes( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" ) as m: backend_mock = Mock() m.side_effect = lease_yielding(backend_mock) @@ -881,7 +881,7 @@ async def test_force_detaches_job_volumes( await session.commit() with ( patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" ) as m, patch( "dstack._internal.server.background.pipeline_tasks.jobs_terminating.get_current_datetime" @@ -910,7 +910,7 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" ) as m: backend_mock = Mock() m.side_effect = lease_yielding(backend_mock) @@ -1018,7 +1018,7 @@ async def test_detaches_job_volumes_on_shared_instance( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" ) as m: backend_mock = Mock() m.side_effect = lease_yielding(backend_mock) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py index 72a05d8cc9..1882a28afc 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py @@ -404,7 +404,7 @@ def _patch_compute(compute: _ListingCompute): backend = Mock() backend.compute.return_value = compute return patch.object( - orphan_cleanup_pipeline.backends_services, + orphan_cleanup_pipeline.backends_cache, "lease_project_backend_by_type", mock_lease(backend), ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py index 85deeb37c0..3fad8533d8 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py @@ -197,7 +197,9 @@ async def test_deletes_placement_group( placement_group.lock_token = uuid.uuid4() placement_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: aws_mock = Mock() m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -226,7 +228,9 @@ async def test_retries_placement_group_deletion_if_still_in_use( placement_group.lock_owner = "PlacementGroupPipeline" original_last_processed_at = placement_group.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: aws_mock = Mock() m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py index 1dc511c4b3..fe15bd18da 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py @@ -216,7 +216,7 @@ async def test_submitted_to_active(self, test_db, session: AsyncSession, worker: await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -257,7 +257,7 @@ async def test_registers_external_volume( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -298,7 +298,7 @@ async def test_marks_volume_failed_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) @@ -330,7 +330,7 @@ async def test_marks_volume_failed_if_backend_returns_error( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -439,7 +439,7 @@ async def test_marks_volume_deleted( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -475,7 +475,7 @@ async def test_marks_external_volume_deleted_without_backend_call( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_not_called() @@ -505,7 +505,7 @@ async def test_marks_volume_deleted_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) @@ -536,7 +536,7 @@ async def test_marks_volume_deleted_if_backend_delete_errors( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.lease_project_backend_by_type" + "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" ) as get_backend_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py index 78ec75d8c4..a31ca3a870 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py @@ -65,7 +65,9 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): ) await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: aws_mock = Mock() m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -109,7 +111,9 @@ async def test_deletes_idle_volume_with_null_auto_cleanup_enabled( volume.auto_cleanup_enabled = None await session.commit() - with patch("dstack._internal.server.services.backends.lease_project_backend_by_type") as m: + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + ) as m: aws_mock = Mock() m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index 2d702e808c..f21c27f153 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -2251,7 +2251,7 @@ async def test_returns_create_plan_for_new_fleet( spec = get_fleet_spec() backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -2296,7 +2296,7 @@ async def test_returns_offers_for_elastic_container_backend_fleet( ) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.RUNPOD @@ -2334,7 +2334,7 @@ async def test_returns_no_offers_for_non_elastic_container_backend_fleet( ) backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.RUNPOD diff --git a/src/tests/_internal/server/routers/test_gpus.py b/src/tests/_internal/server/routers/test_gpus.py index 70f4305bb0..8847b8031a 100644 --- a/src/tests/_internal/server/routers/test_gpus.py +++ b/src/tests/_internal/server/routers/test_gpus.py @@ -484,7 +484,7 @@ async def test_price_order( ), ] with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(create_mock_backends_with_offers({BackendType.AWS: offers})), ): response = await call_gpus_api( @@ -557,7 +557,7 @@ async def test_cpu_groups( {BackendType.AWS: aws, BackendType.AZURE: [unavailable]} ) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api( @@ -632,7 +632,7 @@ async def test_returns_gpus_without_group_by( mocked_backends = create_mock_backends_with_offers(offers_by_backend) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api(client, project.name, user.token, run_spec) @@ -666,7 +666,7 @@ async def test_forwards_full_offers_to_compute_get_offers( mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api( @@ -705,7 +705,7 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api( @@ -776,7 +776,7 @@ async def test_filters_gpus_by_multiple_specified_fleets( mocked_backends = create_mock_backends_with_offers(offers_by_backend) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api( @@ -806,7 +806,7 @@ async def test_includes_backend_offers_when_creation_policy_reuse_or_create( mocked_backends = create_mock_backends_with_offers(offers_by_backend) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) @@ -830,7 +830,7 @@ async def test_skips_backend_offers_when_creation_policy_reuse( mocked_backends = create_mock_backends_with_offers(offers_by_backend) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): # reuse skips backend offers, keeping only the existing instance. @@ -869,7 +869,7 @@ async def test_uses_targeted_instance_offers_when_instances_specified( mocked_backends = create_mock_backends_with_offers(offers_by_backend) with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease(mocked_backends), ): response = await call_gpus_api(client, project.name, user.token, run_spec) @@ -893,7 +893,7 @@ async def test_returns_empty_gpus_when_no_offers( backend_mock_aws = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -1045,7 +1045,7 @@ async def test_exact_aggregation_values( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 5ffb2587d2..ef888668a0 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -1583,7 +1583,7 @@ async def test_sorts_offers_before_limit( if reverse_offers is not None: body["reverse_offers"] = reverse_offers with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend]), ): response = await client.post( @@ -1661,7 +1661,7 @@ async def test_returns_run_plan_privileged_false( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -1713,7 +1713,7 @@ async def test_forwards_full_offers_to_compute_get_offers( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -1770,7 +1770,7 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -1840,7 +1840,7 @@ async def test_returns_run_plan_privileged_true( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -1887,7 +1887,7 @@ async def test_task_with_two_nodes_returns_two_job_plans( body = {"run_spec": json.loads(run_spec.model_dump_json())} backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -1980,7 +1980,7 @@ def offers_by_requirements( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock]), ): backend_mock.TYPE = BackendType.AWS @@ -2033,7 +2033,7 @@ async def test_service_reservation_group_filters_backends_by_reservation_support aws_backend_mock = Mock() verda_backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([aws_backend_mock, verda_backend_mock]), ): aws_backend_mock.TYPE = BackendType.AWS @@ -2111,7 +2111,7 @@ async def test_returns_run_plan_docker_true( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -2176,7 +2176,7 @@ async def test_returns_run_plan_instance_volumes( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -2504,7 +2504,7 @@ async def test_preserves_backend_specific_offer_order( ] with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_vastai]), ): response = await client.post( @@ -2577,7 +2577,7 @@ async def test_offer_cli_preserves_backend_specific_offer_order_across_fleets( ] with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_vastai]), ): response = await client.post( @@ -2714,7 +2714,7 @@ async def test_offer_cli_deduplicates_identical_backend_offers_across_specified_ backend_mock_aws = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws]), ): backend_mock_aws.TYPE = BackendType.AWS @@ -2838,7 +2838,7 @@ async def test_offer_cli_without_fleet_keeps_global_offers( backend_mock_aws = Mock() backend_mock_runpod = Mock() with patch( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend_mock_aws, backend_mock_runpod]), ): backend_mock_aws.TYPE = BackendType.AWS diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index dbbfc3eb31..f3513d9137 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -10,7 +10,7 @@ from dstack._internal.core.errors import BackendNotAvailable from dstack._internal.core.models.backends.base import BackendType from dstack._internal.server.models import BackendModel, ProjectModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache def make_backend_model(backend_type: BackendType, config: str = "{}") -> BackendModel: @@ -35,9 +35,9 @@ def close_mock(backend: Backend) -> Mock: class TestBackendsCache: @pytest.fixture(autouse=True) def _setup(self, monkeypatch): - monkeypatch.setattr(backends_services, "_BACKENDS_CACHE", {}) - monkeypatch.setattr(backends_services, "_BACKENDS_CACHE_LOCKS", {}) - monkeypatch.setattr(backends_services, "get_stored_backend_record", lambda m: m) + monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE", {}) + monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE_LOCKS", {}) + monkeypatch.setattr(backends_cache, "get_stored_backend_record", lambda m: m) self.backends: list[Mock] = [] def get_backend(backend_model): @@ -46,7 +46,7 @@ def get_backend(backend_model): return backend monkeypatch.setattr( - backends_services, + backends_cache, "get_configurator", lambda backend_type: Mock(get_backend=Mock(side_effect=get_backend)), ) @@ -54,9 +54,9 @@ def get_backend(backend_model): @pytest.mark.asyncio async def test_reuses_backends_between_leases(self): project = make_project(make_backend_model(BackendType.AWS)) - async with backends_services.lease_project_backends(project) as first: + async with backends_cache.lease_project_backends(project) as first: pass - async with backends_services.lease_project_backends(project) as second: + async with backends_cache.lease_project_backends(project) as second: pass assert first == second == self.backends self.backends[0].close.assert_not_called() @@ -64,13 +64,13 @@ async def test_reuses_backends_between_leases(self): @pytest.mark.asyncio async def test_closes_backend_with_changed_config_after_release(self): project = make_project(make_backend_model(BackendType.AWS, config="old")) - async with backends_services.lease_project_backends(project) as [old_backend]: + async with backends_cache.lease_project_backends(project) as [old_backend]: project.backends = [make_backend_model(BackendType.AWS, config="new")] - async with backends_services.lease_project_backends(project) as [new_backend]: + async with backends_cache.lease_project_backends(project) as [new_backend]: assert new_backend is not old_backend - await backends_services.close_unused_backends() + await backends_cache.close_unused_backends() close_mock(old_backend).assert_not_called() - await backends_services.close_unused_backends() + await backends_cache.close_unused_backends() close_mock(old_backend).assert_called_once() close_mock(new_backend).assert_not_called() @@ -79,13 +79,13 @@ async def test_closes_deleted_backend(self): project = make_project( make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) ) - async with backends_services.lease_project_backends(project): + async with backends_cache.lease_project_backends(project): pass aws_backend, gcp_backend = self.backends project.backends = project.backends[:1] - async with backends_services.lease_project_backends(project) as backends: + async with backends_cache.lease_project_backends(project) as backends: assert backends == [aws_backend] - await backends_services.close_unused_backends() + await backends_cache.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @@ -94,16 +94,16 @@ async def test_closes_idle_backends(self, monkeypatch): project = make_project( make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) ) - async with backends_services.lease_project_backends(project): + async with backends_cache.lease_project_backends(project): pass aws_backend, gcp_backend = self.backends - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS ) as backend: assert backend is aws_backend # Only GCP is idle: AWS is leased and was used just now - monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) - await backends_services.close_unused_backends() + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", 0) + await backends_cache.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @@ -112,24 +112,24 @@ async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypat project = make_project( make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) ) - async with backends_services.lease_project_backends(project): + async with backends_cache.lease_project_backends(project): pass aws_backend, gcp_backend = self.backends - monkeypatch.setattr(backends_services, "_BACKEND_IDLE_TTL", 0) - async with backends_services.lease_project_backend_by_type(project, BackendType.AWS): - await backends_services.close_unused_backends() + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", 0) + async with backends_cache.lease_project_backend_by_type(project, BackendType.AWS): + await backends_cache.close_unused_backends() close_mock(gcp_backend).assert_called_once() close_mock(aws_backend).assert_not_called() @pytest.mark.asyncio async def test_close_all_backends_does_not_close_leased_backends(self): project = make_project(make_backend_model(BackendType.AWS)) - async with backends_services.lease_project_backends(project) as [backend]: - await backends_services.close_all_backends() + async with backends_cache.lease_project_backends(project) as [backend]: + await backends_cache.close_all_backends() close_mock(backend).assert_not_called() - async with backends_services.lease_project_backends(project) as [new_backend]: + async with backends_cache.lease_project_backends(project) as [new_backend]: assert new_backend is not backend - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(backend).assert_called_once() close_mock(new_backend).assert_called_once() @@ -139,7 +139,7 @@ async def test_releases_lease_if_cancelled(self): leased = asyncio.Event() async def hold_lease(): - async with backends_services.lease_project_backends(project): + async with backends_cache.lease_project_backends(project): leased.set() await asyncio.Event().wait() @@ -148,27 +148,27 @@ async def hold_lease(): task.cancel() with pytest.raises(asyncio.CancelledError): await task - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(self.backends[0]).assert_called_once() @pytest.mark.asyncio async def test_does_not_raise_if_backend_fails_to_close(self): project = make_project(make_backend_model(BackendType.AWS)) - async with backends_services.lease_project_backends(project) as [backend]: + async with backends_cache.lease_project_backends(project) as [backend]: close_mock(backend).side_effect = RuntimeError("close failed") - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(backend).assert_called_once() @pytest.mark.asyncio async def test_or_error_raises_and_releases_leases(self): project = make_project(make_backend_model(BackendType.AWS)) with pytest.raises(BackendNotAvailable): - async with backends_services.lease_project_backend_by_type_or_error( + async with backends_cache.lease_project_backend_by_type_or_error( project, BackendType.GCP ): pass [aws_backend] = self.backends - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(aws_backend).assert_called_once() @pytest.mark.asyncio @@ -178,14 +178,12 @@ async def test_uses_selector_with_overrides(self, monkeypatch): async def select(project, backend_type, backends_with_models): return backends_with_models[0] - monkeypatch.setattr( - backends_services, "_select_project_backend_with_model_by_type", select - ) - async with backends_services.lease_project_backend_by_type( + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) + async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS, overrides=True ) as backend: assert backend is self.backends[0] - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS ) as backend: assert backend is None @@ -197,15 +195,13 @@ async def test_releases_leases_if_selector_fails(self, monkeypatch): async def select(project, backend_type, backends_with_models): raise RuntimeError("select failed") - monkeypatch.setattr( - backends_services, "_select_project_backend_with_model_by_type", select - ) + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) with pytest.raises(RuntimeError): - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS, overrides=True ): pass - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(self.backends[0]).assert_called_once() @pytest.mark.asyncio @@ -217,13 +213,11 @@ async def test_raises_and_releases_leases_if_selector_returns_unknown_backend( async def select(project, backend_type, backends_with_models): return backends_with_models[0][0], Mock() - monkeypatch.setattr( - backends_services, "_select_project_backend_with_model_by_type", select - ) + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) with pytest.raises(ValueError): - async with backends_services.lease_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS, overrides=True ): pass - await backends_services.close_all_backends() + await backends_cache.close_all_backends() close_mock(self.backends[0]).assert_called_once() diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index 3222bddb1b..e4a91a9b17 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -841,7 +841,7 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el offer = get_instance_offer_with_availability() backend = AsyncMock() monkeypatch.setattr( - "dstack._internal.server.services.backends.lease_project_backends", + "dstack._internal.server.services.backends.cache.lease_project_backends", mock_lease([backend]), ) get_offers_by_requirements_mock.return_value = [(backend, offer)] diff --git a/src/tests/_internal/server/services/test_fleets.py b/src/tests/_internal/server/services/test_fleets.py index 1663a68964..ce207d2421 100644 --- a/src/tests/_internal/server/services/test_fleets.py +++ b/src/tests/_internal/server/services/test_fleets.py @@ -37,7 +37,7 @@ class TestGetPlanSSHFleetHostsValidation: def lease_project_backends_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: mock = mock_lease([]) monkeypatch.setattr( - "dstack._internal.server.services.backends.lease_project_backends", mock + "dstack._internal.server.services.backends.cache.lease_project_backends", mock ) return mock From 3ae28e2acf81ac59a3c814ea3ff83125229483f0 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:11:44 +0500 Subject: [PATCH 08/19] Move backend persistence to store.py and backend offers to services/offers.py --- .../_internal/core/backends/configurators.py | 10 + .../_internal/server/routers/backends.py | 8 +- .../server/services/backends/__init__.py | 234 ------------------ .../server/services/backends/handlers.py | 2 +- .../server/services/backends/store.py | 145 ++++++++++- .../_internal/server/services/config.py | 18 +- .../server/services/gateways/__init__.py | 2 +- .../_internal/server/services/offers.py | 65 ++++- .../_internal/server/services/volumes.py | 4 +- .../_internal/server/services/test_config.py | 8 +- 10 files changed, 237 insertions(+), 259 deletions(-) diff --git a/src/dstack/_internal/core/backends/configurators.py b/src/dstack/_internal/core/backends/configurators.py index a277dd9a8a..3973687c6e 100644 --- a/src/dstack/_internal/core/backends/configurators.py +++ b/src/dstack/_internal/core/backends/configurators.py @@ -1,6 +1,7 @@ from typing import List, Optional, Type, Union from dstack._internal.core.backends.base.configurator import Configurator +from dstack._internal.core.errors import BackendNotAvailable from dstack._internal.core.models.backends.base import BackendType _CONFIGURATOR_CLASSES: List[Type[Configurator]] = [] @@ -223,3 +224,12 @@ def register_configurator(configurator: Type[Configurator]): Can be used to extend dstack functionality. """ _BACKEND_TYPE_TO_CONFIGURATOR_CLASS_MAP[configurator.TYPE] = configurator + + +def check_backend_type_available(backend_type: BackendType): + if backend_type not in list_available_backend_types(): + raise BackendNotAvailable( + f"Backend {backend_type.value} not available." + " Ensure that backend dependencies are installed." + f" Available backends: {[b.value for b in list_available_backend_types()]}." + ) diff --git a/src/dstack/_internal/server/routers/backends.py b/src/dstack/_internal/server/routers/backends.py index 8c79993269..36e8e318f3 100644 --- a/src/dstack/_internal/server/routers/backends.py +++ b/src/dstack/_internal/server/routers/backends.py @@ -19,8 +19,8 @@ UpdateBackendYAMLRequest, ) from dstack._internal.server.security.permissions import ProjectAdmin -from dstack._internal.server.services import backends from dstack._internal.server.services.backends import handlers as backends_handlers +from dstack._internal.server.services.backends import store as backends_store from dstack._internal.server.services.config import ( ServerConfigManager, create_backend_config_yaml, @@ -58,7 +58,7 @@ async def create_backend( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.create_backend(session=session, project=project, config=body) + config = await backends_store.create_backend(session=session, project=project, config=body) if settings.SERVER_CONFIG_ENABLED: await ServerConfigManager().sync_config(session=session) return CustomJSONResponse(config) @@ -71,7 +71,7 @@ async def update_backend( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.update_backend(session=session, project=project, config=body) + config = await backends_store.update_backend(session=session, project=project, config=body) if settings.SERVER_CONFIG_ENABLED: await ServerConfigManager().sync_config(session=session) return CustomJSONResponse(config) @@ -101,7 +101,7 @@ async def get_backend_config_info( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.get_backend_config(project=project, backend_type=backend_name) + config = await backends_store.get_backend_config(project=project, backend_type=backend_name) if config is None: raise ResourceNotExistsError() return CustomJSONResponse(config) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 25fb58b4b7..e69de29bb2 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,234 +0,0 @@ -import asyncio -import time -from collections.abc import Iterable, Iterator -from typing import List, Optional, Tuple - -from sqlalchemy import delete, update -from sqlalchemy.ext.asyncio import AsyncSession - -from dstack._internal.core.backends.base.backend import Backend -from dstack._internal.core.backends.base.configurator import Configurator -from dstack._internal.core.backends.configurators import ( - get_configurator, - list_available_backend_types, -) -from dstack._internal.core.backends.models import AnyBackendConfigWithCreds -from dstack._internal.core.errors import ( - BackendError, - BackendNotAvailable, - ResourceExistsError, - ResourceNotExistsError, - ServerClientError, -) -from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.instances import ( - InstanceOfferWithAvailability, -) -from dstack._internal.core.models.runs import Requirements -from dstack._internal.server import settings -from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel -from dstack._internal.server.services.backends.store import ( - get_backend_config_with_creds_from_backend_model, - get_project_backend_model_by_type, - get_source_backend_config_from_backend_model, - serialize_source_backend_config, -) -from dstack._internal.server.services.offers import merge_offer_iterables -from dstack._internal.utils.common import run_async -from dstack._internal.utils.logging import get_logger - -logger = get_logger(__name__) - - -async def create_backend( - session: AsyncSession, - project: ProjectModel, - config: AnyBackendConfigWithCreds, -) -> AnyBackendConfigWithCreds: - configurator = get_configurator(config.type) - if configurator is None: - raise BackendNotAvailable() - backend_exists = any(configurator.TYPE == b.type for b in project.backends) - if backend_exists: - raise ResourceExistsError() - backend = await validate_and_create_backend_model( - project=project, configurator=configurator, config=config - ) - session.add(backend) - await session.commit() - return config - - -async def update_backend( - session: AsyncSession, - project: ProjectModel, - config: AnyBackendConfigWithCreds, -) -> AnyBackendConfigWithCreds: - configurator = get_configurator(config.type) - if configurator is None: - raise BackendNotAvailable() - backend_exists = any(configurator.TYPE == b.type for b in project.backends) - if not backend_exists: - raise ResourceNotExistsError() - backend = await validate_and_create_backend_model( - project=project, configurator=configurator, config=config - ) - # FIXME: potentially long write transaction - await session.execute( - update(BackendModel) - .where( - BackendModel.project_id == backend.project_id, - BackendModel.type == backend.type, - ) - .values( - config=backend.config, - auth=backend.auth, - source_config=backend.source_config, - source_auth=backend.source_auth, - ) - ) - return config - - -async def validate_and_create_backend_model( - project: ProjectModel, - configurator: Configurator, - config: AnyBackendConfigWithCreds, -) -> BackendModel: - # Configurators may mutate `config` while building the effective stored backend config, - # so capture the user-intent payload before validation/create_backend runs. - source_config, source_auth = serialize_source_backend_config(config) - await run_async( - configurator.validate_config, config, default_creds_enabled=settings.DEFAULT_CREDS_ENABLED - ) - backend_record = await run_async( - configurator.create_backend, - project_name=project.name, - config=config, - ) - return BackendModel( - project_id=project.id, - type=configurator.TYPE, - config=backend_record.config, - auth=DecryptedString(plaintext=backend_record.auth), - source_config=source_config, - source_auth=None if source_auth is None else DecryptedString(plaintext=source_auth), - ) - - -async def get_backend_config( - project: ProjectModel, - backend_type: BackendType, -) -> Optional[AnyBackendConfigWithCreds]: - configurator = get_configurator(backend_type) - if configurator is None: - raise BackendNotAvailable() - for backend_model in project.backends: - if not backend_model.auth.decrypted: - logger.warning( - "Failed to decrypt creds for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - if backend_model.type == backend_type: - return get_backend_config_with_creds_from_backend_model(configurator, backend_model) - return None - - -async def get_source_backend_config( - project: ProjectModel, - backend_type: BackendType, -) -> Optional[AnyBackendConfigWithCreds]: - backend_model = await get_project_backend_model_by_type(project, backend_type) - if backend_model is None: - return None - return get_source_backend_config_from_backend_model(backend_model) - - -async def delete_backends( - session: AsyncSession, - project: ProjectModel, - backends_types: List[BackendType], -): - if BackendType.DSTACK in backends_types: - raise ServerClientError("Cannot delete dstack backend") - current_backends_types = set(b.type for b in project.backends) - deleted_backends_types = current_backends_types.intersection(backends_types) - if len(deleted_backends_types) == 0: - return - # FIXME: potentially long write transaction - # Not urgent since backend deletion is a rare operation - await session.execute( - delete(BackendModel).where( - BackendModel.type.in_(deleted_backends_types), - BackendModel.project_id == project.id, - ) - ) - logger.info( - "Deleted backends %s in project %s", - [b.value for b in deleted_backends_types], - project.name, - ) - - -async def get_backend_offers( - backends: List[Backend], - requirements: Requirements, - full_offers: bool, - unallocated_resources: bool, - exclude_not_available: bool = False, -) -> Iterable[Tuple[Backend, InstanceOfferWithAvailability]]: - """ - Returns a lazy iterable of backend offers satisfying `requirements` sorted by price. - """ - - def get_filtered_offers_with_backends( - backend: Backend, - offers: Iterable[InstanceOfferWithAvailability], - ) -> Iterator[Tuple[Backend, InstanceOfferWithAvailability]]: - for offer in offers: - if not exclude_not_available or offer.availability.is_available(): - yield (backend, offer) - - logger.debug("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends]) - tasks = [ - run_async(get_offers_tracked, backend, requirements, full_offers, unallocated_resources) - for backend in backends - ] - offers_by_backend: list[Iterable[tuple[Backend, InstanceOfferWithAvailability]]] = [] - for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)): - if isinstance(result, BackendError): - logger.warning( - "Failed to get offers from backend %s: %s", - backend.TYPE, - repr(result), - ) - continue - elif isinstance(result, BaseException): - logger.error( - "Got exception when requesting offers from backend %s", - backend.TYPE, - exc_info=result, - ) - continue - offers_by_backend.append(get_filtered_offers_with_backends(backend, result)) - return merge_offer_iterables(*offers_by_backend) - - -def check_backend_type_available(backend_type: BackendType): - if backend_type not in list_available_backend_types(): - raise BackendNotAvailable( - f"Backend {backend_type.value} not available." - " Ensure that backend dependencies are installed." - f" Available backends: {[b.value for b in list_available_backend_types()]}." - ) - - -def get_offers_tracked( - backend: Backend, requirements: Requirements, full_offers: bool, unallocated_resources: bool -) -> Iterator[InstanceOfferWithAvailability]: - start = time.time() - res = backend.compute().get_offers(requirements, full_offers, unallocated_resources) - duration = time.time() - start - logger.debug("Got offers from %s in %.6fs", backend.TYPE.value, duration) - return res diff --git a/src/dstack/_internal/server/services/backends/handlers.py b/src/dstack/_internal/server/services/backends/handlers.py index f3f5bab68c..0c1778bba7 100644 --- a/src/dstack/_internal/server/services/backends/handlers.py +++ b/src/dstack/_internal/server/services/backends/handlers.py @@ -5,7 +5,7 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.server.models import ProjectModel -from dstack._internal.server.services.backends import delete_backends +from dstack._internal.server.services.backends.store import delete_backends from dstack._internal.server.services.fleets import list_project_fleet_models from dstack._internal.server.services.volumes import list_project_volumes from dstack._internal.utils.logging import get_logger diff --git a/src/dstack/_internal/server/services/backends/store.py b/src/dstack/_internal/server/services/backends/store.py index ba0d3a9039..9726401a4c 100644 --- a/src/dstack/_internal/server/services/backends/store.py +++ b/src/dstack/_internal/server/services/backends/store.py @@ -1,12 +1,17 @@ import json -from typing import Optional, Tuple +from typing import List, Optional, Tuple from pydantic import ValidationError +from sqlalchemy import delete, update +from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.backends.base.configurator import ( Configurator, StoredBackendRecord, ) +from dstack._internal.core.backends.configurators import ( + get_configurator, +) from dstack._internal.core.backends.models import ( AnyBackendConfigWithCreds, AnyBackendConfigWithoutCreds, @@ -14,9 +19,14 @@ ) from dstack._internal.core.errors import ( BackendNotAvailable, + ResourceExistsError, + ResourceNotExistsError, + ServerClientError, ) from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.server import settings +from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel +from dstack._internal.utils.common import run_async from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -119,3 +129,134 @@ async def get_project_backend_model_by_type_or_error( if backend_model is None: raise BackendNotAvailable() return backend_model + + +async def validate_and_create_backend_model( + project: ProjectModel, + configurator: Configurator, + config: AnyBackendConfigWithCreds, +) -> BackendModel: + # Configurators may mutate `config` while building the effective stored backend config, + # so capture the user-intent payload before validation/create_backend runs. + source_config, source_auth = serialize_source_backend_config(config) + await run_async( + configurator.validate_config, config, default_creds_enabled=settings.DEFAULT_CREDS_ENABLED + ) + backend_record = await run_async( + configurator.create_backend, + project_name=project.name, + config=config, + ) + return BackendModel( + project_id=project.id, + type=configurator.TYPE, + config=backend_record.config, + auth=DecryptedString(plaintext=backend_record.auth), + source_config=source_config, + source_auth=None if source_auth is None else DecryptedString(plaintext=source_auth), + ) + + +async def create_backend( + session: AsyncSession, + project: ProjectModel, + config: AnyBackendConfigWithCreds, +) -> AnyBackendConfigWithCreds: + configurator = get_configurator(config.type) + if configurator is None: + raise BackendNotAvailable() + backend_exists = any(configurator.TYPE == b.type for b in project.backends) + if backend_exists: + raise ResourceExistsError() + backend = await validate_and_create_backend_model( + project=project, configurator=configurator, config=config + ) + session.add(backend) + await session.commit() + return config + + +async def update_backend( + session: AsyncSession, + project: ProjectModel, + config: AnyBackendConfigWithCreds, +) -> AnyBackendConfigWithCreds: + configurator = get_configurator(config.type) + if configurator is None: + raise BackendNotAvailable() + backend_exists = any(configurator.TYPE == b.type for b in project.backends) + if not backend_exists: + raise ResourceNotExistsError() + backend = await validate_and_create_backend_model( + project=project, configurator=configurator, config=config + ) + # FIXME: potentially long write transaction + await session.execute( + update(BackendModel) + .where( + BackendModel.project_id == backend.project_id, + BackendModel.type == backend.type, + ) + .values( + config=backend.config, + auth=backend.auth, + source_config=backend.source_config, + source_auth=backend.source_auth, + ) + ) + return config + + +async def get_backend_config( + project: ProjectModel, + backend_type: BackendType, +) -> Optional[AnyBackendConfigWithCreds]: + configurator = get_configurator(backend_type) + if configurator is None: + raise BackendNotAvailable() + for backend_model in project.backends: + if not backend_model.auth.decrypted: + logger.warning( + "Failed to decrypt creds for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + if backend_model.type == backend_type: + return get_backend_config_with_creds_from_backend_model(configurator, backend_model) + return None + + +async def get_source_backend_config( + project: ProjectModel, + backend_type: BackendType, +) -> Optional[AnyBackendConfigWithCreds]: + backend_model = await get_project_backend_model_by_type(project, backend_type) + if backend_model is None: + return None + return get_source_backend_config_from_backend_model(backend_model) + + +async def delete_backends( + session: AsyncSession, + project: ProjectModel, + backends_types: List[BackendType], +): + if BackendType.DSTACK in backends_types: + raise ServerClientError("Cannot delete dstack backend") + current_backends_types = set(b.type for b in project.backends) + deleted_backends_types = current_backends_types.intersection(backends_types) + if len(deleted_backends_types) == 0: + return + # FIXME: potentially long write transaction + # Not urgent since backend deletion is a rare operation + await session.execute( + delete(BackendModel).where( + BackendModel.type.in_(deleted_backends_types), + BackendModel.project_id == project.id, + ) + ) + logger.info( + "Deleted backends %s in project %s", + [b.value for b in deleted_backends_types], + project.name, + ) diff --git a/src/dstack/_internal/server/services/config.py b/src/dstack/_internal/server/services/config.py index bba6e5d1fa..223523ffe0 100644 --- a/src/dstack/_internal/server/services/config.py +++ b/src/dstack/_internal/server/services/config.py @@ -21,9 +21,9 @@ from dstack._internal.core.models.common import CoreModel from dstack._internal.server import settings from dstack._internal.server.models import ProjectModel, UserModel -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import encryption as encryption_services from dstack._internal.server.services import projects as projects_services +from dstack._internal.server.services.backends import store as backends_store from dstack._internal.server.services.backends.handlers import delete_backends_safe from dstack._internal.server.services.encryption import AnyEncryptionKeyConfig from dstack._internal.server.services.permissions import ( @@ -145,7 +145,7 @@ async def _apply_project_config( backends_to_delete.difference_update([backend_type]) backend_exists = any(backend_type == b.type for b in project.backends) try: - current_backend_config = await backends_services.get_backend_config( + current_backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type, ) @@ -157,7 +157,7 @@ async def _apply_project_config( ) continue if current_backend_config is not None: - current_source_backend_config = await backends_services.get_source_backend_config( + current_source_backend_config = await backends_store.get_source_backend_config( project=project, backend_type=backend_type, ) @@ -170,10 +170,10 @@ async def _apply_project_config( # Update backend in this case. if current_backend_config is None and not backend_exists: apply_action = "create" - apply_func = backends_services.create_backend + apply_func = backends_store.create_backend else: apply_action = "update" - apply_func = backends_services.update_backend + apply_func = backends_store.update_backend try: await apply_func(session=session, project=project, config=backend_config) except Exception as e: @@ -204,7 +204,7 @@ async def _init_config(self, session: AsyncSession) -> Optional[ServerConfig]: for ( backend_type ) in dstack._internal.core.backends.configurators.list_available_backend_types(): - backend_config = await backends_services.get_backend_config( + backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type ) if backend_config is not None: @@ -232,7 +232,7 @@ def _save_config(self, config: ServerConfig): async def get_backend_config_yaml( project: ProjectModel, backend_type: BackendType ) -> BackendInfoYAML: - backend_config = await backends_services.get_backend_config( + backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type ) if backend_config is None: @@ -250,7 +250,7 @@ async def create_backend_config_yaml( config_yaml: str, ): config = config_yaml_to_backend_config(config_yaml) - await backends_services.create_backend(session=session, project=project, config=config) + await backends_store.create_backend(session=session, project=project, config=config) async def update_backend_config_yaml( @@ -259,7 +259,7 @@ async def update_backend_config_yaml( config_yaml: str, ): config = config_yaml_to_backend_config(config_yaml) - await backends_services.update_backend(session=session, project=project, config=config) + await backends_store.update_backend(session=session, project=project, config=config) def config_yaml_to_backend_config(config_yaml: str) -> AnyBackendConfigWithCreds: diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index d188f212df..55660e9228 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -17,6 +17,7 @@ from dstack._internal.core.backends.base.compute import ( get_dstack_gateway_install_command, ) +from dstack._internal.core.backends.configurators import check_backend_type_available from dstack._internal.core.backends.features import ( BACKENDS_WITH_GATEWAY_SUPPORT, BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT, @@ -64,7 +65,6 @@ UserModel, ) from dstack._internal.server.services import events -from dstack._internal.server.services.backends import check_backend_type_available from dstack._internal.server.services.backends.cache import ( lease_project_backend_with_model_by_type_or_error, ) diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index 2f078f07fc..13378cde30 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -1,5 +1,7 @@ +import asyncio import heapq import itertools +import time from collections.abc import Container, Iterable, Iterator from typing import List, Literal, Optional, Tuple, TypeVar, Union @@ -13,6 +15,9 @@ BACKENDS_WITH_PRIVILEGED_SUPPORT, BACKENDS_WITH_RESERVATION_SUPPORT, ) +from dstack._internal.core.errors import ( + BackendError, +) from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.instances import ( InstanceOfferWithAvailability, @@ -23,8 +28,10 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.runs import JobProvisioningData, Requirements from dstack._internal.core.models.volumes import Volume, VolumeConfigurationWithRegion -from dstack._internal.server.services import backends as backends_services from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) async def get_offers_by_requirements( @@ -101,7 +108,7 @@ async def get_offers_by_requirements( if backend_types is not None: backends = [b for b in backends if b.TYPE in backend_types or b.TYPE == BackendType.DSTACK] - offers = await backends_services.get_backend_offers( + offers = await get_backend_offers( backends=backends, requirements=requirements, full_offers=full_offers, @@ -291,3 +298,57 @@ def _get_shareable_offers( new_offer = offer.model_copy() new_offer.total_blocks = total_blocks yield (backend, new_offer) + + +async def get_backend_offers( + backends: List[Backend], + requirements: Requirements, + full_offers: bool, + unallocated_resources: bool, + exclude_not_available: bool = False, +) -> Iterable[Tuple[Backend, InstanceOfferWithAvailability]]: + """ + Returns a lazy iterable of backend offers satisfying `requirements` sorted by price. + """ + + def get_filtered_offers_with_backends( + backend: Backend, + offers: Iterable[InstanceOfferWithAvailability], + ) -> Iterator[Tuple[Backend, InstanceOfferWithAvailability]]: + for offer in offers: + if not exclude_not_available or offer.availability.is_available(): + yield (backend, offer) + + logger.debug("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends]) + tasks = [ + run_async(get_offers_tracked, backend, requirements, full_offers, unallocated_resources) + for backend in backends + ] + offers_by_backend: list[Iterable[tuple[Backend, InstanceOfferWithAvailability]]] = [] + for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)): + if isinstance(result, BackendError): + logger.warning( + "Failed to get offers from backend %s: %s", + backend.TYPE, + repr(result), + ) + continue + elif isinstance(result, BaseException): + logger.error( + "Got exception when requesting offers from backend %s", + backend.TYPE, + exc_info=result, + ) + continue + offers_by_backend.append(get_filtered_offers_with_backends(backend, result)) + return merge_offer_iterables(*offers_by_backend) + + +def get_offers_tracked( + backend: Backend, requirements: Requirements, full_offers: bool, unallocated_resources: bool +) -> Iterator[InstanceOfferWithAvailability]: + start = time.time() + res = backend.compute().get_offers(requirements, full_offers, unallocated_resources) + duration = time.time() - start + logger.debug("Got offers from %s in %.6fs", backend.TYPE.value, duration) + return res diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index 20af3ded26..d77d6f5ea1 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import joinedload, selectinload from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport +from dstack._internal.core.backends.configurators import check_backend_type_available from dstack._internal.core.backends.features import BACKENDS_WITH_VOLUMES_SUPPORT from dstack._internal.core.errors import ( ResourceExistsError, @@ -36,7 +37,6 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data @@ -481,7 +481,7 @@ def _validate_volume_configuration(configuration: AnyVolumeConfiguration): and configuration.size is None ): raise ServerClientError("Volume must specify either existing identifier or size") - backends_services.check_backend_type_available(configuration.backend) + check_backend_type_available(configuration.backend) if configuration.backend not in BACKENDS_WITH_VOLUMES_SUPPORT: raise ServerClientError( f"Volumes are not supported for {configuration.backend.value} backend." diff --git a/src/tests/_internal/server/services/test_config.py b/src/tests/_internal/server/services/test_config.py index bc07da3b57..b9869646fc 100644 --- a/src/tests/_internal/server/services/test_config.py +++ b/src/tests/_internal/server/services/test_config.py @@ -118,7 +118,7 @@ async def test_skips_update_when_source_config_matches( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): @@ -174,7 +174,7 @@ async def test_populates_source_config_for_legacy_backend( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): @@ -212,12 +212,12 @@ async def test_forces_update_when_current_backend_config_is_unavailable( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.get_backend_config", + "dstack._internal.server.services.backends.store.get_backend_config", new_callable=AsyncMock, return_value=None, ), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): From 1a63759053f40c71829b942cae738eb0dc69e412 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:11:53 +0500 Subject: [PATCH 09/19] Rename _close_retired_backends to _retire_and_close_backends --- src/dstack/_internal/server/services/backends/cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index 6dd77df312..fb6968a4e8 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -187,7 +187,7 @@ async def close_unused_backends(): and closes retired backends that are not leased. """ idle_since = time.monotonic() - _BACKEND_IDLE_TTL - await _close_retired_backends( + await _retire_and_close_backends( should_retire=lambda c: c.leases == 0 and c.last_used < idle_since ) @@ -196,10 +196,10 @@ async def close_all_backends(): """ Retires all backends and closes those that are not leased. """ - await _close_retired_backends(should_retire=lambda c: True) + await _retire_and_close_backends(should_retire=lambda c: True) -async def _close_retired_backends(should_retire: Callable[[_CachedBackend], bool]): +async def _retire_and_close_backends(should_retire: Callable[[_CachedBackend], bool]): to_close: List[_CachedBackend] = [] for project_id in list(_BACKENDS_CACHE): async with _get_project_cache_lock(project_id): From b930e295f92c88a73a478ae5adebf821435796e5 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:12:26 +0500 Subject: [PATCH 10/19] Explain the backends cache in a module docstring --- .../server/services/backends/cache.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index fb6968a4e8..1dc582ee15 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -1,3 +1,25 @@ +""" +Caches project backends and controls their lifetime. + +Backends may hold resources that garbage collection does not free, e.g. SDK clients that run +background threads (see `Compute.close()`). So backends are cached per project instead of +being created per use, and every backend is closed explicitly once it is no longer used. + +Callers use backends through leases (`lease_project_backend*()`) and must not use a backend +or its compute after the lease is released. This lets the cache know when a backend is unused: +a backend is never closed while it is leased. + +A cached backend is either active or retired. New leases use only active backends. +A backend is retired if its config changes (a new backend replaces it), if it is deleted +from the project, if it is not leased for `_BACKEND_IDLE_TTL`, or on server shutdown. +A retired backend may stay leased by operations that started before it was retired, and they +keep using it until they finish. + +Retired backends are closed by `close_unused_backends()`, which runs periodically, once they +are not leased. Leases never close backends: closing does not delay callers, and releasing +a lease does not await, so cancellation cannot interrupt it. +""" + import asyncio import time from collections.abc import AsyncGenerator @@ -49,11 +71,7 @@ class _ProjectBackends: _BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} _BACKENDS_CACHE: Dict[UUID, _ProjectBackends] = {} """ -A backend is cached per project and used through leases (`lease_project_backend*()`). -The cache retires a backend if its config changes, if it is deleted from the project, -or if it is not leased for `_BACKEND_IDLE_TTL`. New leases use only active backends. -`close_unused_backends()` closes retired backends once they are not leased, -so callers must not use backends outside of leases. +Active and retired backends by project ID. Retired backends stay here until they are closed. """ From e58cee085bf82f53c8847857c7f42b37af335d2f Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:13:20 +0500 Subject: [PATCH 11/19] Fix why backends are cached in the cache docstring --- src/dstack/_internal/server/services/backends/cache.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index 1dc582ee15..f80e93c14f 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -1,9 +1,10 @@ """ Caches project backends and controls their lifetime. -Backends may hold resources that garbage collection does not free, e.g. SDK clients that run -background threads (see `Compute.close()`). So backends are cached per project instead of -being created per use, and every backend is closed explicitly once it is no longer used. +Backends are cached per project because creating them is expensive, e.g. creating API clients +and filling offers caches. Backends may also hold resources that garbage collection does not +free, e.g. SDK clients that run background threads (see `Compute.close()`), so the cache +closes every backend explicitly once it is no longer used. Callers use backends through leases (`lease_project_backend*()`) and must not use a backend or its compute after the lease is released. This lets the cache know when a backend is unused: From f3e5e0fe373163ee5ccb302feb2ebfc91137a740 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:26:53 +0500 Subject: [PATCH 12/19] Return only offers from run plan functions --- src/dstack/_internal/server/services/gpus.py | 5 +- .../_internal/server/services/runs/plan.py | 81 +++++++++---------- .../server/services/runs/test_plan.py | 10 +-- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/src/dstack/_internal/server/services/gpus.py b/src/dstack/_internal/server/services/gpus.py index 4e9c526fad..78864da714 100644 --- a/src/dstack/_internal/server/services/gpus.py +++ b/src/dstack/_internal/server/services/gpus.py @@ -2,7 +2,6 @@ from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.gpus import BackendGpu, BackendGpus, CpuGroup, GpuGroup @@ -189,7 +188,7 @@ async def _get_gpu_offers( skip_backend_offers = profile.creation_policy == CreationPolicy.REUSE instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]] - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if profile.instances is not None: instance_offers = await get_targeted_instance_offers( session=session, @@ -218,7 +217,7 @@ async def _get_gpu_offers( full_offers=full_offers, unallocated_resources=unallocated_resources, ) - return [offer for _, offer in instance_offers] + [offer for _, offer in backend_offers] + return [offer for _, offer in instance_offers] + backend_offers def _process_offers_into_backend_gpus( diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index f79a37213e..28b6111ffc 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -1,17 +1,17 @@ +import heapq import itertools import math import uuid from collections.abc import Hashable, Mapping from dataclasses import dataclass from enum import Enum -from typing import Optional, Union +from typing import Optional from sqlalchemy import and_, exists, not_, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import contains_eager, raiseload from sqlalchemy.orm.attributes import set_committed_value -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.models.common import EntityReference from dstack._internal.core.models.fleets import FleetSpec, InstanceGroupPlacement from dstack._internal.core.models.instances import ( @@ -59,10 +59,7 @@ is_multinode_job, remove_job_spec_sensitive_info, ) -from dstack._internal.server.services.offers import ( - get_offers_by_requirements, - merge_offer_iterables, -) +from dstack._internal.server.services.offers import get_offers_by_requirements from dstack._internal.server.services.requirements.combine import ( combine_fleet_and_run_profiles, combine_fleet_and_run_requirements, @@ -363,7 +360,7 @@ class _FleetCandidate: @dataclass class _FleetCandidateWithBackendOffers: candidate: _FleetCandidate - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] sort_key: tuple[bool, float, float] @@ -383,7 +380,7 @@ async def find_optimal_fleet_with_offers( ) -> tuple[ Optional[FleetModel], list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Finds the optimal fleet for the run among the given fleets and returns @@ -448,9 +445,7 @@ async def find_optimal_fleet_with_offers( instance_offers=( available_instance_offers if exclude_not_available else all_instance_offers ), - min_instance_offer_price=_get_min_instance_or_backend_offer_price( - available_instance_offers - ), + min_instance_offer_price=_get_min_instance_offer_price(available_instance_offers), # Require at least one available instance so that fleets without matching # instances are not treated as having capacity when nodes_required_num is 0 # (e.g. a service scaling from zero replicas). @@ -471,7 +466,7 @@ async def find_optimal_fleet_with_offers( # Second step: gather backend offers unless skipped. candidates_with_backend_offers: list[_FleetCandidateWithBackendOffers] = [] for candidate in candidates: - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if _skip_backend_offers: backend_offers = [] else: @@ -495,7 +490,7 @@ async def find_optimal_fleet_with_offers( sort_key=( not candidate.has_pool_capacity, candidate.min_instance_offer_price, - _get_min_instance_or_backend_offer_price(available_backend_offers), + _get_min_backend_offer_price(available_backend_offers), ), ) ) @@ -687,8 +682,8 @@ def select_targeted_instance_offers( candidates.append( _TargetedInstanceOffersCandidate( lacks_capacity=not has_capacity, - available_price=_get_min_instance_or_backend_offer_price(available_offers), - selected_price=_get_min_instance_or_backend_offer_price(all_offers), + available_price=_get_min_instance_offer_price(available_offers), + selected_price=_get_min_instance_offer_price(all_offers), offers=all_offers, ) ) @@ -777,7 +772,7 @@ async def _get_backend_offers_in_fleet( max_offers: Optional[int] = None, full_offers: bool = False, unallocated_resources: bool = False, -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: +) -> list[InstanceOfferWithAvailability]: if fleet_spec is None: fleet_spec = get_fleet_spec(fleet_model) try: @@ -799,7 +794,7 @@ async def _get_backend_offers_in_fleet( # TODO: Drop multinode param. multinode = requirements.multinode or is_multinode_job(job) async with backends_cache.lease_project_backends(project) as backends: - backend_offers = await get_offers_by_requirements( + backends_with_offers = await get_offers_by_requirements( backends=backends, profile=profile, requirements=requirements, @@ -812,6 +807,7 @@ async def _get_backend_offers_in_fleet( full_offers=full_offers, unallocated_resources=unallocated_resources, ) + backend_offers = [offer for _, offer in backends_with_offers] return backend_offers @@ -862,7 +858,7 @@ async def get_non_fleet_offers( unallocated_resources: bool = False, ) -> tuple[ list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Returns instance and backend offers for job irrespective of fleets or instances, @@ -875,12 +871,12 @@ async def get_non_fleet_offers( job=job, volumes=volumes, ) - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if skip_backend_offers: backend_offers = [] else: async with backends_cache.lease_project_backends(project) as backends: - backend_offers = await get_offers_by_requirements( + backends_with_offers = await get_offers_by_requirements( backends=backends, profile=run_spec.merged_profile, requirements=job.job_spec.requirements, @@ -892,6 +888,7 @@ async def get_non_fleet_offers( full_offers=full_offers, unallocated_resources=unallocated_resources, ) + backend_offers = [offer for _, offer in backends_with_offers] return instance_offers, backend_offers @@ -904,7 +901,7 @@ async def get_backend_offers_in_run_candidate_fleets( max_offers_per_fleet: Optional[int] = None, full_offers: bool = False, unallocated_resources: bool = False, -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: +) -> list[InstanceOfferWithAvailability]: """ Returns backend offers across the run's selected candidate fleets. @@ -922,10 +919,10 @@ async def get_backend_offers_in_run_candidate_fleets( run_spec=run_spec, ) seen_offer_identities = set() - offers: list[tuple[Backend, InstanceOfferWithAvailability]] = [] + offers: list[InstanceOfferWithAvailability] = [] for candidate_fleet_model in candidate_fleet_models: offers_from_fleet = [] - for backend, offer in await _get_backend_offers_in_fleet( + for offer in await _get_backend_offers_in_fleet( project=project, fleet_model=candidate_fleet_model, run_spec=run_spec, @@ -937,9 +934,9 @@ async def get_backend_offers_in_run_candidate_fleets( ): offer_identity = _get_backend_offer_identity(offer) if offer_identity not in seen_offer_identities: - offers_from_fleet.append((backend, offer)) + offers_from_fleet.append(offer) seen_offer_identities.add(offer_identity) - offers = list(merge_offer_iterables(offers, offers_from_fleet)) + offers = list(heapq.merge(offers, offers_from_fleet, key=lambda o: o.price)) return offers @@ -954,7 +951,7 @@ async def get_offers_in_run_candidate_fleets( unallocated_resources: bool = False, ) -> tuple[ list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Returns existing-instance and backend offers across the run's candidate fleets. @@ -983,7 +980,7 @@ async def get_offers_in_run_candidate_fleets( ) instance_offers.sort(key=lambda offer: offer[1].price or 0) - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if skip_backend_offers: backend_offers = [] else: @@ -1043,14 +1040,14 @@ def _freeze_offer_identity_value(value: object) -> Hashable: def _get_job_plan( instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]], - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]], + backend_offers: list[InstanceOfferWithAvailability], job: Job, max_offers: Optional[int], reverse_offers: bool = False, ) -> JobPlan: job_offers: list[InstanceOfferWithAvailability] = [] job_offers.extend(offer for _, offer in instance_offers) - job_offers.extend(offer for _, offer in backend_offers) + job_offers.extend(backend_offers) if reverse_offers: job_offers.sort(key=lambda offer: (not offer.availability.is_available(), -offer.price)) else: @@ -1077,15 +1074,19 @@ def _get_offers_from_instances( return instances_with_offers -def _get_min_instance_or_backend_offer_price( - offers: Union[ - list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], - ], +def _get_min_instance_offer_price( + instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]], ) -> float: min_offer_price = math.inf - if len(offers) > 0: - min_offer_price = offers[0][1].price + if len(instance_offers) > 0: + min_offer_price = instance_offers[0][1].price + return min_offer_price + + +def _get_min_backend_offer_price(backend_offers: list[InstanceOfferWithAvailability]) -> float: + min_offer_price = math.inf + if len(backend_offers) > 0: + min_offer_price = backend_offers[0].price return min_offer_price @@ -1100,8 +1101,6 @@ def _exclude_non_available_instance_offers( def _exclude_non_available_backend_offers( - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]], -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: - return [ - (backend, offer) for backend, offer in backend_offers if offer.availability.is_available() - ] + backend_offers: list[InstanceOfferWithAvailability], +) -> list[InstanceOfferWithAvailability]: + return [offer for offer in backend_offers if offer.availability.is_available()] diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index e4a91a9b17..3a75af2575 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -230,8 +230,8 @@ async def test_plans_each_node_group_with_its_own_requirements( async def find_optimal_fleet_with_offers_side_effect(*, job, **kwargs): gpu = job.job_spec.requirements.resources.gpu if gpu is not None and gpu.name: - return Mock(), [], [(Mock(), gpu_offer)] - return Mock(), [], [(Mock(), cpu_offer)] + return Mock(), [], [gpu_offer] + return Mock(), [], [cpu_offer] monkeypatch.setattr( "dstack._internal.server.services.runs.plan._select_candidate_fleet_models", @@ -311,8 +311,8 @@ async def test_plans_each_replica_group_with_its_own_requirements( async def find_optimal_fleet_with_offers_side_effect(*, job, **kwargs): gpu = job.job_spec.requirements.resources.gpu if gpu is not None and gpu.name: - return Mock(), [], [(Mock(), gpu_offer)] - return Mock(), [], [(Mock(), cpu_offer)] + return Mock(), [], [gpu_offer] + return Mock(), [], [cpu_offer] monkeypatch.setattr( "dstack._internal.server.services.runs.plan._select_candidate_fleet_models", @@ -854,7 +854,7 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el volumes=None, ) - assert offers == [(backend, offer)] + assert offers == [offer] get_offers_by_requirements_mock.assert_awaited_once() await_args = get_offers_by_requirements_mock.await_args assert await_args is not None From 242edd99c38b7cb5092dbc496e0c1b6772c2ae91 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:26:54 +0500 Subject: [PATCH 13/19] Narrow backend leases in instance check and termination --- .../pipeline_tasks/instances/check.py | 27 +---- .../pipeline_tasks/instances/termination.py | 106 ++++++++++-------- .../pipeline_tasks/instances/test_check.py | 4 +- 3 files changed, 67 insertions(+), 70 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index c3bab21783..76874b5f62 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -1,7 +1,6 @@ import logging import uuid -from collections.abc import AsyncGenerator, Mapping -from contextlib import asynccontextmanager +from collections.abc import Mapping from datetime import timedelta from typing import Optional @@ -13,7 +12,6 @@ from sqlalchemy.orm import joinedload from dstack._internal import settings -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.backends.base.compute import ( get_dstack_runner_download_url, get_dstack_runner_version, @@ -22,7 +20,6 @@ ) from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT from dstack._internal.core.errors import ProvisioningError -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.health import HealthStatus from dstack._internal.core.models.instances import ( GpuDriverInfo, @@ -335,9 +332,9 @@ async def _process_wait_for_instance_provisioning_data( ) return result - async with _lease_backend_for_provisioning_wait( - project_id=instance_model.project_id, - backend_type=job_provisioning_data.backend, + project_model = await _load_project_with_backends(instance_model.project_id) + async with backends_cache.lease_project_backend_by_type( + project=project_model, backend_type=job_provisioning_data.backend ) as backend: if backend is None: logger.warning( @@ -384,26 +381,14 @@ async def _process_wait_for_instance_provisioning_data( return result -@asynccontextmanager -async def _lease_backend_for_provisioning_wait( - project_id: uuid.UUID, - backend_type: BackendType, -) -> AsyncGenerator[Optional[Backend], None]: +async def _load_project_with_backends(project_id: uuid.UUID) -> ProjectModel: async with get_session_ctx() as session: res = await session.execute( select(ProjectModel) .where(ProjectModel.id == project_id) .options(joinedload(ProjectModel.backends)) ) - project_model = res.unique().scalar_one_or_none() - if project_model is None: - yield None - return - async with backends_cache.lease_project_backend_by_type( - project=project_model, - backend_type=backend_type, - ) as backend: - yield backend + return res.unique().scalar_one() @runner_ssh_tunnel diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py index 470c26e167..0fd3420770 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py @@ -1,3 +1,5 @@ +from typing import Optional + from dstack._internal.core.backends.features import get_backends_with_resource_listing_support from dstack._internal.core.errors import BackendError, NotYetTerminated from dstack._internal.core.models.backends.base import BackendType @@ -35,58 +37,38 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is not None and job_provisioning_data.backend != BackendType.REMOTE: - async with backends_cache.lease_project_backend_by_type( - project=instance_model.project, - backend_type=job_provisioning_data.backend, - ) as backend: - if backend is None: - logger.error( - "Failed to terminate instance %s. Backend %s not available.", - instance_model.name, - job_provisioning_data.backend, - ) - else: - logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) - try: - await run_async( - backend.compute().terminate_instance, - job_provisioning_data.instance_id, - job_provisioning_data.region, - job_provisioning_data.backend_data, - ) - except Exception as exc: - first_retry_at = instance_model.first_termination_retry_at - if first_retry_at is None: - first_retry_at = now - result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER - result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER - if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): - if isinstance(exc, NotYetTerminated): - logger.debug( - "Instance %s termination in progress: %s", - instance_model.name, - exc, - ) - else: - logger.warning( - "Failed to terminate instance %s. Will retry. Error: %r", - instance_model.name, - exc, - exc_info=not isinstance(exc, BackendError), - ) - return result - next_step = ( - "Please terminate the instance manually to avoid unexpected charges." + exc = await _terminate_in_backend(instance_model, job_provisioning_data) + if exc is not None: + first_retry_at = instance_model.first_termination_retry_at + if first_retry_at is None: + first_retry_at = now + result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER + result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER + if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): + if isinstance(exc, NotYetTerminated): + logger.debug( + "Instance %s termination in progress: %s", + instance_model.name, + exc, ) - if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): - next_step = "Orphan cleanup will retry deleting the instance." - logger.error( - "Failed all attempts to terminate instance %s. %s Error: %r", + else: + logger.warning( + "Failed to terminate instance %s. Will retry. Error: %r", instance_model.name, - next_step, exc, exc_info=not isinstance(exc, BackendError), ) + return result + next_step = "Please terminate the instance manually to avoid unexpected charges." + if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): + next_step = "Orphan cleanup will retry deleting the instance." + logger.error( + "Failed all attempts to terminate instance %s. %s Error: %r", + instance_model.name, + next_step, + exc, + exc_info=not isinstance(exc, BackendError), + ) if job_provisioning_data is not None: instance_connection_pool.drop_by_jpd(job_provisioning_data) @@ -102,6 +84,36 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: return result +async def _terminate_in_backend( + instance_model: InstanceModel, job_provisioning_data: JobProvisioningData +) -> Optional[Exception]: + """ + Returns the exception if the backend failed to terminate the instance. + """ + async with backends_cache.lease_project_backend_by_type( + project=instance_model.project, + backend_type=job_provisioning_data.backend, + ) as backend: + if backend is None: + logger.error( + "Failed to terminate instance %s. Backend %s not available.", + instance_model.name, + job_provisioning_data.backend, + ) + return None + logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) + try: + await run_async( + backend.compute().terminate_instance, + job_provisioning_data.instance_id, + job_provisioning_data.region, + job_provisioning_data.backend_data, + ) + except Exception as exc: + return exc + return None + + def _is_to_be_deleted_by_orphan_cleanup( instance_model: InstanceModel, job_provisioning_data: JobProvisioningData ) -> bool: diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py index c79bdccc55..7b0f825c23 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py @@ -166,8 +166,8 @@ async def test_cloudrift_waits_for_backend_within_timeout( await session.commit() backend = Mock() monkeypatch.setattr( - instances_check, - "_lease_backend_for_provisioning_wait", + instances_check.backends_cache, + "lease_project_backend_by_type", mock_lease(backend), ) From 858966ad05e0503f1cd7f1fa4fc7aad9f39e605b Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:26:55 +0500 Subject: [PATCH 14/19] Lease the gateway backend through backends_cache --- src/dstack/_internal/server/services/gateways/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 55660e9228..6f947f7258 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -65,9 +65,7 @@ UserModel, ) from dstack._internal.server.services import events -from dstack._internal.server.services.backends.cache import ( - lease_project_backend_with_model_by_type_or_error, -) +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.server.services.gateways.pool import gateway_connections_pool from dstack._internal.server.services.locking import ( @@ -254,7 +252,7 @@ async def create_gateway( configuration = effective_configuration # Leasing the backend also checks that it can be initialized, not just that it's configured. - async with lease_project_backend_with_model_by_type_or_error( + async with backends_cache.lease_project_backend_with_model_by_type_or_error( project=project, backend_type=configuration.backend ) as (backend_model, _): pass From 2fad06da8f48161d4ff9e61d97594442370292d3 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:26:55 +0500 Subject: [PATCH 15/19] Type the backend selector and document Compute.close() --- contributing/BACKENDS.md | 1 + .../server/services/backends/cache.py | 19 ++++++++++++------- .../_internal/server/services/offers.py | 2 +- src/dstack/_internal/server/testing/common.py | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/contributing/BACKENDS.md b/contributing/BACKENDS.md index a3ee705c05..8201089662 100644 --- a/contributing/BACKENDS.md +++ b/contributing/BACKENDS.md @@ -145,6 +145,7 @@ Refer to examples: Go to `compute.py` and implement `Compute` methods. Optionally, extend and implement `ComputeWith*` classes to support additional features such as fleets, volumes, gateways, placement groups, etc. For example, extend `ComputeWithCreateInstanceSupport` to support fleets. To support the cleanup of orphaned resources, extend `ComputeWithResourceListingSupport` as described in [Orphan cleanup support](#32-orphan-cleanup-support). +If the compute holds resources that garbage collection does not free, such as SDK clients that run background threads, implement `Compute.close()` to release them. The server caches backends and calls `close()` once a backend is no longer used. Refer to examples: [verda](https://github.com/dstackai/dstack/blob/master/src/dstack/_internal/core/backends/verda/compute.py), diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index f80e93c14f..dd97b0941e 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -52,6 +52,15 @@ BackendTuple = Tuple[BackendModel, Backend] +BackendSelector = Callable[ + [ProjectModel, BackendType, List[BackendTuple]], + Coroutine[None, None, Optional[BackendTuple]], +] +""" +Selects a backend of the given type from the project backends, +see `set_select_project_backend_with_model_by_type()`. +Must return one of the given backends or `None`. +""" @dataclass @@ -97,15 +106,10 @@ async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[B yield [b for _, b in backends_with_models] -_select_project_backend_with_model_by_type = None +_select_project_backend_with_model_by_type: Optional[BackendSelector] = None -def set_select_project_backend_with_model_by_type( - func: Callable[ - [ProjectModel, BackendType, List[BackendTuple]], - Coroutine[None, None, Optional[BackendTuple]], - ], -): +def set_select_project_backend_with_model_by_type(func: BackendSelector): """ Overrides how `lease_project_backend*_by_type()` select a backend from the project backends if `overrides=True` is passed. This can be used if a backend needs to be replaced with @@ -214,6 +218,7 @@ async def close_unused_backends(): async def close_all_backends(): """ Retires all backends and closes those that are not leased. + Backends that are still leased stay open: shutdown does not wait for their leases. """ await _retire_and_close_backends(should_retire=lambda c: True) diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index 13378cde30..a0fd5e0f0f 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -52,7 +52,7 @@ async def get_offers_by_requirements( ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]: """ Returns offers of `backends` matching the requirements. The caller must lease `backends` - (see `backends_cache.lease_project_backends()`) and must not use the returned backends + (see `services.backends.cache.lease_project_backends()`) and must not use the returned backends after the lease is released. """ backend_types: Optional[list[BackendType]] = profile.backends diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 451d6dfc67..b081e65641 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -150,7 +150,7 @@ def lease_yielding(value: Any) -> Callable[..., AbstractAsyncContextManager[Any]]: """ - Returns a replacement for a `lease_*()` function from `services.backends` + Returns a replacement for a `lease_*()` function from `services.backends.cache` that yields `value`, e.g. to use as a mock's `side_effect`. """ @@ -163,7 +163,7 @@ async def lease(*args, **kwargs): def mock_lease(value: Any) -> Mock: """ - Returns a mock to patch a `lease_*()` function from `services.backends` with. + Returns a mock to patch a `lease_*()` function from `services.backends.cache` with. The lease yields `value`. """ return Mock(side_effect=lease_yielding(value)) From 9361cc5c48aad3b9a6796ffa40f59fe2db594919 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 3 Nov 2026 16:58:17 +0500 Subject: [PATCH 16/19] Do not close backends on shutdown and document cancellation limits Closing backends on shutdown could close a backend still used by an executor thread of a cancelled worker. The process exits anyway, so leave them open. --- src/dstack/_internal/server/app.py | 2 -- .../server/services/backends/cache.py | 36 +++++++++---------- .../server/services/backends/test_cache.py | 33 +++++++++-------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 76b6f03db6..00fd7ac6c8 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -49,7 +49,6 @@ volumes, ) from dstack._internal.server.services import prometheus as prometheus_service -from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.config import ServerConfigManager from dstack._internal.server.services.gateways import gateway_connections_pool from dstack._internal.server.services.jobs.server_connection import job_server_connections_pool @@ -225,7 +224,6 @@ async def lifespan(app: FastAPI): scheduler.shutdown() if pipeline_manager is not None: await pipeline_manager.drain() - await backends_cache.close_all_backends() await gateway_connections_pool.remove_all() await job_server_connections_pool.remove_all() service_conn_pool = await get_injector_from_app(app).get_service_connection_pool() diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index dd97b0941e..cb55596d6a 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -12,13 +12,27 @@ A cached backend is either active or retired. New leases use only active backends. A backend is retired if its config changes (a new backend replaces it), if it is deleted -from the project, if it is not leased for `_BACKEND_IDLE_TTL`, or on server shutdown. -A retired backend may stay leased by operations that started before it was retired, and they -keep using it until they finish. +from the project, or if it is not leased for `_BACKEND_IDLE_TTL`. A retired backend may stay +leased by operations that started before it was retired, and they keep using it until they finish. Retired backends are closed by `close_unused_backends()`, which runs periodically, once they are not leased. Leases never close backends: closing does not delay callers, and releasing a lease does not await, so cancellation cannot interrupt it. + +Cancelling a task that leases backends has two known problems: + +* Backends are initialized and used in executor threads (`run_async()`), which cancellation + does not interrupt. If the task is cancelled while a backend is in use, its lease is + released while the thread still uses the backend, so a retired backend can be closed under it. +* If the task is cancelled while backends are initialized for its lease, the initialized + backends are not cached, so they are never closed. + +They are accepted because such tasks are only cancelled on server shutdown, when pipelines +cancel their workers, and backends are not closed on shutdown: the process exits anyway. +If code that leases backends is ever cancelled in other cases, e.g. by `asyncio.wait_for()` +around it, both problems can happen while the server runs. Then the lease must not be released +until the executor work finishes (e.g. shield `run_async()` and re-raise the cancellation after +it completes), and backends initialized for a cancelled lease must still be cached. """ import asyncio @@ -210,20 +224,6 @@ async def close_unused_backends(): and closes retired backends that are not leased. """ idle_since = time.monotonic() - _BACKEND_IDLE_TTL - await _retire_and_close_backends( - should_retire=lambda c: c.leases == 0 and c.last_used < idle_since - ) - - -async def close_all_backends(): - """ - Retires all backends and closes those that are not leased. - Backends that are still leased stay open: shutdown does not wait for their leases. - """ - await _retire_and_close_backends(should_retire=lambda c: True) - - -async def _retire_and_close_backends(should_retire: Callable[[_CachedBackend], bool]): to_close: List[_CachedBackend] = [] for project_id in list(_BACKENDS_CACHE): async with _get_project_cache_lock(project_id): @@ -231,7 +231,7 @@ async def _retire_and_close_backends(should_retire: Callable[[_CachedBackend], b if project_backends is None: continue for backend_type, cached_backend in list(project_backends.active.items()): - if should_retire(cached_backend): + if cached_backend.leases == 0 and cached_backend.last_used < idle_since: del project_backends.active[backend_type] project_backends.retired.append(cached_backend) to_close.extend(c for c in project_backends.retired if c.leases == 0) diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index f3513d9137..93fdf6811c 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -32,6 +32,14 @@ def close_mock(backend: Backend) -> Mock: return cast(Mock, backend.close) +async def close_unused_backends_now(monkeypatch: pytest.MonkeyPatch): + """ + Closes all backends that are not leased, as if they all were idle. + """ + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", -1) + await backends_cache.close_unused_backends() + + class TestBackendsCache: @pytest.fixture(autouse=True) def _setup(self, monkeypatch): @@ -122,19 +130,16 @@ async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypat close_mock(aws_backend).assert_not_called() @pytest.mark.asyncio - async def test_close_all_backends_does_not_close_leased_backends(self): + async def test_does_not_close_leased_backends(self, monkeypatch): project = make_project(make_backend_model(BackendType.AWS)) async with backends_cache.lease_project_backends(project) as [backend]: - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(backend).assert_not_called() - async with backends_cache.lease_project_backends(project) as [new_backend]: - assert new_backend is not backend - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(backend).assert_called_once() - close_mock(new_backend).assert_called_once() @pytest.mark.asyncio - async def test_releases_lease_if_cancelled(self): + async def test_releases_lease_if_cancelled(self, monkeypatch): project = make_project(make_backend_model(BackendType.AWS)) leased = asyncio.Event() @@ -148,19 +153,19 @@ async def hold_lease(): task.cancel() with pytest.raises(asyncio.CancelledError): await task - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(self.backends[0]).assert_called_once() @pytest.mark.asyncio - async def test_does_not_raise_if_backend_fails_to_close(self): + async def test_does_not_raise_if_backend_fails_to_close(self, monkeypatch): project = make_project(make_backend_model(BackendType.AWS)) async with backends_cache.lease_project_backends(project) as [backend]: close_mock(backend).side_effect = RuntimeError("close failed") - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(backend).assert_called_once() @pytest.mark.asyncio - async def test_or_error_raises_and_releases_leases(self): + async def test_or_error_raises_and_releases_leases(self, monkeypatch): project = make_project(make_backend_model(BackendType.AWS)) with pytest.raises(BackendNotAvailable): async with backends_cache.lease_project_backend_by_type_or_error( @@ -168,7 +173,7 @@ async def test_or_error_raises_and_releases_leases(self): ): pass [aws_backend] = self.backends - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(aws_backend).assert_called_once() @pytest.mark.asyncio @@ -201,7 +206,7 @@ async def select(project, backend_type, backends_with_models): project, BackendType.AWS, overrides=True ): pass - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(self.backends[0]).assert_called_once() @pytest.mark.asyncio @@ -219,5 +224,5 @@ async def select(project, backend_type, backends_with_models): project, BackendType.AWS, overrides=True ): pass - await backends_cache.close_all_backends() + await close_unused_backends_now(monkeypatch) close_mock(self.backends[0]).assert_called_once() From 4e0dcc34cd544b4af5dbe94658928827e8601a3f Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 4 Nov 2026 11:30:10 +0500 Subject: [PATCH 17/19] Reduce backends cache lease API to three functions --- .../pipeline_tasks/gateway_replicas.py | 21 ++--- .../pipeline_tasks/jobs_submitted.py | 9 +- .../server/services/backends/cache.py | 83 +++---------------- .../server/services/gateways/__init__.py | 14 +++- .../pipeline_tasks/test_gateway_replicas.py | 68 +++++++-------- .../pipeline_tasks/test_jobs_submitted.py | 56 ++++++++++++- .../server/services/backends/test_cache.py | 21 +++-- 7 files changed, 141 insertions(+), 131 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index 457935fdf5..63da7292ef 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -454,17 +454,16 @@ async def _provision_gateway_replica( gateway_model: GatewayModel, replica_model: GatewayReplicaModel, ) -> _GatewayReplicaUpdateMap: - async with backends_cache.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_by_id( project=gateway_model.project, backend_id=replica_model.backend_id - ) as backend_with_model: - if backend_with_model is None: + ) as backend: + if backend is None: return _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.TERMINATED, status_message="Backend not available", active=False, deleted=True, ) - _, backend = backend_with_model compute = backend.compute() assert isinstance(compute, ComputeWithGatewaySupport) replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) @@ -581,12 +580,11 @@ async def _register_replica_with_load_balancer( """ if replica_model.instance_id is None: return "instance_id is None, cannot register with load balancer" - async with backends_cache.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_by_id( project=gateway_model.project, backend_id=replica_model.backend_id - ) as backend_with_model: - if backend_with_model is None: + ) as backend: + if backend is None: return "Backend not available" - _, backend = backend_with_model compute = backend.compute() if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): return "Backend does not support load balancer operations" @@ -1468,11 +1466,11 @@ async def _process_terminating_item(item: GatewayReplicaPipelineItem): mark_terminated_update_map = _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.TERMINATED, active=False, deleted=True ) - async with backends_cache.lease_project_backend_with_model_by_id( + async with backends_cache.lease_project_backend_by_id( project=gateway_model.project, backend_id=replica_model.backend_id, - ) as backend_with_model: - if backend_with_model is None: + ) as backend: + if backend is None: logger.error( "%s: backend not available, cannot terminate." " Marking TERMINATED without termination", @@ -1480,7 +1478,6 @@ async def _process_terminating_item(item: GatewayReplicaPipelineItem): ) await _commit_update(item, replica_model, mark_terminated_update_map) return - _, backend = backend_with_model compute = backend.compute() assert isinstance(compute, ComputeWithGatewaySupport) replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 606f51609b..58eaec1a68 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -1886,9 +1886,16 @@ async def _process_volume_attachments( if len(prepared_job_volumes.volume_model_ids) == 0: return _VolumeAttachmentResult(attachments=[], locked_volume_ids=[]) - async with backends_cache.lease_project_backend_by_type_or_error( + async with backends_cache.lease_project_backend_by_type( project=project, backend_type=job_provisioning_data.backend ) as backend: + if backend is None: + logger.info("%s: failed to attach volume: backend not available", fmt(job_model)) + return _VolumeAttachmentResult( + attachments=[], + locked_volume_ids=[], + termination_message="Failed to attach volume: backend not available", + ) compute = backend.compute() assert isinstance(compute, ComputeWithVolumeSupport) diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py index cb55596d6a..17b0163e89 100644 --- a/src/dstack/_internal/server/services/backends/cache.py +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -54,7 +54,6 @@ from dstack._internal.core.errors import ( BackendAuthError, BackendInvalidCredentialsError, - BackendNotAvailable, ) from dstack._internal.core.models.backends.base import BackendType from dstack._internal.server.models import BackendModel, ProjectModel @@ -104,28 +103,20 @@ def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: @asynccontextmanager -async def lease_project_backends_with_models( - project: ProjectModel, -) -> AsyncGenerator[List[BackendTuple], None]: +async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: cached_backends = await _acquire_project_backends(project) try: - yield [(c.model, c.backend) for c in cached_backends] + yield [c.backend for c in cached_backends] finally: _release_backends(cached_backends) -@asynccontextmanager -async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: - async with lease_project_backends_with_models(project) as backends_with_models: - yield [b for _, b in backends_with_models] - - _select_project_backend_with_model_by_type: Optional[BackendSelector] = None def set_select_project_backend_with_model_by_type(func: BackendSelector): """ - Overrides how `lease_project_backend*_by_type()` select a backend from the project backends + Overrides how `lease_project_backend_by_type()` selects a backend from the project backends if `overrides=True` is passed. This can be used if a backend needs to be replaced with another backend. For example, DstackBackend in dstack Sky can be used in place of other backends. @@ -135,11 +126,11 @@ def set_select_project_backend_with_model_by_type(func: BackendSelector): @asynccontextmanager -async def lease_project_backend_with_model_by_type( +async def lease_project_backend_by_type( project: ProjectModel, backend_type: BackendType, overrides: bool = False, -) -> AsyncGenerator[Optional[BackendTuple], None]: +) -> AsyncGenerator[Optional[Backend], None]: async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: if overrides and _select_project_backend_with_model_by_type is not None: return await _select_project_backend_with_model_by_type( @@ -150,72 +141,22 @@ async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTu return backend_model, backend return None - async with _lease_selected_project_backend(project, select) as backend_with_model: - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_with_model_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[BackendTuple, None]: - async with lease_project_backend_with_model_by_type( - project=project, backend_type=backend_type, overrides=overrides - ) as backend_with_model: - if backend_with_model is None: - raise BackendNotAvailable() - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[Optional[Backend], None]: - async with lease_project_backend_with_model_by_type( - project=project, backend_type=backend_type, overrides=overrides - ) as backend_with_model: - yield backend_with_model[1] if backend_with_model is not None else None - - -@asynccontextmanager -async def lease_project_backend_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> AsyncGenerator[Backend, None]: - async with lease_project_backend_with_model_by_type_or_error( - project=project, backend_type=backend_type, overrides=overrides - ) as (_, backend): + async with _lease_selected_project_backend(project, select) as backend: yield backend @asynccontextmanager -async def lease_project_backend_with_model_by_id( +async def lease_project_backend_by_id( project: ProjectModel, backend_id: UUID -) -> AsyncGenerator[Optional[BackendTuple], None]: +) -> AsyncGenerator[Optional[Backend], None]: async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: for backend_model, backend in backends_with_models: if backend_model.id == backend_id: return backend_model, backend return None - async with _lease_selected_project_backend(project, select) as backend_with_model: - yield backend_with_model - - -@asynccontextmanager -async def lease_project_backend_with_model_by_id_or_error( - project: ProjectModel, backend_id: UUID -) -> AsyncGenerator[BackendTuple, None]: - async with lease_project_backend_with_model_by_id( - project=project, backend_id=backend_id - ) as backend_with_model: - if backend_with_model is None: - raise BackendNotAvailable() - yield backend_with_model + async with _lease_selected_project_backend(project, select) as backend: + yield backend async def close_unused_backends(): @@ -245,7 +186,7 @@ async def close_unused_backends(): async def _lease_selected_project_backend( project: ProjectModel, select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], -) -> AsyncGenerator[Optional[BackendTuple], None]: +) -> AsyncGenerator[Optional[Backend], None]: """ Leases the backend chosen by `select` from the project backends. @@ -273,7 +214,7 @@ async def _lease_selected_project_backend( return selected.last_used = time.monotonic() try: - yield backend_with_model + yield selected.backend finally: _release_backends([selected]) diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 6f947f7258..678649426d 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -23,6 +23,7 @@ BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT, ) from dstack._internal.core.errors import ( + BackendNotAvailable, GatewayError, ResourceNotExistsError, ServerClientError, @@ -66,6 +67,9 @@ ) from dstack._internal.server.services import events from dstack._internal.server.services.backends import cache as backends_cache +from dstack._internal.server.services.backends.store import ( + get_project_backend_model_by_type_or_error, +) from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.server.services.gateways.pool import gateway_connections_pool from dstack._internal.server.services.locking import ( @@ -252,10 +256,14 @@ async def create_gateway( configuration = effective_configuration # Leasing the backend also checks that it can be initialized, not just that it's configured. - async with backends_cache.lease_project_backend_with_model_by_type_or_error( + async with backends_cache.lease_project_backend_by_type( + project=project, backend_type=configuration.backend + ) as backend: + if backend is None: + raise BackendNotAvailable() + backend_model = await get_project_backend_model_by_type_or_error( project=project, backend_type=configuration.backend - ) as (backend_model, _): - pass + ) lock_namespace = f"gateway_names_{project.name}" if is_db_sqlite(): diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index 4df3565949..72ddcec83d 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -414,10 +414,10 @@ async def test_submitted_to_provisioning( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: aws = Mock() - m.side_effect = lease_yielding((backend, aws)) + m.side_effect = lease_yielding(aws) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.return_value = ( GatewayReplicaProvisioningData( @@ -460,10 +460,10 @@ async def test_submitted_backend_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: aws = Mock() - m.side_effect = lease_yielding((backend, aws)) + m.side_effect = lease_yielding(aws) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = BackendError( "Some error" @@ -500,7 +500,7 @@ async def test_submitted_backend_not_available_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: m.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) @@ -536,7 +536,7 @@ async def test_submitted_skips_provisioning_if_gateway_to_be_deleted( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -571,7 +571,7 @@ async def test_submitted_skips_provisioning_if_gateway_failed( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -606,10 +606,10 @@ async def test_submitted_unexpected_error_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: aws = Mock() - m.side_effect = lease_yielding((backend, aws)) + m.side_effect = lease_yielding(aws) aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = RuntimeError( "Unexpected!" @@ -648,7 +648,7 @@ async def test_submitted_to_terminated_when_scaled_in( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -2214,14 +2214,14 @@ async def test_provisioning_to_running_registers_with_load_balancer( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2263,7 +2263,7 @@ async def test_provisioning_skips_load_balancer_registration_without_hostname( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2305,7 +2305,7 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2315,7 +2315,7 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails backend_mock.compute.return_value.register_gateway_replica_with_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2352,14 +2352,14 @@ async def test_provisioning_to_terminating_when_backend_does_not_support_load_ba "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2591,7 +2591,7 @@ async def test_terminating_to_terminated( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2599,7 +2599,7 @@ async def test_terminating_to_terminated( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2669,7 +2669,7 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2677,7 +2677,7 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica_to_terminate)) @@ -2720,7 +2720,7 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2728,7 +2728,7 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2773,7 +2773,7 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2784,7 +2784,7 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( backend_mock.compute.return_value.deregister_gateway_replica_from_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2824,7 +2824,7 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2832,7 +2832,7 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2868,11 +2868,11 @@ async def test_terminating_waits_for_pending_acm_gateway_migration( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock: backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2916,7 +2916,7 @@ async def test_terminating_to_terminated_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock: get_backends_mock.side_effect = lease_yielding(None) await worker.process(_replica_to_pipeline_item(replica)) @@ -2961,7 +2961,7 @@ async def test_terminating_to_terminated_with_no_instance_id( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -2969,7 +2969,7 @@ async def test_terminating_to_terminated_with_no_instance_id( ): backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -3015,7 +3015,7 @@ async def test_terminating_retries_if_terminate_fails( with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_with_model_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" @@ -3026,7 +3026,7 @@ async def test_terminating_retries_if_terminate_fails( backend_mock.compute.return_value.terminate_gateway_replica.side_effect = Exception( "Terminate failed" ) - get_backends_mock.side_effect = lease_yielding((backend, backend_mock)) + get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py index f618280989..e3360dd575 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py @@ -2490,7 +2490,7 @@ async def test_attaches_volume_on_existing_instance( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS @@ -2563,7 +2563,7 @@ async def test_terminates_job_when_volume_is_locked_for_processing( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS @@ -2582,6 +2582,56 @@ async def test_terminates_job_when_volume_is_locked_for_processing( assert volume.lock_expires_at is not None backend_mock.compute.return_value.attach_volume.assert_not_called() + async def test_terminates_job_when_volume_backend_is_not_available( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + volume = await create_volume( + session=session, + project=project, + user=user, + status=VolumeStatus.ACTIVE, + volume_provisioning_data=get_volume_provisioning_data(), + backend=BackendType.AWS, + region="us-east-1", + ) + fleet = await create_fleet(session=session, project=project) + instance = await create_instance( + session=session, + project=project, + fleet=fleet, + status=InstanceStatus.BUSY, + busy_blocks=1, + backend=BackendType.AWS, + region="us-east-1", + ) + run_spec = get_run_spec(repo_id=repo.name) + run_spec.configuration.volumes = [VolumeMountPoint(name=volume.name, path="/volume")] + run = await create_run( + session=session, project=project, repo=repo, user=user, run_spec=run_spec + ) + job = await create_job( + session=session, + run=run, + instance=instance, + instance_assigned=True, + ) + + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), + ): + await _process_job(session=session, worker=worker, job_model=job) + + await session.refresh(job) + await session.refresh(volume) + assert job.status == JobStatus.TERMINATING + assert job.termination_reason == JobTerminationReason.VOLUME_ERROR + assert job.termination_reason_message == "Failed to attach volume: backend not available" + assert volume.lock_owner is None + async def test_reclaims_stale_related_volume_lock( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): @@ -2625,7 +2675,7 @@ async def test_reclaims_stale_related_volume_lock( backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type_or_error", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend_mock), ): backend_mock.TYPE = BackendType.AWS diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index 93fdf6811c..4d2017ddb8 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -7,7 +7,6 @@ import pytest from dstack._internal.core.backends.base.backend import Backend -from dstack._internal.core.errors import BackendNotAvailable from dstack._internal.core.models.backends.base import BackendType from dstack._internal.server.models import BackendModel, ProjectModel from dstack._internal.server.services.backends import cache as backends_cache @@ -165,13 +164,21 @@ async def test_does_not_raise_if_backend_fails_to_close(self, monkeypatch): close_mock(backend).assert_called_once() @pytest.mark.asyncio - async def test_or_error_raises_and_releases_leases(self, monkeypatch): + async def test_leases_backend_by_id(self): + aws_model = make_backend_model(BackendType.AWS) + project = make_project(aws_model, make_backend_model(BackendType.GCP)) + async with backends_cache.lease_project_backend_by_id(project, aws_model.id) as backend: + assert backend is self.backends[0] + + @pytest.mark.asyncio + async def test_returns_none_and_releases_leases_if_backend_not_found(self, monkeypatch): project = make_project(make_backend_model(BackendType.AWS)) - with pytest.raises(BackendNotAvailable): - async with backends_cache.lease_project_backend_by_type_or_error( - project, BackendType.GCP - ): - pass + async with backends_cache.lease_project_backend_by_type( + project, BackendType.GCP + ) as backend: + assert backend is None + async with backends_cache.lease_project_backend_by_id(project, uuid.uuid4()) as backend: + assert backend is None [aws_backend] = self.backends await close_unused_backends_now(monkeypatch) close_mock(aws_backend).assert_called_once() From 09c32b2594eabc8915ab3b5eb3a1090717dd0313 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 4 Nov 2026 11:41:17 +0500 Subject: [PATCH 18/19] Simplify termination lease and lease mocks in tests --- .../pipeline_tasks/instances/termination.py | 30 ++--- src/dstack/_internal/server/testing/common.py | 18 +-- .../pipeline_tasks/instances/test_check.py | 3 +- .../instances/test_cloud_provisioning.py | 112 ++++++++++------ .../instances/test_termination.py | 5 +- .../pipeline_tasks/test_compute_groups.py | 26 ++-- .../pipeline_tasks/test_gateway_replicas.py | 122 +++++++++--------- .../pipeline_tasks/test_gateways.py | 50 +++---- .../pipeline_tasks/test_jobs_terminating.py | 40 +++--- .../pipeline_tasks/test_orphan_cleanup.py | 5 +- .../pipeline_tasks/test_placement_groups.py | 18 +-- .../background/pipeline_tasks/test_volumes.py | 42 +++--- .../scheduled_tasks/test_idle_volumes.py | 14 +- 13 files changed, 251 insertions(+), 234 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py index 0fd3420770..36c40b8684 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py @@ -1,5 +1,3 @@ -from typing import Optional - from dstack._internal.core.backends.features import get_backends_with_resource_listing_support from dstack._internal.core.errors import BackendError, NotYetTerminated from dstack._internal.core.models.backends.base import BackendType @@ -37,8 +35,9 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is not None and job_provisioning_data.backend != BackendType.REMOTE: - exc = await _terminate_in_backend(instance_model, job_provisioning_data) - if exc is not None: + try: + await _terminate_in_backend(instance_model, job_provisioning_data) + except Exception as exc: first_retry_at = instance_model.first_termination_retry_at if first_retry_at is None: first_retry_at = now @@ -86,10 +85,7 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: async def _terminate_in_backend( instance_model: InstanceModel, job_provisioning_data: JobProvisioningData -) -> Optional[Exception]: - """ - Returns the exception if the backend failed to terminate the instance. - """ +): async with backends_cache.lease_project_backend_by_type( project=instance_model.project, backend_type=job_provisioning_data.backend, @@ -100,18 +96,14 @@ async def _terminate_in_backend( instance_model.name, job_provisioning_data.backend, ) - return None + return logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) - try: - await run_async( - backend.compute().terminate_instance, - job_provisioning_data.instance_id, - job_provisioning_data.region, - job_provisioning_data.backend_data, - ) - except Exception as exc: - return exc - return None + await run_async( + backend.compute().terminate_instance, + job_provisioning_data.instance_id, + job_provisioning_data.region, + job_provisioning_data.backend_data, + ) def _is_to_be_deleted_by_orphan_cleanup( diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index b081e65641..73b27b70a1 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -1,7 +1,7 @@ import json import uuid from collections.abc import Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager +from contextlib import asynccontextmanager, contextmanager from datetime import datetime, timezone from http import HTTPStatus from typing import Any, Dict, List, Literal, Optional, Union @@ -148,25 +148,17 @@ from dstack._internal.server.services.users import get_token_hash -def lease_yielding(value: Any) -> Callable[..., AbstractAsyncContextManager[Any]]: +def mock_lease(value: Any) -> Mock: """ - Returns a replacement for a `lease_*()` function from `services.backends.cache` - that yields `value`, e.g. to use as a mock's `side_effect`. + Returns a mock to patch a `lease_*()` function from `services.backends.cache` with. + The lease yields `value`. """ @asynccontextmanager async def lease(*args, **kwargs): yield value - return lease - - -def mock_lease(value: Any) -> Mock: - """ - Returns a mock to patch a `lease_*()` function from `services.backends.cache` with. - The lease yields `value`. - """ - return Mock(side_effect=lease_yielding(value)) + return Mock(side_effect=lease) def get_auth_headers(token: Union[DecryptedString, str]) -> Dict: diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py index 7b0f825c23..47f6cc7a41 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py @@ -166,8 +166,7 @@ async def test_cloudrift_waits_for_backend_within_timeout( await session.commit() backend = Mock() monkeypatch.setattr( - instances_check.backends_cache, - "lease_project_backend_by_type", + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend), ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py index e89b0c07af..816613d344 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py @@ -35,7 +35,7 @@ get_instance_offer_with_availability, get_job_provisioning_data, get_placement_group_provisioning_data, - lease_yielding, + mock_lease, ) from tests._internal.server.background.pipeline_tasks.instances.helpers import ( instance_to_pipeline_item, @@ -87,9 +87,11 @@ async def test_creates_instance( total_blocks=requested_blocks, busy_blocks=0, ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - backend_mock = Mock() - m.side_effect = lease_yielding([backend_mock]) + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS gpu = Gpu(name="T4", memory_mib=16384, vendor=gpuhunt.AcceleratorVendor.NVIDIA) offer = InstanceOfferWithAvailability( @@ -162,8 +164,10 @@ async def test_tries_second_offer_if_first_fails( region=offer.region, price=offer.price, ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([aws_mock, gcp_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([aws_mock, gcp_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -191,8 +195,10 @@ async def test_fails_if_all_offers_fail( aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.get_offers.return_value = [offer] aws_mock.compute.return_value.create_instance.side_effect = err - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([aws_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([aws_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -211,8 +217,10 @@ async def test_fails_if_no_offers( project=project, status=InstanceStatus.PENDING, ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -249,8 +257,10 @@ async def test_waits_when_fleet_has_no_current_master( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -297,8 +307,10 @@ async def test_waits_for_current_master_to_determine_cluster_placement( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(master_instance) @@ -350,8 +362,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi master_item = instance_to_pipeline_item(master_instance) sibling_item = instance_to_pipeline_item(sibling_instance) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([]), + ): await worker.process(master_item) await session.refresh(master_instance) @@ -384,8 +398,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([gcp_mock, aws_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([gcp_mock, aws_mock]), + ): await worker.process(sibling_item) await session.refresh(sibling_instance) @@ -410,8 +426,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([promoted_backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([promoted_backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -479,8 +497,10 @@ async def test_follows_current_master_backend_and_region_constraints( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([gcp_mock, aws_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([gcp_mock, aws_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -541,8 +561,10 @@ async def test_non_master_does_not_create_new_placement_group_without_master_pg( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -606,8 +628,10 @@ async def test_non_master_reuses_existing_current_master_placement_group( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -678,8 +702,10 @@ async def test_allows_parallel_processing_after_master_is_provisioned( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, later_instance) assert backend_mock.compute.return_value.create_instance.call_count == 1 await process_instance(session, worker, earlier_instance) @@ -737,8 +763,10 @@ async def test_create_placement_group_if_placement_cluster( backend_mock.compute.return_value.create_placement_group.return_value = ( get_placement_group_provisioning_data() ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -791,8 +819,10 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.create_instance.return_value = get_job_provisioning_data() compute_mock.create_placement_group.return_value = get_placement_group_provisioning_data() - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -860,8 +890,10 @@ def create_instance_method( get_placement_group_provisioning_data() ) backend_mock.compute.return_value.is_suitable_placement_group.return_value = can_reuse - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -919,8 +951,10 @@ async def test_master_reuses_existing_placement_group( backend_mock.compute.return_value.create_instance.return_value = ( get_job_provisioning_data() ) - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, master_instance) await session.refresh(master_instance) @@ -980,8 +1014,10 @@ def create_placement_group_method( raise err backend_mock.compute.return_value.create_placement_group = create_placement_group_method - with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: - m.side_effect = lease_yielding([backend_mock]) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py index 17394f3806..18e4050244 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py @@ -41,9 +41,8 @@ def mock_terminate_in_backend(error: Optional[Exception] = None): terminate_instance = backend.compute.return_value.terminate_instance if error is not None: terminate_instance.side_effect = error - with patch.object( - instances_termination.backends_cache, - "lease_project_backend_by_type", + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend), ): yield terminate_instance diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py index 38f318022c..15acef062b 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py @@ -21,7 +21,7 @@ create_compute_group, create_fleet, create_project, - lease_yielding, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -165,13 +165,13 @@ async def test_terminates_compute_group( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD await worker.process(_compute_group_to_pipeline_item(compute_group)) compute_mock.terminate_compute_group.assert_called_once() @@ -193,13 +193,13 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) @@ -215,13 +215,13 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.side_effect = lease_yielding(backend_mock) backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index 72ddcec83d..5b2d0241b3 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -48,8 +48,8 @@ get_gateway_replica_configuration, get_job_provisioning_data, get_run_spec, - lease_yielding, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -413,11 +413,11 @@ async def test_submitted_to_provisioning( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as m: - aws = Mock() - m.side_effect = lease_yielding(aws) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.return_value = ( GatewayReplicaProvisioningData( @@ -459,11 +459,11 @@ async def test_submitted_backend_error_marks_terminated( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as m: - aws = Mock() - m.side_effect = lease_yielding(aws) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = BackendError( "Some error" @@ -500,9 +500,9 @@ async def test_submitted_backend_not_available_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as m: - m.side_effect = lease_yielding(None) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(None), + ): await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -605,11 +605,11 @@ async def test_submitted_unexpected_error_marks_terminated( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as m: - aws = Mock() - m.side_effect = lease_yielding(aws) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = RuntimeError( "Unexpected!" @@ -2209,19 +2209,19 @@ async def test_provisioning_to_running_registers_with_load_balancer( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2300,22 +2300,22 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.register_gateway_replica_with_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2347,19 +2347,19 @@ async def test_provisioning_to_terminating_when_backend_does_not_support_load_ba _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2589,17 +2589,17 @@ async def test_terminating_to_terminated( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2667,17 +2667,17 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( _lock_replica(replica_to_terminate) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica_to_terminate)) @@ -2718,17 +2718,17 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2771,20 +2771,20 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.deregister_gateway_replica_from_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2822,17 +2822,17 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2867,12 +2867,12 @@ async def test_terminating_waits_for_pending_acm_gateway_migration( original_last_processed_at = replica.last_processed_at await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -2916,9 +2916,9 @@ async def test_terminating_to_terminated_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock: - get_backends_mock.side_effect = lease_yielding(None) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(None), + ): await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -2959,17 +2959,17 @@ async def test_terminating_to_terminated_with_no_instance_id( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) @@ -3013,20 +3013,20 @@ async def test_terminating_retries_if_terminate_fails( original_last_processed_at = replica.last_processed_at await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.terminate_gateway_replica.side_effect = Exception( "Terminate failed" ) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_replica_to_pipeline_item(replica)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py index 6fddad65f4..47aa3c0694 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py @@ -35,8 +35,8 @@ create_gateway_replica, create_project, get_gateway_replica_configuration, - lease_yielding, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -535,10 +535,11 @@ async def test_submitted_to_provisioning_creates_load_balancer( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_gateway_load_balancer.return_value = ( @@ -546,7 +547,6 @@ async def test_submitted_to_provisioning_creates_load_balancer( hostname="gateway-lb.example.com", backend_data="lb-backend-data" ) ) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -602,13 +602,13 @@ async def test_submitted_to_failed_when_backend_does_not_support_load_balancer( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -643,14 +643,14 @@ async def test_submitted_to_failed_when_load_balancer_creation_raises( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_gateway_load_balancer.side_effect = exception - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -1698,13 +1698,13 @@ async def test_deletes_gateway_and_terminates_load_balancer_when_hostname_set( gateway.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1778,16 +1778,16 @@ async def test_delete_deferred_when_load_balancer_termination_fails( original_last_processed_at = gateway.last_processed_at await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.terminate_gateway_load_balancer.side_effect = ( Exception("boom") ) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1827,13 +1827,13 @@ async def test_delete_deferred_when_backend_does_not_support_load_balancer( gateway.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.side_effect = lease_yielding(backend_mock) await worker.process(_gateway_to_pipeline_item(gateway)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index 6874d523ca..5a64fd62a0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -45,8 +45,8 @@ get_runner_job_submission_mismatch_error, get_volume_configuration, get_volume_provisioning_data, - lease_yielding, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -810,11 +810,11 @@ async def test_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as m: - backend_mock = Mock() - m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) @@ -861,11 +861,11 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.side_effect = lease_yielding(backend_mock) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -879,10 +879,12 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" - ) as m, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.jobs_terminating.get_current_datetime" ) as datetime_mock, @@ -890,8 +892,6 @@ async def test_force_detaches_job_volumes( datetime_mock.return_value = job.volumes_detached_at.replace( tzinfo=timezone.utc ) + timedelta(minutes=30) - backend_mock = Mock() - m.side_effect = lease_yielding(backend_mock) backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -909,11 +909,11 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.side_effect = lease_yielding(backend_mock) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) @@ -1017,11 +1017,11 @@ async def test_detaches_job_volumes_on_shared_instance( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_cache.lease_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.side_effect = lease_yielding(backend_mock) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py index 1882a28afc..8b7f4892e0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py @@ -403,9 +403,8 @@ async def _lock(session: AsyncSession, orphan_cleanup: OrphanCleanupModel) -> Pi def _patch_compute(compute: _ListingCompute): backend = Mock() backend.compute.return_value = compute - return patch.object( - orphan_cleanup_pipeline.backends_cache, - "lease_project_backend_by_type", + return patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", mock_lease(backend), ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py index 3fad8533d8..3cf058dbff 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py @@ -19,7 +19,7 @@ create_fleet, create_placement_group, create_project, - lease_yielding, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -197,11 +197,11 @@ async def test_deletes_placement_group( placement_group.lock_token = uuid.uuid4() placement_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as m: - aws_mock = Mock() - m.side_effect = lease_yielding(aws_mock) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), + ): aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await worker.process(_placement_group_to_pipeline_item(placement_group)) aws_mock.compute.return_value.delete_placement_group.assert_called_once() @@ -228,11 +228,11 @@ async def test_retries_placement_group_deletion_if_still_in_use( placement_group.lock_owner = "PlacementGroupPipeline" original_last_processed_at = placement_group.last_processed_at await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" - ) as m: - aws_mock = Mock() - m.side_effect = lease_yielding(aws_mock) + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), + ): aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.delete_placement_group.side_effect = ( PlacementGroupInUseError() diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py index fe15bd18da..3cac7f3f05 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py @@ -24,8 +24,8 @@ create_volume, get_volume_configuration, get_volume_provisioning_data, - lease_yielding, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -215,17 +215,17 @@ async def test_submitted_to_active(self, test_db, session: AsyncSession, worker: volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_volume.return_value = VolumeProvisioningData( backend=BackendType.AWS, volume_id="vol-1234", size_gb=100, ) - get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -256,10 +256,11 @@ async def test_registers_external_volume( volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.register_volume.return_value = ( VolumeProvisioningData( @@ -268,7 +269,6 @@ async def test_registers_external_volume( size_gb=100, ) ) - get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -298,9 +298,9 @@ async def test_marks_volume_failed_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), ) as get_backend_mock: - get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -329,15 +329,15 @@ async def test_marks_volume_failed_if_backend_returns_error( volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_volume.side_effect = BackendError( "Some error" ) - get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -438,12 +438,12 @@ async def test_marks_volume_deleted( volume.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) @@ -475,7 +475,7 @@ async def test_marks_external_volume_deleted_without_backend_call( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backend_mock: await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_not_called() @@ -505,9 +505,9 @@ async def test_marks_volume_deleted_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), ) as get_backend_mock: - get_backend_mock.side_effect = lease_yielding(None) await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -535,15 +535,15 @@ async def test_marks_volume_deleted_if_backend_delete_errors( volume.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.delete_volume.side_effect = BackendError( "Delete failed" ) - get_backend_mock.side_effect = lease_yielding(backend_mock) await worker.process(_volume_to_pipeline_item(volume)) diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py index a31ca3a870..b2f63c9e42 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py @@ -20,8 +20,8 @@ create_volume, get_volume_configuration, get_volume_provisioning_data, - lease_yielding, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -65,11 +65,11 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): ) await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), ) as m: - aws_mock = Mock() - m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() @@ -111,11 +111,11 @@ async def test_deletes_idle_volume_with_null_auto_cleanup_enabled( volume.auto_cleanup_enabled = None await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), ) as m: - aws_mock = Mock() - m.side_effect = lease_yielding(aws_mock) aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() From b980253169a520399030d4dbf5a04317fc888183 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 4 Nov 2026 12:15:43 +0500 Subject: [PATCH 19/19] Look up backends by type in cache tests --- .../server/services/backends/test_cache.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py index 4d2017ddb8..ddec5f7e6c 100644 --- a/src/tests/_internal/server/services/backends/test_cache.py +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -45,6 +45,7 @@ def _setup(self, monkeypatch): monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE", {}) monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE_LOCKS", {}) monkeypatch.setattr(backends_cache, "get_stored_backend_record", lambda m: m) + # Backends are created concurrently, so their order is not deterministic self.backends: list[Mock] = [] def get_backend(backend_model): @@ -58,6 +59,9 @@ def get_backend(backend_model): lambda backend_type: Mock(get_backend=Mock(side_effect=get_backend)), ) + def get_created_backend(self, backend_type: BackendType) -> Mock: + return next(b for b in self.backends if b.TYPE == backend_type) + @pytest.mark.asyncio async def test_reuses_backends_between_leases(self): project = make_project(make_backend_model(BackendType.AWS)) @@ -88,7 +92,8 @@ async def test_closes_deleted_backend(self): ) async with backends_cache.lease_project_backends(project): pass - aws_backend, gcp_backend = self.backends + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) project.backends = project.backends[:1] async with backends_cache.lease_project_backends(project) as backends: assert backends == [aws_backend] @@ -103,7 +108,8 @@ async def test_closes_idle_backends(self, monkeypatch): ) async with backends_cache.lease_project_backends(project): pass - aws_backend, gcp_backend = self.backends + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) async with backends_cache.lease_project_backend_by_type( project, BackendType.AWS ) as backend: @@ -121,7 +127,8 @@ async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypat ) async with backends_cache.lease_project_backends(project): pass - aws_backend, gcp_backend = self.backends + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", 0) async with backends_cache.lease_project_backend_by_type(project, BackendType.AWS): await backends_cache.close_unused_backends() @@ -168,7 +175,7 @@ async def test_leases_backend_by_id(self): aws_model = make_backend_model(BackendType.AWS) project = make_project(aws_model, make_backend_model(BackendType.GCP)) async with backends_cache.lease_project_backend_by_id(project, aws_model.id) as backend: - assert backend is self.backends[0] + assert backend is self.get_created_backend(BackendType.AWS) @pytest.mark.asyncio async def test_returns_none_and_releases_leases_if_backend_not_found(self, monkeypatch):