diff --git a/contributing/BACKENDS.md b/contributing/BACKENDS.md index a3ee705c05..8201089662 100644 --- a/contributing/BACKENDS.md +++ b/contributing/BACKENDS.md @@ -145,6 +145,7 @@ Refer to examples: Go to `compute.py` and implement `Compute` methods. Optionally, extend and implement `ComputeWith*` classes to support additional features such as fleets, volumes, gateways, placement groups, etc. For example, extend `ComputeWithCreateInstanceSupport` to support fleets. To support the cleanup of orphaned resources, extend `ComputeWithResourceListingSupport` as described in [Orphan cleanup support](#32-orphan-cleanup-support). +If the compute holds resources that garbage collection does not free, such as SDK clients that run background threads, implement `Compute.close()` to release them. The server caches backends and calls `close()` once a backend is no longer used. Refer to examples: [verda](https://github.com/dstackai/dstack/blob/master/src/dstack/_internal/core/backends/verda/compute.py), diff --git a/src/dstack/_internal/core/backends/base/backend.py b/src/dstack/_internal/core/backends/base/backend.py index ee19fd9897..c43fba5c7f 100644 --- a/src/dstack/_internal/core/backends/base/backend.py +++ b/src/dstack/_internal/core/backends/base/backend.py @@ -16,3 +16,10 @@ def compute(self) -> Compute: Returns Compute instance. """ pass + + def close(self) -> None: + """ + Releases resources held by the backend. The backend must not be used after `close()`. + See `Compute.close()`. + """ + self.compute().close() diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 7616f4bcb7..1b4fbc094a 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -204,6 +204,14 @@ def update_provisioning_data( """ pass + def close(self) -> None: + """ + Releases resources that are not freed by garbage collection, e.g. SDK clients that run + background threads. The compute must not be used after `close()`. + It is called from async code in executor, so it can block. Must be idempotent. + """ + pass + class ComputeWithAllOffersCached(ABC): """ diff --git a/src/dstack/_internal/core/backends/configurators.py b/src/dstack/_internal/core/backends/configurators.py index a277dd9a8a..3973687c6e 100644 --- a/src/dstack/_internal/core/backends/configurators.py +++ b/src/dstack/_internal/core/backends/configurators.py @@ -1,6 +1,7 @@ from typing import List, Optional, Type, Union from dstack._internal.core.backends.base.configurator import Configurator +from dstack._internal.core.errors import BackendNotAvailable from dstack._internal.core.models.backends.base import BackendType _CONFIGURATOR_CLASSES: List[Type[Configurator]] = [] @@ -223,3 +224,12 @@ def register_configurator(configurator: Type[Configurator]): Can be used to extend dstack functionality. """ _BACKEND_TYPE_TO_CONFIGURATOR_CLASS_MAP[configurator.TYPE] = configurator + + +def check_backend_type_available(backend_type: BackendType): + if backend_type not in list_available_backend_types(): + raise BackendNotAvailable( + f"Backend {backend_type.value} not available." + " Ensure that backend dependencies are installed." + f" Available backends: {[b.value for b in list_available_backend_types()]}." + ) diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index 89b5795a69..23b3ba24e4 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -132,6 +132,13 @@ def _sdk(self) -> SDK: assert isinstance(self.config.creds, NebiusServiceAccountCreds) return resources.make_sdk(self.config.creds) + def close(self) -> None: + # `_sdk` is created on first use. Close only an existing SDK and only once + if "_sdk" in self.__dict__: + sdk = self._sdk + del self._sdk + resources.close_sdk(sdk) + @cached_property def _region_to_project_id(self) -> dict[str, str]: return resources.get_region_to_project_id_map( @@ -496,11 +503,9 @@ def _wait_for_instance(sdk: SDK, op: resources.NebiusOperation) -> None: op.status(), ) time.sleep(WAIT_FOR_INSTANCE_UPDATE_INTERVAL) - resources.LOOP.await_( - op.update( - per_retry_timeout=resources.REQUEST_TIMEOUT, - auth_options=resources.REQUEST_AUTH_OPTIONS, - ) + op.sync_update( + per_retry_timeout=resources.REQUEST_TIMEOUT, + auth_options=resources.REQUEST_AUTH_OPTIONS, ) diff --git a/src/dstack/_internal/core/backends/nebius/configurator.py b/src/dstack/_internal/core/backends/nebius/configurator.py index 483d4c4137..71c13aa993 100644 --- a/src/dstack/_internal/core/backends/nebius/configurator.py +++ b/src/dstack/_internal/core/backends/nebius/configurator.py @@ -35,6 +35,7 @@ class NebiusConfigurator( def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_enabled: bool): assert isinstance(config.creds, NebiusServiceAccountCreds) + sdk = None try: sdk = resources.make_sdk(config.creds) # check that it's possible to build the projects map with configured settings @@ -46,6 +47,9 @@ def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_en fields=[["creds"]], details=str(e), ) + finally: + if sdk is not None: + resources.close_sdk(sdk) valid_fabrics = get_all_infiniband_fabrics() if invalid_fabrics := set(config.fabrics or []) - valid_fabrics: raise_invalid_credentials_error( diff --git a/src/dstack/_internal/core/backends/nebius/resources.py b/src/dstack/_internal/core/backends/nebius/resources.py index 2928cfc7ab..5ede70067f 100644 --- a/src/dstack/_internal/core/backends/nebius/resources.py +++ b/src/dstack/_internal/core/backends/nebius/resources.py @@ -2,7 +2,7 @@ import re import time from collections import defaultdict -from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence +from collections.abc import Callable, Generator, Iterable, Sequence from collections.abc import Container as ContainerT from contextlib import contextmanager from tempfile import NamedTemporaryFile @@ -10,6 +10,7 @@ from grpc import StatusCode from nebius.aio.operation import Operation as SDKOperation +from nebius.aio.request import Request from nebius.aio.service_error import RequestError from nebius.aio.token.options import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS from nebius.api.nebius.billing.v1 import ( @@ -78,16 +79,15 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.utils.common import get_or_error -from dstack._internal.utils.event_loop import DaemonEventLoop from dstack._internal.utils.logging import get_logger from dstack.version import __version__ # # Guidelines on using the Nebius SDK: # -# Do not use Request.wait() or other sync SDK methods, they suffer from deadlocks. -# Instead, use async methods and await them with LOOP.await_() -LOOP = DaemonEventLoop() +# Use sync methods such as Request.wait() and Operation.sync_update(). The SDK runs requests +# on its own event loop thread, and its sync methods raise LoopError if called from a thread +# with a running event loop, so call them only from sync code, e.g. via run_async(). # Pass a timeout to all methods to avoid infinite waiting REQUEST_TIMEOUT = 10 # Pass REQUEST_AUTH_OPTIONS to all methods to avoid infinite retries in case of invalid credentials @@ -96,6 +96,9 @@ OPTION_RENEW_REQUEST_TIMEOUT: "5", } LIST_PAGE_SIZE = 100 +# Close every SDK created with make_sdk() using close_sdk(). An SDK runs an event loop thread +# that keeps it alive, so an SDK that is not closed is never garbage collected. +SDK_CLOSE_TIMEOUT = 10 # The SDK's own `Operation` does not satisfy the `OperationMessage` bound of `SDKOperation`: # the protocol declares a mutable `id: str`, while the `Operation` setter accepts `str | None` NebiusOperation = SDKOperation[Operation] # pyright: ignore[reportInvalidTypeArguments] @@ -136,6 +139,13 @@ def make_sdk(creds: NebiusServiceAccountCreds) -> SDK: ) +def close_sdk(sdk: SDK) -> None: + try: + sdk.sync_close(timeout=SDK_CLOSE_TIMEOUT) + except Exception: + logger.warning("Failed to close Nebius SDK", exc_info=True) + + def wait_for_operation( op: NebiusOperation, timeout: float, @@ -147,9 +157,7 @@ def wait_for_operation( if time.monotonic() + interval > deadline: raise TimeoutError(f"Operation {op.id} wait timeout") time.sleep(interval) - LOOP.await_( - op.update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS) - ) + op.sync_update(per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS) def get_operation_error(op: NebiusOperation) -> str: @@ -188,22 +196,26 @@ def validate_regions(configured: set[str], available: set[str]) -> None: def list_tenant_projects(sdk: SDK) -> Sequence[Container]: - tenants = LOOP.await_( - TenantServiceClient(sdk).list( + tenants = ( + TenantServiceClient(sdk) + .list( ListTenantsRequest(), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) if len(tenants.items) != 1: raise ValueError(f"Expected to find 1 tenant, found {(len(tenants.items))}") tenant_id = tenants.items[0].metadata.id - projects = LOOP.await_( - ProjectServiceClient(sdk).list( + projects = ( + ProjectServiceClient(sdk) + .list( ListProjectsRequest(parent_id=tenant_id, page_size=999), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) return projects.items @@ -272,12 +284,14 @@ def _get_region_to_configured_project_id_map( def get_default_subnet(sdk: SDK, project_id: str) -> Subnet: - subnets = LOOP.await_( - SubnetServiceClient(sdk).list( + subnets = ( + SubnetServiceClient(sdk) + .list( ListSubnetsRequest(parent_id=project_id, page_size=999), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) for subnet in subnets.items: if subnet.metadata.name.startswith("default-subnet"): @@ -311,21 +325,17 @@ def create_disk( ), ) with wrap_capacity_errors(): - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def delete_disk(sdk: SDK, disk_id: str) -> None: - LOOP.await_( - DiskServiceClient(sdk).delete( - DeleteDiskRequest(id=disk_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + DiskServiceClient(sdk).delete( + DeleteDiskRequest(id=disk_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def list_disks(sdk: SDK, project_id: str) -> list[Disk]: @@ -352,31 +362,29 @@ def create_pricing_policy( pricing=PricingMethod(max_price_v1=MaxPriceV1(max_price=max_price)), ), ) - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def get_pricing_policy(sdk: SDK, pricing_policy_id: str) -> PricingPolicy: - return LOOP.await_( - PricingPolicyServiceClient(sdk).get( + return ( + PricingPolicyServiceClient(sdk) + .get( GetPricingPolicyRequest(id=pricing_policy_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_pricing_policy(sdk: SDK, pricing_policy_id: str) -> None: - LOOP.await_( - PricingPolicyServiceClient(sdk).delete( - DeletePricingPolicyRequest(id=pricing_policy_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + PricingPolicyServiceClient(sdk).delete( + DeletePricingPolicyRequest(id=pricing_policy_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def list_pricing_policies(sdk: SDK, project_id: str) -> list[PricingPolicy]: @@ -446,30 +454,32 @@ def create_instance( ), ) with wrap_capacity_errors(): - return LOOP.await_( - client.create( - request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS - ) - ) + return client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ).wait() def get_instance(sdk: SDK, instance_id: str) -> Instance: - return LOOP.await_( - InstanceServiceClient(sdk).get( + return ( + InstanceServiceClient(sdk) + .get( GetInstanceRequest(id=instance_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_instance(sdk: SDK, instance_id: str) -> NebiusOperation: - return LOOP.await_( - InstanceServiceClient(sdk).delete( + return ( + InstanceServiceClient(sdk) + .delete( DeleteInstanceRequest(id=instance_id), per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) @@ -488,8 +498,9 @@ def list_instances(sdk: SDK, project_id: str) -> list[Instance]: def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusOperation: with wrap_capacity_errors(): - return LOOP.await_( - GpuClusterServiceClient(sdk).create( + return ( + GpuClusterServiceClient(sdk) + .create( CreateGpuClusterRequest( metadata=ResourceMetadata(name=name, parent_id=project_id), spec=GpuClusterSpec(infiniband_fabric=fabric), @@ -497,17 +508,16 @@ def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusO per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS, ) + .wait() ) def delete_cluster(sdk: SDK, cluster_id: str) -> None: - LOOP.await_( - GpuClusterServiceClient(sdk).delete( - DeleteGpuClusterRequest(id=cluster_id), - per_retry_timeout=REQUEST_TIMEOUT, - auth_options=REQUEST_AUTH_OPTIONS, - ) - ) + GpuClusterServiceClient(sdk).delete( + DeleteGpuClusterRequest(id=cluster_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ).wait() def filter_invalid_labels(labels: Dict[str, str]) -> Dict[str, str]: @@ -549,14 +559,14 @@ def is_valid_label_value(value: str) -> bool: return match is not None -def _list_all_pages(list_page: Callable[[str], Awaitable[Any]]) -> list: +def _list_all_pages(list_page: Callable[[str], Request[Any, Any]]) -> list: """ Calls `list_page` with the next page token until all pages are fetched. """ items = [] page_token = "" while True: - response = LOOP.await_(list_page(page_token)) + response = list_page(page_token).wait() items.extend(response.items) page_token = response.next_page_token if not page_token: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py index 5cccbd5204..af8c0510f0 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/compute_groups.py @@ -28,7 +28,7 @@ ) from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ComputeGroupModel, InstanceModel, ProjectModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.compute_groups import compute_group_model_to_compute_group from dstack._internal.server.services.instances import emit_instance_status_change_event from dstack._internal.server.services.locking import get_locker @@ -289,52 +289,52 @@ async def _terminate_compute_group(compute_group_model: ComputeGroupModel) -> _T return result compute_group = compute_group_model_to_compute_group(compute_group_model) cgpd = compute_group.provisioning_data - backend = await backends_services.get_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=compute_group_model.project, backend_type=cgpd.backend, - ) - if backend is None: - logger.error( - "Failed to terminate compute group %s. Backend %s not available." - " Please terminate it manually to avoid unexpected charges.", - compute_group.name, - cgpd.backend, - ) - return _get_terminated_result() - logger.debug("Terminating compute group %s", compute_group.name) - compute = backend.compute() - assert isinstance(compute, ComputeWithGroupProvisioningSupport) - try: - await run_async( - compute.terminate_compute_group, - compute_group, - ) - except Exception as e: - retry_at = get_current_datetime() - first_termination_retry_at = compute_group_model.first_termination_retry_at - if compute_group_model.first_termination_retry_at is None: - result.compute_group_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER - first_termination_retry_at = retry_at - assert first_termination_retry_at is not None - result.compute_group_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER - if _next_termination_retry_at(retry_at) < _get_termination_deadline( - first_termination_retry_at - ): - logger.warning( - "Failed to terminate compute group %s. Will retry. Error: %r", + ) as backend: + if backend is None: + logger.error( + "Failed to terminate compute group %s. Backend %s not available." + " Please terminate it manually to avoid unexpected charges.", + compute_group.name, + cgpd.backend, + ) + return _get_terminated_result() + logger.debug("Terminating compute group %s", compute_group.name) + compute = backend.compute() + assert isinstance(compute, ComputeWithGroupProvisioningSupport) + try: + await run_async( + compute.terminate_compute_group, + compute_group, + ) + except Exception as e: + retry_at = get_current_datetime() + first_termination_retry_at = compute_group_model.first_termination_retry_at + if compute_group_model.first_termination_retry_at is None: + result.compute_group_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER + first_termination_retry_at = retry_at + assert first_termination_retry_at is not None + result.compute_group_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER + if _next_termination_retry_at(retry_at) < _get_termination_deadline( + first_termination_retry_at + ): + logger.warning( + "Failed to terminate compute group %s. Will retry. Error: %r", + compute_group.name, + e, + exc_info=not isinstance(e, BackendError), + ) + return result + logger.error( + "Failed all attempts to terminate compute group %s." + " Please terminate it manually to avoid unexpected charges." + " Error: %r", compute_group.name, e, exc_info=not isinstance(e, BackendError), ) - return result - logger.error( - "Failed all attempts to terminate compute group %s." - " Please terminate it manually to avoid unexpected charges." - " Error: %r", - compute_group.name, - e, - exc_info=not isinstance(e, BackendError), - ) terminated_result = _get_terminated_result() terminated_result.compute_group_update_map.update(result.compute_group_update_map) terminated_result.instances_update_map.update(result.instances_update_map) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index 2530e10ec3..63da7292ef 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -14,7 +14,7 @@ ComputeWithGatewayLoadBalancerSupport, ComputeWithGatewaySupport, ) -from dstack._internal.core.errors import BackendError, BackendNotAvailable, GatewayError +from dstack._internal.core.errors import BackendError, GatewayError from dstack._internal.core.models.common import validate_json_extra_ignore from dstack._internal.core.models.gateways import GatewayReplicaStatus, GatewayStatus from dstack._internal.core.models.runs import JobSpec, JobStatus, RunStatus, ServiceSpec @@ -45,9 +45,9 @@ ServiceRegistrationModel, ServiceReplicaRegistrationModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import gateways as gateways_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.gateways import ( get_gateway_configuration, get_gateway_lb_configuration, @@ -454,45 +454,45 @@ async def _provision_gateway_replica( gateway_model: GatewayModel, replica_model: GatewayReplicaModel, ) -> _GatewayReplicaUpdateMap: - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, backend_id=replica_model.backend_id - ) - except BackendNotAvailable: - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message="Backend not available", - active=False, - deleted=True, - ) - - compute = backend.compute() - assert isinstance(compute, ComputeWithGatewaySupport) - replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) + async with backends_cache.lease_project_backend_by_id( + project=gateway_model.project, backend_id=replica_model.backend_id + ) as backend: + if backend is None: + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message="Backend not available", + active=False, + deleted=True, + ) + compute = backend.compute() + assert isinstance(compute, ComputeWithGatewaySupport) + replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) - logger.debug("%s: creating gateway replica", fmt(replica_model)) - try: - gpd = await run_async( - compute.create_gateway_replica, replica_configuration, gateway_model.backend_data - ) - except BackendError as e: - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message=status_message, - active=False, - deleted=True, - ) - except Exception: - logger.exception("%s: unexpected error when creating gateway replica", fmt(replica_model)) - return _GatewayReplicaUpdateMap( - status=GatewayReplicaStatus.TERMINATED, - status_message="Unexpected error", - active=False, - deleted=True, - ) + logger.debug("%s: creating gateway replica", fmt(replica_model)) + try: + gpd = await run_async( + compute.create_gateway_replica, replica_configuration, gateway_model.backend_data + ) + except BackendError as e: + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message=status_message, + active=False, + deleted=True, + ) + except Exception: + logger.exception( + "%s: unexpected error when creating gateway replica", fmt(replica_model) + ) + return _GatewayReplicaUpdateMap( + status=GatewayReplicaStatus.TERMINATED, + status_message="Unexpected error", + active=False, + deleted=True, + ) return _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.PROVISIONING, @@ -580,26 +580,25 @@ async def _register_replica_with_load_balancer( """ if replica_model.instance_id is None: return "instance_id is None, cannot register with load balancer" - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, backend_id=replica_model.backend_id - ) - except BackendNotAvailable: - return "Backend not available" - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - return "Backend does not support load balancer operations" - lb_configuration = get_gateway_lb_configuration(gateway_model) - try: - await run_async( - compute.register_gateway_replica_with_load_balancer, - replica_model.instance_id, - lb_configuration, - gateway_model.backend_data, - ) - except Exception: - logger.exception("%s: error registering with load balancer", fmt(replica_model)) - return "Error registering with load balancer" + async with backends_cache.lease_project_backend_by_id( + project=gateway_model.project, backend_id=replica_model.backend_id + ) as backend: + if backend is None: + return "Backend not available" + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + return "Backend does not support load balancer operations" + lb_configuration = get_gateway_lb_configuration(gateway_model) + try: + await run_async( + compute.register_gateway_replica_with_load_balancer, + replica_model.instance_id, + lb_configuration, + gateway_model.backend_data, + ) + except Exception: + logger.exception("%s: error registering with load balancer", fmt(replica_model)) + return "Error registering with load balancer" logger.info("%s: registered with load balancer", fmt(replica_model)) return None @@ -1467,46 +1466,48 @@ async def _process_terminating_item(item: GatewayReplicaPipelineItem): mark_terminated_update_map = _GatewayReplicaUpdateMap( status=GatewayReplicaStatus.TERMINATED, active=False, deleted=True ) - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_id_or_error( - project=gateway_model.project, - backend_id=replica_model.backend_id, - ) - except BackendNotAvailable: - logger.error( - "%s: backend not available, cannot terminate. Marking TERMINATED without termination", - fmt(replica_model), - ) - await _commit_update(item, replica_model, mark_terminated_update_map) - return - compute = backend.compute() - assert isinstance(compute, ComputeWithGatewaySupport) - replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) - if replica_model.instance_id is None: - logger.warning( - "%s: instance_id is None, skipping gateway replica termination", fmt(replica_model) - ) - await _commit_update(item, replica_model, mark_terminated_update_map) - return - if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model): - await _commit_update(item, replica_model, update_map={}) - return + async with backends_cache.lease_project_backend_by_id( + project=gateway_model.project, + backend_id=replica_model.backend_id, + ) as backend: + if backend is None: + logger.error( + "%s: backend not available, cannot terminate." + " Marking TERMINATED without termination", + fmt(replica_model), + ) + await _commit_update(item, replica_model, mark_terminated_update_map) + return + compute = backend.compute() + assert isinstance(compute, ComputeWithGatewaySupport) + replica_configuration = get_gateway_replica_configuration(replica_model, gateway_model) + if replica_model.instance_id is None: + logger.warning( + "%s: instance_id is None, skipping gateway replica termination", fmt(replica_model) + ) + await _commit_update(item, replica_model, mark_terminated_update_map) + return + if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model): + await _commit_update(item, replica_model, update_map={}) + return - if gateway_model.hostname is not None: - await _deregister_gateway_replica_from_load_balancer(compute, gateway_model, replica_model) + if gateway_model.hostname is not None: + await _deregister_gateway_replica_from_load_balancer( + compute, gateway_model, replica_model + ) - logger.debug("%s: terminating gateway replica", fmt(replica_model)) - try: - await run_async( - compute.terminate_gateway_replica, - replica_model.instance_id, - replica_configuration, - replica_model.backend_data, - ) - except Exception: - logger.exception("%s: error when terminating gateway replica", fmt(replica_model)) - await _commit_update(item, replica_model, update_map={}) - return + logger.debug("%s: terminating gateway replica", fmt(replica_model)) + try: + await run_async( + compute.terminate_gateway_replica, + replica_model.instance_id, + replica_configuration, + replica_model.backend_data, + ) + except Exception: + logger.exception("%s: error when terminating gateway replica", fmt(replica_model)) + await _commit_update(item, replica_model, update_map={}) + return if replica_model.ip_address is not None: await gateway_connections_pool.remove(replica_model.ip_address) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py index 377f82dd7c..300c6a1c66 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateways.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateways.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import joinedload, load_only, selectinload from dstack._internal.core.backends.base.compute import ComputeWithGatewayLoadBalancerSupport -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.gateways import ( GATEWAY_REPLICAS_DEFAULT, GatewayReplicaStatus, @@ -39,9 +39,9 @@ GatewayReplicaModel, ProjectModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import gateways as gateways_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.gateways import ( emit_gateway_status_change_event, get_gateway_lb_configuration, @@ -351,60 +351,58 @@ async def _process_submitted_gateway(gateway_model: GatewayModel) -> _SubmittedR configuration = gateways_services.get_gateway_configuration(gateway_model) update_map: _GatewayUpdateMap = {} if get_effective_load_balancer(configuration) is not None: - try: - ( - _, - backend, - ) = await backends_services.get_project_backend_with_model_by_type_or_error( - project=gateway_model.project, backend_type=configuration.backend - ) - except BackendNotAvailable: - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Backend not available", - } - ) - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - logger.error( - "%s: backend does not support load balancer operations", fmt(gateway_model) - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Backend does not support load balancer operations", - } - ) - lb_configuration = get_gateway_lb_configuration(gateway_model) - logger.info("%s: creating gateway load balancer...", fmt(gateway_model)) - try: - backend_data = await run_async(compute.create_gateway_load_balancer, lb_configuration) - except BackendError as e: - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - logger.warning( - "%s: failed to create gateway load balancer: %s", - fmt(gateway_model), - status_message, - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": status_message, - } - ) - except Exception: - logger.exception( - "%s: unexpected error when creating gateway load balancer", fmt(gateway_model) - ) - return _SubmittedResult( - update_map={ - "status": GatewayStatus.FAILED, - "status_message": "Unexpected error when creating load balancer", - } - ) + async with backends_cache.lease_project_backend_by_type( + project=gateway_model.project, backend_type=configuration.backend + ) as backend: + if backend is None: + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Backend not available", + } + ) + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + logger.error( + "%s: backend does not support load balancer operations", fmt(gateway_model) + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Backend does not support load balancer operations", + } + ) + lb_configuration = get_gateway_lb_configuration(gateway_model) + logger.info("%s: creating gateway load balancer...", fmt(gateway_model)) + try: + backend_data = await run_async( + compute.create_gateway_load_balancer, lb_configuration + ) + except BackendError as e: + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + logger.warning( + "%s: failed to create gateway load balancer: %s", + fmt(gateway_model), + status_message, + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": status_message, + } + ) + except Exception: + logger.exception( + "%s: unexpected error when creating gateway load balancer", fmt(gateway_model) + ) + return _SubmittedResult( + update_map={ + "status": GatewayStatus.FAILED, + "status_message": "Unexpected error when creating load balancer", + } + ) logger.info("%s: gateway load balancer created.", fmt(gateway_model)) update_map["hostname"] = backend_data.hostname update_map["backend_data"] = backend_data.backend_data @@ -682,28 +680,33 @@ async def _process_to_be_deleted_gateway(gateway_model: GatewayModel) -> _Proces async def _terminate_gateway_load_balancer(gateway_model: GatewayModel) -> bool: """Terminates the gateway's load balancer. Returns True on success, False on failure.""" configuration = gateways_services.get_gateway_configuration(gateway_model) - try: - (_, backend) = await backends_services.get_project_backend_with_model_by_type_or_error( - project=gateway_model.project, backend_type=configuration.backend - ) - except BackendNotAvailable: - logger.error( - "%s: backend not available, cannot terminate load balancer", fmt(gateway_model) - ) - return False - compute = backend.compute() - if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): - logger.error("%s: backend does not support load balancer operations", fmt(gateway_model)) - return False - lb_configuration = get_gateway_lb_configuration(gateway_model) - logger.info("%s: terminating gateway load balancer...", fmt(gateway_model)) - try: - await run_async( - compute.terminate_gateway_load_balancer, lb_configuration, gateway_model.backend_data - ) - except Exception: - logger.exception("%s: error when terminating gateway load balancer", fmt(gateway_model)) - return False + async with backends_cache.lease_project_backend_by_type( + project=gateway_model.project, backend_type=configuration.backend + ) as backend: + if backend is None: + logger.error( + "%s: backend not available, cannot terminate load balancer", fmt(gateway_model) + ) + return False + compute = backend.compute() + if not isinstance(compute, ComputeWithGatewayLoadBalancerSupport): + logger.error( + "%s: backend does not support load balancer operations", fmt(gateway_model) + ) + return False + lb_configuration = get_gateway_lb_configuration(gateway_model) + logger.info("%s: terminating gateway load balancer...", fmt(gateway_model)) + try: + await run_async( + compute.terminate_gateway_load_balancer, + lb_configuration, + gateway_model.backend_data, + ) + except Exception: + logger.exception( + "%s: error when terminating gateway load balancer", fmt(gateway_model) + ) + return False logger.info("%s: gateway load balancer terminated.", fmt(gateway_model)) return True diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index 0bd31d06c8..76874b5f62 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -12,7 +12,6 @@ from sqlalchemy.orm import joinedload from dstack._internal import settings -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.backends.base.compute import ( get_dstack_runner_download_url, get_dstack_runner_version, @@ -21,7 +20,6 @@ ) from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT from dstack._internal.core.errors import ProvisioningError -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.health import HealthStatus from dstack._internal.core.models.instances import ( GpuDriverInfo, @@ -52,7 +50,7 @@ ComponentStatus, InstanceHealthResponse, ) -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( get_instance_provisioning_data, get_instance_ssh_private_keys, @@ -334,72 +332,63 @@ async def _process_wait_for_instance_provisioning_data( ) return result - backend = await _get_backend_for_provisioning_wait( - project_id=instance_model.project_id, - backend_type=job_provisioning_data.backend, - ) - if backend is None: - logger.warning( - "Instance %s failed because instance's backend is not available", - instance_model.name, - ) - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATING, - termination_reason=InstanceTerminationReason.ERROR, - termination_reason_message="Backend not available", - ) - return result - - try: - await run_async( - backend.compute().update_provisioning_data, - job_provisioning_data, - instance_model.project.ssh_public_key, - instance_model.project.ssh_private_key, - ) - result.instance_update_map["job_provisioning_data"] = ( - job_provisioning_data.model_dump_json() - ) - except ProvisioningError as exc: - logger.warning( - "Error while waiting for instance %s to become running: %s", - instance_model.name, - repr(exc), - ) - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATING, - termination_reason=InstanceTerminationReason.ERROR, - termination_reason_message="Error while waiting for instance to become running", - ) - except Exception: - logger.exception( - "Got exception when updating instance %s provisioning data", - instance_model.name, - ) + project_model = await _load_project_with_backends(instance_model.project_id) + async with backends_cache.lease_project_backend_by_type( + project=project_model, backend_type=job_provisioning_data.backend + ) as backend: + if backend is None: + logger.warning( + "Instance %s failed because instance's backend is not available", + instance_model.name, + ) + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.TERMINATING, + termination_reason=InstanceTerminationReason.ERROR, + termination_reason_message="Backend not available", + ) + return result + + try: + await run_async( + backend.compute().update_provisioning_data, + job_provisioning_data, + instance_model.project.ssh_public_key, + instance_model.project.ssh_private_key, + ) + result.instance_update_map["job_provisioning_data"] = ( + job_provisioning_data.model_dump_json() + ) + except ProvisioningError as exc: + logger.warning( + "Error while waiting for instance %s to become running: %s", + instance_model.name, + repr(exc), + ) + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.TERMINATING, + termination_reason=InstanceTerminationReason.ERROR, + termination_reason_message="Error while waiting for instance to become running", + ) + except Exception: + logger.exception( + "Got exception when updating instance %s provisioning data", + instance_model.name, + ) return result -async def _get_backend_for_provisioning_wait( - project_id: uuid.UUID, - backend_type: BackendType, -) -> Optional[Backend]: +async def _load_project_with_backends(project_id: uuid.UUID) -> ProjectModel: async with get_session_ctx() as session: res = await session.execute( select(ProjectModel) .where(ProjectModel.id == project_id) .options(joinedload(ProjectModel.backends)) ) - project_model = res.unique().scalar_one_or_none() - if project_model is None: - return None - return await backends_services.get_project_backend_by_type( - project=project_model, - backend_type=backend_type, - ) + return res.unique().scalar_one() @runner_ssh_tunnel diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py index bd869b8325..45e7384b0e 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py @@ -37,6 +37,7 @@ ) from dstack._internal.server.db import get_session_ctx from dstack._internal.server.models import FleetModel, InstanceModel, PlacementGroupModel +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.fleets import get_fleet_offers, is_cloud_cluster from dstack._internal.server.services.instances import ( get_instance_configuration, @@ -103,148 +104,151 @@ async def create_cloud_instance(instance_model: InstanceModel) -> ProcessResult: ) master_job_provisioning_data = cluster_context.master_job_provisioning_data - offers = await get_fleet_offers( - project=instance_model.project, - profile=profile, - requirements=requirements, - fleet_model=instance_model.fleet, - placement_group=placement_group_model_to_placement_group_optional(placement_group_model), - blocks="auto" if instance_model.total_blocks is None else instance_model.total_blocks, - exclude_not_available=True, - master_job_provisioning_data=master_job_provisioning_data, - infer_master_job_provisioning_data_from_fleet_instances=False, - include_only_create_instance_supported_backends=True, - ) + async with backends_cache.lease_project_backends(instance_model.project) as backends: + offers = await get_fleet_offers( + backends=backends, + profile=profile, + requirements=requirements, + fleet_model=instance_model.fleet, + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), + blocks="auto" if instance_model.total_blocks is None else instance_model.total_blocks, + exclude_not_available=True, + master_job_provisioning_data=master_job_provisioning_data, + infer_master_job_provisioning_data_from_fleet_instances=False, + include_only_create_instance_supported_backends=True, + ) - offers_iter = iter(offers) - offers_tried = 0 - # Limit number of offers tried to prevent long-running processing in case all offers fail. - while offers_tried < server_settings.MAX_OFFERS_TRIED: - backend_with_instance_offer = next(offers_iter, None) - if backend_with_instance_offer is None: - break - backend, instance_offer = backend_with_instance_offer - - if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT: - continue - compute = backend.compute() - assert isinstance(compute, ComputeWithCreateInstanceSupport) - if master_job_provisioning_data is not None: - # `get_fleet_offers()` already restricts backend and region from the master. - # Availability zone still has to be narrowed per offer. - instance_offer = get_instance_offer_with_restricted_az( - instance_offer=instance_offer, - master_job_provisioning_data=master_job_provisioning_data, - ) - if ( - cluster_context is not None - and cluster_context.is_current_instance_master - and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT - and isinstance(compute, ComputeWithPlacementGroupSupport) - and await can_use_placement_groups( - compute=compute, - instance_offer=instance_offer, - reservation=instance_configuration.reservation, - ) - ): - ( - placement_group_model, - created_placement_group_model, - ) = await _find_or_create_suitable_placement_group_model( - instance_model=instance_model, - placement_group_models=placement_group_models, - instance_offer=instance_offer, - compute=compute, - ) - if placement_group_model is None: + offers_iter = iter(offers) + offers_tried = 0 + # Limit number of offers tried to prevent long-running processing in case all offers fail. + while offers_tried < server_settings.MAX_OFFERS_TRIED: + backend_with_instance_offer = next(offers_iter, None) + if backend_with_instance_offer is None: + break + backend, instance_offer = backend_with_instance_offer + + if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT: continue - if created_placement_group_model: - placement_group_models.append(placement_group_model) - result.new_placement_group_models.append(placement_group_model) + compute = backend.compute() + assert isinstance(compute, ComputeWithCreateInstanceSupport) + if master_job_provisioning_data is not None: + # `get_fleet_offers()` already restricts backend and region from the master. + # Availability zone still has to be narrowed per offer. + instance_offer = get_instance_offer_with_restricted_az( + instance_offer=instance_offer, + master_job_provisioning_data=master_job_provisioning_data, + ) + if ( + cluster_context is not None + and cluster_context.is_current_instance_master + and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT + and isinstance(compute, ComputeWithPlacementGroupSupport) + and await can_use_placement_groups( + compute=compute, + instance_offer=instance_offer, + reservation=instance_configuration.reservation, + ) + ): + ( + placement_group_model, + created_placement_group_model, + ) = await _find_or_create_suitable_placement_group_model( + instance_model=instance_model, + placement_group_models=placement_group_models, + instance_offer=instance_offer, + compute=compute, + ) + if placement_group_model is None: + continue + if created_placement_group_model: + placement_group_models.append(placement_group_model) + result.new_placement_group_models.append(placement_group_model) - logger.debug( - "Trying %s in %s/%s for $%0.4f per hour", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, - instance_offer.price, - ) - offers_tried += 1 - try: - job_provisioning_data = await run_async( - compute.create_instance, - instance_offer, - instance_configuration, - placement_group_model_to_placement_group_optional(placement_group_model), - ) - except SkipOffer as exc: - offers_tried -= 1 - logger.info( - "%s launch in %s/%s skipped: %s", + logger.debug( + "Trying %s in %s/%s for $%0.4f per hour", instance_offer.instance.name, instance_offer.backend.value, instance_offer.region, - exc, - extra={"instance_name": instance_model.name}, + instance_offer.price, ) - continue - except BackendError as exc: - logger.warning( - "%s launch in %s/%s failed: %s", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, - repr(exc), - extra={"instance_name": instance_model.name}, + offers_tried += 1 + try: + job_provisioning_data = await run_async( + compute.create_instance, + instance_offer, + instance_configuration, + placement_group_model_to_placement_group_optional(placement_group_model), + ) + except SkipOffer as exc: + offers_tried -= 1 + logger.info( + "%s launch in %s/%s skipped: %s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + exc, + extra={"instance_name": instance_model.name}, + ) + continue + except BackendError as exc: + logger.warning( + "%s launch in %s/%s failed: %s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + repr(exc), + extra={"instance_name": instance_model.name}, + ) + continue + except Exception: + logger.exception( + "Got exception when launching %s in %s/%s", + instance_offer.instance.name, + instance_offer.backend.value, + instance_offer.region, + ) + continue + + set_status_update( + update_map=result.instance_update_map, + instance_model=instance_model, + new_status=InstanceStatus.PROVISIONING, ) - continue - except Exception: - logger.exception( - "Got exception when launching %s in %s/%s", - instance_offer.instance.name, - instance_offer.backend.value, - instance_offer.region, + result.instance_update_map["backend"] = backend.TYPE + result.instance_update_map["region"] = instance_offer.region + result.instance_update_map["price"] = instance_offer.price + result.instance_update_map["instance_configuration"] = ( + instance_configuration.model_dump_json() + ) + result.instance_update_map["job_provisioning_data"] = ( + job_provisioning_data.model_dump_json() ) - continue + result.instance_update_map["offer"] = instance_offer.model_dump_json() + result.instance_update_map["total_blocks"] = instance_offer.total_blocks + result.instance_update_map["started_at"] = NOW_PLACEHOLDER + + if ( + instance_model.fleet_id is not None + and cluster_context is not None + and cluster_context.is_current_instance_master + ): + # Clean up placement groups that did not end up being used. + result.schedule_pg_deletion_fleet_id = instance_model.fleet_id + if placement_group_model is not None: + result.schedule_pg_deletion_except_id = placement_group_model.id + return result set_status_update( update_map=result.instance_update_map, instance_model=instance_model, - new_status=InstanceStatus.PROVISIONING, - ) - result.instance_update_map["backend"] = backend.TYPE - result.instance_update_map["region"] = instance_offer.region - result.instance_update_map["price"] = instance_offer.price - result.instance_update_map["instance_configuration"] = ( - instance_configuration.model_dump_json() - ) - result.instance_update_map["job_provisioning_data"] = ( - job_provisioning_data.model_dump_json() + new_status=InstanceStatus.TERMINATED, + termination_reason=InstanceTerminationReason.NO_OFFERS, + termination_reason_message="All offers failed" if offers else "No offers found", ) - result.instance_update_map["offer"] = instance_offer.model_dump_json() - result.instance_update_map["total_blocks"] = instance_offer.total_blocks - result.instance_update_map["started_at"] = NOW_PLACEHOLDER - - if ( - instance_model.fleet_id is not None - and cluster_context is not None - and cluster_context.is_current_instance_master - ): - # Clean up placement groups that did not end up being used. - result.schedule_pg_deletion_fleet_id = instance_model.fleet_id - if placement_group_model is not None: - result.schedule_pg_deletion_except_id = placement_group_model.id return result - set_status_update( - update_map=result.instance_update_map, - instance_model=instance_model, - new_status=InstanceStatus.TERMINATED, - termination_reason=InstanceTerminationReason.NO_OFFERS, - termination_reason_message="All offers failed" if offers else "No offers found", - ) - return result - async def _get_cluster_master_context( instance_model: InstanceModel, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py index 7b05d7294b..36c40b8684 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/termination.py @@ -13,7 +13,7 @@ set_status_update, ) from dstack._internal.server.models import InstanceModel -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.runner.pool import ( instance_connection_pool, @@ -35,56 +35,39 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is not None and job_provisioning_data.backend != BackendType.REMOTE: - backend = await backends_services.get_project_backend_by_type( - project=instance_model.project, - backend_type=job_provisioning_data.backend, - ) - if backend is None: + try: + await _terminate_in_backend(instance_model, job_provisioning_data) + except Exception as exc: + first_retry_at = instance_model.first_termination_retry_at + if first_retry_at is None: + first_retry_at = now + result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER + result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER + if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): + if isinstance(exc, NotYetTerminated): + logger.debug( + "Instance %s termination in progress: %s", + instance_model.name, + exc, + ) + else: + logger.warning( + "Failed to terminate instance %s. Will retry. Error: %r", + instance_model.name, + exc, + exc_info=not isinstance(exc, BackendError), + ) + return result + next_step = "Please terminate the instance manually to avoid unexpected charges." + if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): + next_step = "Orphan cleanup will retry deleting the instance." logger.error( - "Failed to terminate instance %s. Backend %s not available.", + "Failed all attempts to terminate instance %s. %s Error: %r", instance_model.name, - job_provisioning_data.backend, + next_step, + exc, + exc_info=not isinstance(exc, BackendError), ) - else: - logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) - try: - await run_async( - backend.compute().terminate_instance, - job_provisioning_data.instance_id, - job_provisioning_data.region, - job_provisioning_data.backend_data, - ) - except Exception as exc: - first_retry_at = instance_model.first_termination_retry_at - if first_retry_at is None: - first_retry_at = now - result.instance_update_map["first_termination_retry_at"] = NOW_PLACEHOLDER - result.instance_update_map["last_termination_retry_at"] = NOW_PLACEHOLDER - if next_termination_retry_at(now) < get_termination_deadline(first_retry_at): - if isinstance(exc, NotYetTerminated): - logger.debug( - "Instance %s termination in progress: %s", - instance_model.name, - exc, - ) - else: - logger.warning( - "Failed to terminate instance %s. Will retry. Error: %r", - instance_model.name, - exc, - exc_info=not isinstance(exc, BackendError), - ) - return result - next_step = "Please terminate the instance manually to avoid unexpected charges." - if _is_to_be_deleted_by_orphan_cleanup(instance_model, job_provisioning_data): - next_step = "Orphan cleanup will retry deleting the instance." - logger.error( - "Failed all attempts to terminate instance %s. %s Error: %r", - instance_model.name, - next_step, - exc, - exc_info=not isinstance(exc, BackendError), - ) if job_provisioning_data is not None: instance_connection_pool.drop_by_jpd(job_provisioning_data) @@ -100,6 +83,29 @@ async def terminate_instance(instance_model: InstanceModel) -> ProcessResult: return result +async def _terminate_in_backend( + instance_model: InstanceModel, job_provisioning_data: JobProvisioningData +): + async with backends_cache.lease_project_backend_by_type( + project=instance_model.project, + backend_type=job_provisioning_data.backend, + ) as backend: + if backend is None: + logger.error( + "Failed to terminate instance %s. Backend %s not available.", + instance_model.name, + job_provisioning_data.backend, + ) + return + logger.debug("Terminating runner instance %s", job_provisioning_data.hostname) + await run_async( + backend.compute().terminate_instance, + job_provisioning_data.instance_id, + job_provisioning_data.region, + job_provisioning_data.backend_data, + ) + + def _is_to_be_deleted_by_orphan_cleanup( instance_model: InstanceModel, job_provisioning_data: JobProvisioningData ) -> bool: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index fb340d4ddc..58eaec1a68 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -84,7 +84,7 @@ VolumeModel, ) from dstack._internal.server.services import events -from dstack._internal.server.services.backends import get_project_backend_by_type_or_error +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.docker import apply_server_docker_defaults from dstack._internal.server.services.fleets import ( can_create_new_cloud_instance_in_fleet, @@ -1886,90 +1886,96 @@ async def _process_volume_attachments( if len(prepared_job_volumes.volume_model_ids) == 0: return _VolumeAttachmentResult(attachments=[], locked_volume_ids=[]) - backend = await get_project_backend_by_type_or_error( - project=project, - backend_type=job_provisioning_data.backend, - ) - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) + async with backends_cache.lease_project_backend_by_type( + project=project, backend_type=job_provisioning_data.backend + ) as backend: + if backend is None: + logger.info("%s: failed to attach volume: backend not available", fmt(job_model)) + return _VolumeAttachmentResult( + attachments=[], + locked_volume_ids=[], + termination_message="Failed to attach volume: backend not available", + ) + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) - volume_models = await _lock_related_volume_models( - item=item, volume_model_ids=prepared_job_volumes.volume_model_ids - ) - if volume_models is None: - return _VolumeAttachmentResult( - attachments=[], - locked_volume_ids=[], - termination_message="Failed to attach volume: Cannot attach a volume locked for processing", + volume_models = await _lock_related_volume_models( + item=item, volume_model_ids=prepared_job_volumes.volume_model_ids ) + if volume_models is None: + return _VolumeAttachmentResult( + attachments=[], + locked_volume_ids=[], + termination_message="Failed to attach volume: Cannot attach a volume locked for processing", + ) - locked_volume_ids = sorted( - { - volume_model.id - for mount_point_volume_models in volume_models - for volume_model in mount_point_volume_models - } - ) - attachments: list[_VolumeAttachmentPayload] = [] - related_volume_lock_owner = _get_related_volume_lock_owner(item.id) - for mount_point_volume_models in volume_models: - for volume_model in mount_point_volume_models: - volume = volume_model_to_volume(volume_model) - try: - if volume_model.deleted: - raise ServerClientError("Cannot attach a deleted volume") - if volume_model.to_be_deleted: - raise ServerClientError("Cannot attach a volume marked for deletion") - if ( - volume_model.lock_expires_at is not None - and volume_model.lock_owner != related_volume_lock_owner - ): - raise ServerClientError("Cannot attach a volume locked for processing") - if ( - job_provisioning_data.get_base_backend() != volume.get_backend() - or job_provisioning_data.region.lower() != volume.get_region().lower() - ): - continue - if volume.provisioning_data is None or not volume.provisioning_data.attachable: - continue - attachment_data = await run_async( - compute.attach_volume, - volume=volume, - provisioning_data=job_provisioning_data, - ) - attachments.append( - _VolumeAttachmentPayload( - volume_id=volume_model.id, - attachment_data=attachment_data.model_dump_json(), - volume_name=volume.name, + locked_volume_ids = sorted( + { + volume_model.id + for mount_point_volume_models in volume_models + for volume_model in mount_point_volume_models + } + ) + attachments: list[_VolumeAttachmentPayload] = [] + related_volume_lock_owner = _get_related_volume_lock_owner(item.id) + for mount_point_volume_models in volume_models: + for volume_model in mount_point_volume_models: + volume = volume_model_to_volume(volume_model) + try: + if volume_model.deleted: + raise ServerClientError("Cannot attach a deleted volume") + if volume_model.to_be_deleted: + raise ServerClientError("Cannot attach a volume marked for deletion") + if ( + volume_model.lock_expires_at is not None + and volume_model.lock_owner != related_volume_lock_owner + ): + raise ServerClientError("Cannot attach a volume locked for processing") + if ( + job_provisioning_data.get_base_backend() != volume.get_backend() + or job_provisioning_data.region.lower() != volume.get_region().lower() + ): + continue + if volume.provisioning_data is None or not volume.provisioning_data.attachable: + continue + attachment_data = await run_async( + compute.attach_volume, + volume=volume, + provisioning_data=job_provisioning_data, ) - ) - break - except ServerClientError as e: - logger.info("%s: failed to attach volume: %s", fmt(job_model), repr(e)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message=f"Failed to attach volume: {e.msg}", - ) - except BackendError as e: - logger.warning("%s: failed to attach volume: %s", fmt(job_model), repr(e)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message=f"Failed to attach volume: {str(e)}", - ) - except Exception: - logger.exception("%s: got exception when attaching volume", fmt(job_model)) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - termination_message="Failed to attach volume: unexpected error", - ) - return _VolumeAttachmentResult( - attachments=attachments, - locked_volume_ids=locked_volume_ids, - ) + attachments.append( + _VolumeAttachmentPayload( + volume_id=volume_model.id, + attachment_data=attachment_data.model_dump_json(), + volume_name=volume.name, + ) + ) + break + except ServerClientError as e: + logger.info("%s: failed to attach volume: %s", fmt(job_model), repr(e)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message=f"Failed to attach volume: {e.msg}", + ) + except BackendError as e: + logger.warning("%s: failed to attach volume: %s", fmt(job_model), repr(e)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message=f"Failed to attach volume: {str(e)}", + ) + except Exception: + logger.exception("%s: got exception when attaching volume", fmt(job_model)) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + termination_message="Failed to attach volume: unexpected error", + ) + return _VolumeAttachmentResult( + attachments=attachments, + locked_volume_ids=locked_volume_ids, + ) async def _lock_related_volume_models( @@ -2430,98 +2436,134 @@ async def _provision_new_capacity( fleet_model=fleet_model, ) multinode = requirements.multinode or is_multinode_job(job) - offers = await get_offers_by_requirements( - project=project, - profile=profile, - requirements=requirements, - exclude_not_available=True, - multinode=multinode, - master_job_provisioning_data=master_job_provisioning_data, - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), - placement_group=placement_group_model_to_placement_group_optional(placement_group_model), - ) - extra_authorized_keys = get_extra_authorized_keys(run.run_spec) - offers_iter = iter(offers) - offers_tried = 0 - offers_taken = 0 - skip_reasons: list[str] = [] - offer_errors: list[_OfferAttemptError] = [] - while offers_tried < settings.MAX_OFFERS_TRIED: - backend_with_offer = next(offers_iter, None) - if backend_with_offer is None: - break - offers_taken += 1 - backend, offer = backend_with_offer - logger.debug( - "%s: trying %s in %s/%s for $%0.4f per hour", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - offer.price, + async with backends_cache.lease_project_backends(project) as backends: + offers = await get_offers_by_requirements( + backends=backends, + profile=profile, + requirements=requirements, + exclude_not_available=True, + multinode=multinode, + master_job_provisioning_data=master_job_provisioning_data, + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), ) - offer_volumes = _get_offer_volumes(volumes, offer) - job_configurations = [ - JobConfiguration(job=job_to_run, volumes=offer_volumes) for job_to_run in jobs - ] - compute = backend.compute() - if master_job_provisioning_data is not None: - offer = get_instance_offer_with_restricted_az( - instance_offer=offer, - master_job_provisioning_data=master_job_provisioning_data, - ) - if ( - # The first real instance in an empty cluster fleet is responsible - # for creating/selecting the placement group. A placeholder alone - # must not suppress that path. - not filter_non_placeholder_instances(fleet_model.instances) - and is_cloud_cluster(fleet_model) - and offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT - and isinstance(compute, ComputeWithPlacementGroupSupport) - and await can_use_placement_groups( - compute=compute, - instance_offer=offer, - reservation=requirements.reservation, - ) - ): - placement_group_model = await find_or_create_suitable_placement_group( - fleet_model=fleet_model, - placement_groups=placement_group_models, - instance_offer=offer, - compute=compute, - ) - if placement_group_model is None: - skip_reasons.append("no compatible placement group") - continue - if placement_group_model.id not in known_placement_group_ids: - new_placement_group_models.append(placement_group_model) - placement_group_models.append(placement_group_model) - known_placement_group_ids.add(placement_group_model.id) - offers_tried += 1 - try: - # Use run_jobs only for a real batch (2+ jobs). A 1-job batch must - # stay on run_job: Slurm gets node_count=1 there, and RunPod's - # Instant Cluster API rejects pod_count=1 (only sizes like 2/4/8). - use_group_provisioning = ( - offer.backend in BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT and len(jobs) > 1 + extra_authorized_keys = get_extra_authorized_keys(run.run_spec) + offers_iter = iter(offers) + offers_tried = 0 + offers_taken = 0 + skip_reasons: list[str] = [] + offer_errors: list[_OfferAttemptError] = [] + while offers_tried < settings.MAX_OFFERS_TRIED: + backend_with_offer = next(offers_iter, None) + if backend_with_offer is None: + break + offers_taken += 1 + backend, offer = backend_with_offer + logger.debug( + "%s: trying %s in %s/%s for $%0.4f per hour", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + offer.price, ) - if use_group_provisioning: - assert isinstance(compute, ComputeWithGroupProvisioningSupport) - compute_group_provisioning_data = await run_async( - compute.run_jobs, + offer_volumes = _get_offer_volumes(volumes, offer) + job_configurations = [ + JobConfiguration(job=job_to_run, volumes=offer_volumes) for job_to_run in jobs + ] + compute = backend.compute() + if master_job_provisioning_data is not None: + offer = get_instance_offer_with_restricted_az( + instance_offer=offer, + master_job_provisioning_data=master_job_provisioning_data, + ) + if ( + # The first real instance in an empty cluster fleet is responsible + # for creating/selecting the placement group. A placeholder alone + # must not suppress that path. + not filter_non_placeholder_instances(fleet_model.instances) + and is_cloud_cluster(fleet_model) + and offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT + and isinstance(compute, ComputeWithPlacementGroupSupport) + and await can_use_placement_groups( + compute=compute, + instance_offer=offer, + reservation=requirements.reservation, + ) + ): + placement_group_model = await find_or_create_suitable_placement_group( + fleet_model=fleet_model, + placement_groups=placement_group_models, + instance_offer=offer, + compute=compute, + ) + if placement_group_model is None: + skip_reasons.append("no compatible placement group") + continue + if placement_group_model.id not in known_placement_group_ids: + new_placement_group_models.append(placement_group_model) + placement_group_models.append(placement_group_model) + known_placement_group_ids.add(placement_group_model.id) + offers_tried += 1 + try: + # Use run_jobs only for a real batch (2+ jobs). A 1-job batch must + # stay on run_job: Slurm gets node_count=1 there, and RunPod's + # Instant Cluster API rejects pod_count=1 (only sizes like 2/4/8). + use_group_provisioning = ( + offer.backend in BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT and len(jobs) > 1 + ) + if use_group_provisioning: + assert isinstance(compute, ComputeWithGroupProvisioningSupport) + compute_group_provisioning_data = await run_async( + compute.run_jobs, + run, + job_configurations, + offer, + project_ssh_public_key, + project_ssh_private_key, + placement_group_model_to_placement_group_optional(placement_group_model), + requirements, + extra_authorized_keys, + ) + return _ProvisionNewCapacityResult( + provisioning_data=compute_group_provisioning_data, + offer=offer, + effective_profile=profile, + placement_group_cleanup=_build_placement_group_cleanup( + fleet_model=fleet_model, + offers_tried=offers_tried, + selected_placement_group_id=( + None if placement_group_model is None else placement_group_model.id + ), + new_placement_group_models=new_placement_group_models, + ), + ) + placeholder_instance = job_model.instance + if ( + placeholder_instance is not None + and is_placeholder_instance(placeholder_instance) + and len(job.job_submissions) > 0 + ): + # `run_job()` sets the placeholder's `provider_ref` on the resources it creates + job.job_submissions[-1].provider_ref = placeholder_instance.provider_ref + job_provisioning_data = await run_async( + compute.run_job, run, - job_configurations, + job, offer, project_ssh_public_key, project_ssh_private_key, + offer_volumes, placement_group_model_to_placement_group_optional(placement_group_model), requirements, extra_authorized_keys, ) return _ProvisionNewCapacityResult( - provisioning_data=compute_group_provisioning_data, + provisioning_data=job_provisioning_data, offer=offer, effective_profile=profile, placement_group_cleanup=_build_placement_group_cleanup( @@ -2533,91 +2575,58 @@ async def _provision_new_capacity( new_placement_group_models=new_placement_group_models, ), ) - placeholder_instance = job_model.instance - if ( - placeholder_instance is not None - and is_placeholder_instance(placeholder_instance) - and len(job.job_submissions) > 0 - ): - # `run_job()` sets the placeholder's `provider_ref` on the resources it creates - job.job_submissions[-1].provider_ref = placeholder_instance.provider_ref - job_provisioning_data = await run_async( - compute.run_job, - run, - job, - offer, - project_ssh_public_key, - project_ssh_private_key, - offer_volumes, - placement_group_model_to_placement_group_optional(placement_group_model), - requirements, - extra_authorized_keys, - ) - return _ProvisionNewCapacityResult( - provisioning_data=job_provisioning_data, - offer=offer, - effective_profile=profile, - placement_group_cleanup=_build_placement_group_cleanup( - fleet_model=fleet_model, - offers_tried=offers_tried, - selected_placement_group_id=( - None if placement_group_model is None else placement_group_model.id - ), - new_placement_group_models=new_placement_group_models, + except SkipOffer as e: + offers_tried -= 1 + skip_reasons.append(str(e) or "offer skipped") + logger.info( + "%s: %s launch in %s/%s skipped: %s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + e, + ) + continue + except BackendError as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) + logger.warning( + "%s: %s launch in %s/%s failed: %s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + repr(e), + ) + continue + except Exception as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) + logger.exception( + "%s: got exception when launching %s in %s/%s", + fmt(job_model), + offer.instance.name, + offer.backend.value, + offer.region, + ) + continue + return _FailedNewCapacityProvisioning( + message=_get_new_capacity_failure_message( + fleet_name=fleet_model.name, + attempts=_NewCapacityAttempts( + total=len(offers), + tried=offers_tried, + skip_reasons=skip_reasons, + errors=offer_errors, + # Offers are left only if the attempt limit, not the offer list, ended the loop. + limit_reached=offers_taken < len(offers), ), - ) - except SkipOffer as e: - offers_tried -= 1 - skip_reasons.append(str(e) or "offer skipped") - logger.info( - "%s: %s launch in %s/%s skipped: %s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - e, - ) - continue - except BackendError as e: - offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) - logger.warning( - "%s: %s launch in %s/%s failed: %s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - repr(e), - ) - continue - except Exception as e: - offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) - logger.exception( - "%s: got exception when launching %s in %s/%s", - fmt(job_model), - offer.instance.name, - offer.backend.value, - offer.region, - ) - continue - return _FailedNewCapacityProvisioning( - message=_get_new_capacity_failure_message( - fleet_name=fleet_model.name, - attempts=_NewCapacityAttempts( - total=len(offers), - tried=offers_tried, - skip_reasons=skip_reasons, - errors=offer_errors, - # Offers are left only if the attempt limit, not the offer list, ended the loop. - limit_reached=offers_taken < len(offers), ), - ), - placement_group_cleanup=_build_placement_group_cleanup( - fleet_model=fleet_model, - offers_tried=offers_tried, - selected_placement_group_id=None, - new_placement_group_models=new_placement_group_models, - ), - ) + placement_group_cleanup=_build_placement_group_cleanup( + fleet_model=fleet_model, + offers_tried=offers_tried, + selected_placement_group_id=None, + new_placement_group_models=new_placement_group_models, + ), + ) _MAX_OFFER_ERROR_LEN = 200 diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index 269e2e10c5..0f7bab7992 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -48,9 +48,9 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events from dstack._internal.server.services import logs as logs_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( emit_instance_status_change_event, get_instance_ssh_private_keys, @@ -964,32 +964,32 @@ async def _detach_volumes_from_job_instance( run_termination_reason: Optional[RunTerminationReason], ) -> _VolumeDetachResult: job_spec = get_job_spec(job_model) - backend = await backends_services.get_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=instance_model.project, backend_type=jpd.backend, - ) - if backend is None: - logger.error( - "Failed to detach volumes from %s. Backend not available.", instance_model.name - ) - return _VolumeDetachResult(all_detached=False) + ) as backend: + if backend is None: + logger.error( + "Failed to detach volumes from %s. Backend not available.", instance_model.name + ) + return _VolumeDetachResult(all_detached=False) - detached_volume_ids = set() - all_detached = True - for volume_model in volume_models: - detached = await _detach_volume_from_job_instance( - backend=backend, - job_model=job_model, - jpd=jpd, - job_spec=job_spec, - instance_model=instance_model, - volume_model=volume_model, - run_termination_reason=run_termination_reason, - ) - if detached: - detached_volume_ids.add(volume_model.id) - else: - all_detached = False + detached_volume_ids = set() + all_detached = True + for volume_model in volume_models: + detached = await _detach_volume_from_job_instance( + backend=backend, + job_model=job_model, + jpd=jpd, + job_spec=job_spec, + instance_model=instance_model, + volume_model=volume_model, + run_termination_reason=run_termination_reason, + ) + if detached: + detached_volume_ids.add(volume_model.id) + else: + all_detached = False return _VolumeDetachResult( all_detached=all_detached, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py index 8406b1a664..d47d99c982 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py @@ -45,8 +45,8 @@ ProjectModel, ProviderRefModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol @@ -382,67 +382,69 @@ async def _process_orphan_cleanup(process_context: _ProcessContext) -> _ProcessR orphan_cleanup_model = process_context.orphan_cleanup_model if not process_context.needs_cleanup: return _ProcessResult(update_map=_OrphanCleanupUpdateMap()) - backend = await backends_services.get_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=orphan_cleanup_model.backend.project, backend_type=orphan_cleanup_model.backend.type, - ) - if backend is None: - logger.warning( - "Failed to clean up orphaned resources in %s/%s: backend is not available", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - compute = backend.compute() - assert isinstance(compute, ComputeWithResourceListingSupport) - listed_at = get_current_datetime() - try: - resource_listing = await run_async(compute.list_resource_groups) - except BackendError as e: - logger.warning( - "Failed to list resource groups in %s/%s due to backend error: %s", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - repr(e), - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - except Exception: - logger.exception( - "Failed to list resource groups in %s/%s due to unexpected error", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - ) - return _get_backoff_cleanup_process_result(orphan_cleanup_model) - for failure in resource_listing.failures: - logger.warning( - "Failed to list resources of %s in %s for orphan cleanup in %s/%s: %s", - failure.backend.value, - failure.region if failure.region is not None else "all regions", - orphan_cleanup_model.backend.project.name, - orphan_cleanup_model.backend.type.value, - failure.message, - ) - resource_groups = resource_listing.resource_groups - owner_records = await _load_owner_records( - project_id=orphan_cleanup_model.backend.project_id, - resource_groups=resource_groups, - ) - decisions = [ - _get_resource_group_decision( - orphan_cleanup_model, resource_group, owner_records, listed_at + ) as backend: + if backend is None: + logger.warning( + "Failed to clean up orphaned resources in %s/%s: backend is not available", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + compute = backend.compute() + assert isinstance(compute, ComputeWithResourceListingSupport) + listed_at = get_current_datetime() + try: + resource_listing = await run_async(compute.list_resource_groups) + except BackendError as e: + logger.warning( + "Failed to list resource groups in %s/%s due to backend error: %s", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + repr(e), + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + except Exception: + logger.exception( + "Failed to list resource groups in %s/%s due to unexpected error", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + ) + return _get_backoff_cleanup_process_result(orphan_cleanup_model) + for failure in resource_listing.failures: + logger.warning( + "Failed to list resources of %s in %s for orphan cleanup in %s/%s: %s", + failure.backend.value, + failure.region if failure.region is not None else "all regions", + orphan_cleanup_model.backend.project.name, + orphan_cleanup_model.backend.type.value, + failure.message, + ) + resource_groups = resource_listing.resource_groups + owner_records = await _load_owner_records( + project_id=orphan_cleanup_model.backend.project_id, + resource_groups=resource_groups, ) - for resource_group in resource_groups - ] - final_decisions: list[_ResourceGroupDecision] = [] - orphan_events: list[_OrphanEvent] = [] - for resource_group, decision in zip(resource_groups, decisions): - if decision == _ResourceGroupDecision.ORPHAN: - decision = await _handle_orphaned_resource_group( - compute, orphan_cleanup_model, resource_group + decisions = [ + _get_resource_group_decision( + orphan_cleanup_model, resource_group, owner_records, listed_at ) - if decision in (_ResourceGroupDecision.ORPHAN, _ResourceGroupDecision.DELETING): - orphan_events.append(_get_orphan_event(resource_group, owner_records, decision)) - final_decisions.append(decision) + for resource_group in resource_groups + ] + final_decisions: list[_ResourceGroupDecision] = [] + orphan_events: list[_OrphanEvent] = [] + for resource_group, decision in zip(resource_groups, decisions): + if decision == _ResourceGroupDecision.ORPHAN: + decision = await _handle_orphaned_resource_group( + compute, orphan_cleanup_model, resource_group + ) + if decision in (_ResourceGroupDecision.ORPHAN, _ResourceGroupDecision.DELETING): + orphan_events.append( + _get_orphan_event(resource_group, owner_records, decision) + ) + final_decisions.append(decision) if any(d is _ResourceGroupDecision.DELETE_FAILED for d in final_decisions): result = _get_backoff_cleanup_process_result(orphan_cleanup_model) elif ( diff --git a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py index ff8439dadc..53c10f8694 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/placement_groups.py @@ -29,7 +29,7 @@ PlacementGroupModel, ProjectModel, ) -from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.placement import placement_group_model_to_placement_group @@ -245,32 +245,33 @@ async def _delete_placement_group( "Failed to delete placement group %s. provisioning_data is None.", placement_group.name ) return _get_deleted_result() - backend = await backends_services.get_project_backend_by_type( + async with backends_cache.lease_project_backend_by_type( project=placement_group_model.project, backend_type=placement_group.provisioning_data.backend, - ) - if backend is None: - # TODO: Retry deletion - logger.error( - "Failed to delete placement group %s. Backend not available. Please delete it manually.", - placement_group.name, - ) - return _get_deleted_result() - compute = backend.compute() - assert isinstance(compute, ComputeWithPlacementGroupSupport) - try: - await run_async(compute.delete_placement_group, placement_group) - except PlacementGroupInUseError: - logger.info( - "Placement group %s is still in use. Skipping deletion for now.", placement_group.name - ) - return _DeleteResult() - except Exception: - # TODO: Retry deletion - logger.exception( - "Got exception when deleting placement group %s. Please delete it manually.", - placement_group.name, - ) + ) as backend: + if backend is None: + # TODO: Retry deletion + logger.error( + "Failed to delete placement group %s. Backend not available. Please delete it manually.", + placement_group.name, + ) + return _get_deleted_result() + compute = backend.compute() + assert isinstance(compute, ComputeWithPlacementGroupSupport) + try: + await run_async(compute.delete_placement_group, placement_group) + except PlacementGroupInUseError: + logger.info( + "Placement group %s is still in use. Skipping deletion for now.", + placement_group.name, + ) + return _DeleteResult() + except Exception: + # TODO: Retry deletion + logger.exception( + "Got exception when deleting placement group %s. Please delete it manually.", + placement_group.name, + ) return _get_deleted_result() diff --git a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py index a5d08ac803..23189fd79d 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/volumes.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/volumes.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import joinedload, load_only from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.volumes import VolumeStatus from dstack._internal.server.background.pipeline_tasks.base import ( NOW_PLACEHOLDER, @@ -34,8 +34,8 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.volumes import ( @@ -306,59 +306,57 @@ class _ProcessResult: async def _process_submitted_volume(volume_model: VolumeModel) -> _ProcessResult: volume = volume_model_to_volume(volume_model) - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.configuration.backend, - overrides=True, - ) - except BackendNotAvailable: - logger.error( - "Failed to process volume %s. Backend %s not available.", - volume.name, - volume.configuration.backend.value, - ) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": "Backend not available", - } - ) - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - try: - if volume.configuration.is_external: - logger.info("Registering external volume %s", volume_model.name) - vpd = await run_async( - compute.register_volume, - volume=volume, + async with backends_cache.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.configuration.backend, + overrides=True, + ) as backend: + if backend is None: + logger.error( + "Failed to process volume %s. Backend %s not available.", + volume.name, + volume.configuration.backend.value, ) - else: - logger.info("Provisioning new volume %s", volume_model.name) - vpd = await run_async( - compute.create_volume, - volume=volume, + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": "Backend not available", + } + ) + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + try: + if volume.configuration.is_external: + logger.info("Registering external volume %s", volume_model.name) + vpd = await run_async( + compute.register_volume, + volume=volume, + ) + else: + logger.info("Provisioning new volume %s", volume_model.name) + vpd = await run_async( + compute.create_volume, + volume=volume, + ) + except BackendError as e: + logger.info("Failed to create volume %s: %s", volume_model.name, repr(e)) + status_message = f"Backend error: {repr(e)}" + if len(e.args) > 0: + status_message = str(e.args[0]) + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": status_message, + } + ) + except Exception as e: + logger.exception("Got exception when creating volume %s", volume_model.name) + return _ProcessResult( + update_map={ + "status": VolumeStatus.FAILED, + "status_message": f"Unexpected error: {repr(e)}", + } ) - except BackendError as e: - logger.info("Failed to create volume %s: %s", volume_model.name, repr(e)) - status_message = f"Backend error: {repr(e)}" - if len(e.args) > 0: - status_message = str(e.args[0]) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": status_message, - } - ) - except Exception as e: - logger.exception("Got exception when creating volume %s", volume_model.name) - return _ProcessResult( - update_map={ - "status": VolumeStatus.FAILED, - "status_message": f"Unexpected error: {repr(e)}", - } - ) logger.info("Added new volume %s", volume_model.name) # Provisioned volumes marked as active since they become available almost immediately in AWS @@ -383,32 +381,30 @@ async def _process_to_be_deleted_volume(volume_model: VolumeModel) -> _ProcessRe f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None." ) return _get_deleted_result() - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.provisioning_data.backend, - ) - except BackendNotAvailable: - # TODO: Retry deletion - logger.error( - f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." - " Please terminate it manually to avoid unexpected charges.", - ) - return _get_deleted_result() - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - try: - await run_async( - compute.delete_volume, - volume=volume, - ) - except Exception: - # TODO: Retry deletion - logger.exception( - "Got exception when deleting volume %s. Please terminate it manually to avoid unexpected charges.", - volume.name, - ) + async with backends_cache.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.provisioning_data.backend, + ) as backend: + if backend is None: + # TODO: Retry deletion + logger.error( + f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + " Please terminate it manually to avoid unexpected charges.", + ) + return _get_deleted_result() + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + try: + await run_async( + compute.delete_volume, + volume=volume, + ) + except Exception: + # TODO: Retry deletion + logger.exception( + "Got exception when deleting volume %s. Please terminate it manually to avoid unexpected charges.", + volume.name, + ) return _get_deleted_result() diff --git a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py index 33aa278303..697133a1c8 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py @@ -30,6 +30,7 @@ collect_prometheus_metrics, delete_prometheus_metrics, ) +from dstack._internal.server.services.backends.cache import close_unused_backends _scheduler = AsyncIOScheduler() @@ -53,6 +54,7 @@ def start_scheduled_tasks() -> AsyncIOScheduler: _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1) _scheduler.add_job(delete_events, IntervalTrigger(minutes=7), max_instances=1) _scheduler.add_job(process_gateways_connections, IntervalTrigger(seconds=15)) + _scheduler.add_job(close_unused_backends, IntervalTrigger(minutes=1), max_instances=1) _scheduler.add_job( process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1 ) diff --git a/src/dstack/_internal/server/routers/backends.py b/src/dstack/_internal/server/routers/backends.py index 8c79993269..36e8e318f3 100644 --- a/src/dstack/_internal/server/routers/backends.py +++ b/src/dstack/_internal/server/routers/backends.py @@ -19,8 +19,8 @@ UpdateBackendYAMLRequest, ) from dstack._internal.server.security.permissions import ProjectAdmin -from dstack._internal.server.services import backends from dstack._internal.server.services.backends import handlers as backends_handlers +from dstack._internal.server.services.backends import store as backends_store from dstack._internal.server.services.config import ( ServerConfigManager, create_backend_config_yaml, @@ -58,7 +58,7 @@ async def create_backend( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.create_backend(session=session, project=project, config=body) + config = await backends_store.create_backend(session=session, project=project, config=body) if settings.SERVER_CONFIG_ENABLED: await ServerConfigManager().sync_config(session=session) return CustomJSONResponse(config) @@ -71,7 +71,7 @@ async def update_backend( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.update_backend(session=session, project=project, config=body) + config = await backends_store.update_backend(session=session, project=project, config=body) if settings.SERVER_CONFIG_ENABLED: await ServerConfigManager().sync_config(session=session) return CustomJSONResponse(config) @@ -101,7 +101,7 @@ async def get_backend_config_info( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): _, project = user_project - config = await backends.get_backend_config(project=project, backend_type=backend_name) + config = await backends_store.get_backend_config(project=project, backend_type=backend_name) if config is None: raise ResourceNotExistsError() return CustomJSONResponse(config) diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 8472375b7a..e69de29bb2 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,530 +0,0 @@ -import asyncio -import json -import time -from collections.abc import Iterable, Iterator -from typing import Callable, Coroutine, Dict, List, Optional, Tuple -from uuid import UUID - -from cachetools import TTLCache -from pydantic import ValidationError -from sqlalchemy import delete, update -from sqlalchemy.ext.asyncio import AsyncSession - -from dstack._internal.core.backends.base.backend import Backend -from dstack._internal.core.backends.base.configurator import ( - Configurator, - StoredBackendRecord, -) -from dstack._internal.core.backends.configurators import ( - get_configurator, - list_available_backend_types, -) -from dstack._internal.core.backends.models import ( - AnyBackendConfigWithCreds, - AnyBackendConfigWithoutCreds, - BackendConfigWithCreds, -) -from dstack._internal.core.errors import ( - BackendAuthError, - BackendError, - BackendInvalidCredentialsError, - BackendNotAvailable, - ResourceExistsError, - ResourceNotExistsError, - ServerClientError, -) -from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.instances import ( - InstanceOfferWithAvailability, -) -from dstack._internal.core.models.runs import Requirements -from dstack._internal.server import settings -from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel -from dstack._internal.server.services.offers import merge_offer_iterables -from dstack._internal.utils.common import run_async -from dstack._internal.utils.logging import get_logger - -logger = get_logger(__name__) - - -def serialize_source_backend_config( - config: AnyBackendConfigWithCreds, -) -> Tuple[str, Optional[str]]: - """Split user-intent backend config into non-sensitive and sensitive JSON blobs.""" - source_config_dict = config.model_dump() - source_auth = source_config_dict.pop("creds", None) - source_auth_json = None if source_auth is None else json.dumps(source_auth) - return json.dumps(source_config_dict), source_auth_json - - -async def create_backend( - session: AsyncSession, - project: ProjectModel, - config: AnyBackendConfigWithCreds, -) -> AnyBackendConfigWithCreds: - configurator = get_configurator(config.type) - if configurator is None: - raise BackendNotAvailable() - backend = await get_project_backend_by_type(project=project, backend_type=configurator.TYPE) - if backend is not None: - raise ResourceExistsError() - backend = await validate_and_create_backend_model( - project=project, configurator=configurator, config=config - ) - session.add(backend) - await session.commit() - return config - - -async def update_backend( - session: AsyncSession, - project: ProjectModel, - config: AnyBackendConfigWithCreds, -) -> AnyBackendConfigWithCreds: - configurator = get_configurator(config.type) - if configurator is None: - raise BackendNotAvailable() - backend_exists = any(configurator.TYPE == b.type for b in project.backends) - if not backend_exists: - raise ResourceNotExistsError() - backend = await validate_and_create_backend_model( - project=project, configurator=configurator, config=config - ) - # FIXME: potentially long write transaction - await session.execute( - update(BackendModel) - .where( - BackendModel.project_id == backend.project_id, - BackendModel.type == backend.type, - ) - .values( - config=backend.config, - auth=backend.auth, - source_config=backend.source_config, - source_auth=backend.source_auth, - ) - ) - return config - - -async def validate_and_create_backend_model( - project: ProjectModel, - configurator: Configurator, - config: AnyBackendConfigWithCreds, -) -> BackendModel: - # Configurators may mutate `config` while building the effective stored backend config, - # so capture the user-intent payload before validation/create_backend runs. - source_config, source_auth = serialize_source_backend_config(config) - await run_async( - configurator.validate_config, config, default_creds_enabled=settings.DEFAULT_CREDS_ENABLED - ) - backend_record = await run_async( - configurator.create_backend, - project_name=project.name, - config=config, - ) - return BackendModel( - project_id=project.id, - type=configurator.TYPE, - config=backend_record.config, - auth=DecryptedString(plaintext=backend_record.auth), - source_config=source_config, - source_auth=None if source_auth is None else DecryptedString(plaintext=source_auth), - ) - - -async def get_backend_config( - project: ProjectModel, - backend_type: BackendType, -) -> Optional[AnyBackendConfigWithCreds]: - configurator = get_configurator(backend_type) - if configurator is None: - raise BackendNotAvailable() - for backend_model in project.backends: - if not backend_model.auth.decrypted: - logger.warning( - "Failed to decrypt creds for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - if backend_model.type == backend_type: - return get_backend_config_with_creds_from_backend_model(configurator, backend_model) - return None - - -async def get_source_backend_config( - project: ProjectModel, - backend_type: BackendType, -) -> Optional[AnyBackendConfigWithCreds]: - backend_model = await get_project_backend_model_by_type(project, backend_type) - if backend_model is None: - return None - return get_source_backend_config_from_backend_model(backend_model) - - -def get_backend_config_with_creds_from_backend_model( - configurator: Configurator, - backend_model: BackendModel, -) -> AnyBackendConfigWithCreds: - backend_record = get_stored_backend_record(backend_model) - backend_config = configurator.get_backend_config_with_creds(backend_record) - return backend_config - - -def get_backend_config_without_creds_from_backend_model( - configurator: Configurator, - backend_model: BackendModel, -) -> AnyBackendConfigWithoutCreds: - backend_record = get_stored_backend_record(backend_model) - backend_config = configurator.get_backend_config_without_creds(backend_record) - return backend_config - - -def get_source_backend_config_from_backend_model( - backend_model: BackendModel, -) -> Optional[AnyBackendConfigWithCreds]: - """Reconstruct user-intent backend config from `source_config`/`source_auth`.""" - - if backend_model.source_config is None: - return None - try: - source_config_dict = json.loads(backend_model.source_config) - except ValueError: - logger.warning( - "Failed to parse source config for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - if backend_model.source_auth is not None: - if not backend_model.source_auth.decrypted: - logger.warning( - "Failed to decrypt source creds for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - try: - source_config_dict["creds"] = json.loads( - backend_model.source_auth.get_plaintext_or_error() - ) - except ValueError: - logger.warning( - "Failed to parse source creds for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - try: - return BackendConfigWithCreds.model_validate(source_config_dict).root - except ValidationError: - logger.warning( - "Failed to validate source config for %s backend. Falling back to stored config.", - backend_model.type.value, - ) - return None - - -def get_stored_backend_record(backend_model: BackendModel) -> StoredBackendRecord: - return StoredBackendRecord( - config=backend_model.config, - auth=backend_model.auth.get_plaintext_or_error(), - project_id=backend_model.project_id, - backend_id=backend_model.id, - ) - - -async def delete_backends( - session: AsyncSession, - project: ProjectModel, - backends_types: List[BackendType], -): - if BackendType.DSTACK in backends_types: - raise ServerClientError("Cannot delete dstack backend") - current_backends_types = set(b.type for b in project.backends) - deleted_backends_types = current_backends_types.intersection(backends_types) - if len(deleted_backends_types) == 0: - return - # FIXME: potentially long write transaction - # Not urgent since backend deletion is a rare operation - await session.execute( - delete(BackendModel).where( - BackendModel.type.in_(deleted_backends_types), - BackendModel.project_id == project.id, - ) - ) - logger.info( - "Deleted backends %s in project %s", - [b.value for b in deleted_backends_types], - project.name, - ) - - -BackendTuple = Tuple[BackendModel, Backend] - - -_BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} -_BACKENDS_CACHE = TTLCache[UUID, Dict[BackendType, BackendTuple]](maxsize=1000, ttl=300) - - -def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: - return _BACKENDS_CACHE_LOCKS.setdefault(project_id, asyncio.Lock()) - - -async def get_project_backends_with_models(project: ProjectModel) -> List[BackendTuple]: - async with _get_project_cache_lock(project.id): - key = project.id - project_backends = _BACKENDS_CACHE.get(key, {}) - to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] - for backend_model in project.backends: - cached_backend = project_backends.get(backend_model.type) - if ( - cached_backend is not None - and cached_backend[0].config == backend_model.config - and cached_backend[0].auth == backend_model.auth - ): - continue - configurator = get_configurator(backend_model.type) - if configurator is None: - logger.warning( - "Missing dependencies for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - if not backend_model.auth.decrypted: - logger.warning( - "Failed to decrypt creds for %s backend. Backend will be ignored.", - backend_model.type.value, - ) - continue - backend_record = get_stored_backend_record(backend_model) - to_init.append((backend_model, configurator, backend_record)) - - if to_init: - t0 = time.time() - tasks = [ - _get_backend_tracked(configurator, backend_record) - for _, configurator, backend_record in to_init - ] - results = await asyncio.gather(*tasks, return_exceptions=True) - initialized_results = [] - for (backend_model, _, _), result in zip(to_init, results): - if isinstance(result, BaseException): - if isinstance(result, (BackendInvalidCredentialsError, BackendAuthError)): - logger.warning( - "Credentials for %s backend are invalid. Backend will be ignored.", - backend_model.type.value, - ) - else: - logger.error( - "Failed to initialize %s backend. Backend will be ignored.", - backend_model.type.value, - exc_info=result, - ) - else: - backend, duration = result - project_backends[backend_model.type] = (backend_model, backend) - initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") - logger.debug( - "Initialized %d backends in %.1fs: %s", - len(initialized_results), - time.time() - t0, - ", ".join(initialized_results), - ) - - # `__setitem__()` will also expire the cache. - # Note that there is no global cache lock so a race condition is possible: - # one coroutine updates/re-assigns backends expired by another coroutine. - # This is ok since the only effect is that project's cache gets restored. - _BACKENDS_CACHE[key] = project_backends - return list(project_backends.values()) - - -async def _get_backend_tracked( - configurator: Configurator, backend_record: StoredBackendRecord -) -> Tuple[Backend, float]: - t = time.time() - backend = await run_async(configurator.get_backend, backend_record) - return backend, time.time() - t - - -_get_project_backend_with_model_by_type = None - - -def set_get_project_backend_with_model_by_type( - func: Callable[[ProjectModel, BackendType], Coroutine[None, None, Optional[BackendTuple]]], -): - """ - Overrides `get_project_effective_backend_with_model_by_type` with `func`. - Then get_project_backend_* functions can pass overrides=True to call `func` - This can be used if a backend needs to be replaced with another backend. - For example, DstackBackend in dstack Sky can be used in place of other backends. - """ - global _get_project_backend_with_model_by_type - _get_project_backend_with_model_by_type = func - - -async def get_project_backend_with_model_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Optional[BackendTuple]: - if overrides and _get_project_backend_with_model_by_type is not None: - return await _get_project_backend_with_model_by_type(project, backend_type) - backends_with_models = await get_project_backends_with_models(project=project) - for backend_model, backend in backends_with_models: - if backend.TYPE == backend_type: - return backend_model, backend - return None - - -async def get_project_backend_with_model_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> BackendTuple: - backend_with_model = await get_project_backend_with_model_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend_with_model is None: - raise BackendNotAvailable() - return backend_with_model - - -async def get_project_backends(project: ProjectModel) -> List[Backend]: - backends_with_models = await get_project_backends_with_models(project) - backends = [b for _, b in backends_with_models] - return backends - - -async def get_project_backend_by_type( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Optional[Backend]: - backend_with_model = await get_project_backend_with_model_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend_with_model is None: - return None - return backend_with_model[1] - - -async def get_project_backend_by_type_or_error( - project: ProjectModel, - backend_type: BackendType, - overrides: bool = False, -) -> Backend: - backend = await get_project_backend_by_type( - project=project, - backend_type=backend_type, - overrides=overrides, - ) - if backend is None: - raise BackendNotAvailable() - return backend - - -async def get_project_backend_model_by_type( - project: ProjectModel, backend_type: BackendType -) -> Optional[BackendModel]: - for backend in project.backends: - if backend.type == backend_type: - return backend - return None - - -async def get_project_backend_model_by_type_or_error( - project: ProjectModel, backend_type: BackendType -) -> BackendModel: - backend_model = await get_project_backend_model_by_type( - project=project, backend_type=backend_type - ) - if backend_model is None: - raise BackendNotAvailable() - return backend_model - - -async def get_project_backend_with_model_by_id( - project: ProjectModel, backend_id: UUID -) -> Optional[BackendTuple]: - backends_with_models = await get_project_backends_with_models(project=project) - for backend_model, backend in backends_with_models: - if backend_model.id == backend_id: - return backend_model, backend - return None - - -async def get_project_backend_with_model_by_id_or_error( - project: ProjectModel, backend_id: UUID -) -> BackendTuple: - backend_with_model = await get_project_backend_with_model_by_id( - project=project, backend_id=backend_id - ) - if backend_with_model is None: - raise BackendNotAvailable() - return backend_with_model - - -async def get_backend_offers( - backends: List[Backend], - requirements: Requirements, - full_offers: bool, - unallocated_resources: bool, - exclude_not_available: bool = False, -) -> Iterable[Tuple[Backend, InstanceOfferWithAvailability]]: - """ - Returns a lazy iterable of backend offers satisfying `requirements` sorted by price. - """ - - def get_filtered_offers_with_backends( - backend: Backend, - offers: Iterable[InstanceOfferWithAvailability], - ) -> Iterator[Tuple[Backend, InstanceOfferWithAvailability]]: - for offer in offers: - if not exclude_not_available or offer.availability.is_available(): - yield (backend, offer) - - logger.debug("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends]) - tasks = [ - run_async(get_offers_tracked, backend, requirements, full_offers, unallocated_resources) - for backend in backends - ] - offers_by_backend: list[Iterable[tuple[Backend, InstanceOfferWithAvailability]]] = [] - for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)): - if isinstance(result, BackendError): - logger.warning( - "Failed to get offers from backend %s: %s", - backend.TYPE, - repr(result), - ) - continue - elif isinstance(result, BaseException): - logger.error( - "Got exception when requesting offers from backend %s", - backend.TYPE, - exc_info=result, - ) - continue - offers_by_backend.append(get_filtered_offers_with_backends(backend, result)) - return merge_offer_iterables(*offers_by_backend) - - -def check_backend_type_available(backend_type: BackendType): - if backend_type not in list_available_backend_types(): - raise BackendNotAvailable( - f"Backend {backend_type.value} not available." - " Ensure that backend dependencies are installed." - f" Available backends: {[b.value for b in list_available_backend_types()]}." - ) - - -def get_offers_tracked( - backend: Backend, requirements: Requirements, full_offers: bool, unallocated_resources: bool -) -> Iterator[InstanceOfferWithAvailability]: - start = time.time() - res = backend.compute().get_offers(requirements, full_offers, unallocated_resources) - duration = time.time() - start - logger.debug("Got offers from %s in %.6fs", backend.TYPE.value, duration) - return res diff --git a/src/dstack/_internal/server/services/backends/cache.py b/src/dstack/_internal/server/services/backends/cache.py new file mode 100644 index 0000000000..17b0163e89 --- /dev/null +++ b/src/dstack/_internal/server/services/backends/cache.py @@ -0,0 +1,329 @@ +""" +Caches project backends and controls their lifetime. + +Backends are cached per project because creating them is expensive, e.g. creating API clients +and filling offers caches. Backends may also hold resources that garbage collection does not +free, e.g. SDK clients that run background threads (see `Compute.close()`), so the cache +closes every backend explicitly once it is no longer used. + +Callers use backends through leases (`lease_project_backend*()`) and must not use a backend +or its compute after the lease is released. This lets the cache know when a backend is unused: +a backend is never closed while it is leased. + +A cached backend is either active or retired. New leases use only active backends. +A backend is retired if its config changes (a new backend replaces it), if it is deleted +from the project, or if it is not leased for `_BACKEND_IDLE_TTL`. A retired backend may stay +leased by operations that started before it was retired, and they keep using it until they finish. + +Retired backends are closed by `close_unused_backends()`, which runs periodically, once they +are not leased. Leases never close backends: closing does not delay callers, and releasing +a lease does not await, so cancellation cannot interrupt it. + +Cancelling a task that leases backends has two known problems: + +* Backends are initialized and used in executor threads (`run_async()`), which cancellation + does not interrupt. If the task is cancelled while a backend is in use, its lease is + released while the thread still uses the backend, so a retired backend can be closed under it. +* If the task is cancelled while backends are initialized for its lease, the initialized + backends are not cached, so they are never closed. + +They are accepted because such tasks are only cancelled on server shutdown, when pipelines +cancel their workers, and backends are not closed on shutdown: the process exits anyway. +If code that leases backends is ever cancelled in other cases, e.g. by `asyncio.wait_for()` +around it, both problems can happen while the server runs. Then the lease must not be released +until the executor work finishes (e.g. shield `run_async()` and re-raise the cancellation after +it completes), and backends initialized for a cancelled lease must still be cached. +""" + +import asyncio +import time +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import Callable, Coroutine, Dict, List, Optional, Tuple +from uuid import UUID + +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.backends.base.configurator import ( + Configurator, + StoredBackendRecord, +) +from dstack._internal.core.backends.configurators import ( + get_configurator, +) +from dstack._internal.core.errors import ( + BackendAuthError, + BackendInvalidCredentialsError, +) +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.server.services.backends.store import get_stored_backend_record +from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +BackendTuple = Tuple[BackendModel, Backend] +BackendSelector = Callable[ + [ProjectModel, BackendType, List[BackendTuple]], + Coroutine[None, None, Optional[BackendTuple]], +] +""" +Selects a backend of the given type from the project backends, +see `set_select_project_backend_with_model_by_type()`. +Must return one of the given backends or `None`. +""" + + +@dataclass +class _CachedBackend: + model: BackendModel + backend: Backend + leases: int = 0 + last_used: float = field(default_factory=time.monotonic) + + +@dataclass +class _ProjectBackends: + active: Dict[BackendType, _CachedBackend] = field(default_factory=dict) + retired: List[_CachedBackend] = field(default_factory=list) + + +_BACKEND_IDLE_TTL = 300 +_BACKENDS_CACHE_LOCKS: Dict[UUID, asyncio.Lock] = {} +_BACKENDS_CACHE: Dict[UUID, _ProjectBackends] = {} +""" +Active and retired backends by project ID. Retired backends stay here until they are closed. +""" + + +def _get_project_cache_lock(project_id: UUID) -> asyncio.Lock: + return _BACKENDS_CACHE_LOCKS.setdefault(project_id, asyncio.Lock()) + + +@asynccontextmanager +async def lease_project_backends(project: ProjectModel) -> AsyncGenerator[List[Backend], None]: + cached_backends = await _acquire_project_backends(project) + try: + yield [c.backend for c in cached_backends] + finally: + _release_backends(cached_backends) + + +_select_project_backend_with_model_by_type: Optional[BackendSelector] = None + + +def set_select_project_backend_with_model_by_type(func: BackendSelector): + """ + Overrides how `lease_project_backend_by_type()` selects a backend from the project backends + if `overrides=True` is passed. This can be used if a backend needs to be replaced with + another backend. For example, DstackBackend in dstack Sky can be used in place of + other backends. + """ + global _select_project_backend_with_model_by_type + _select_project_backend_with_model_by_type = func + + +@asynccontextmanager +async def lease_project_backend_by_type( + project: ProjectModel, + backend_type: BackendType, + overrides: bool = False, +) -> AsyncGenerator[Optional[Backend], None]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + if overrides and _select_project_backend_with_model_by_type is not None: + return await _select_project_backend_with_model_by_type( + project, backend_type, backends_with_models + ) + for backend_model, backend in backends_with_models: + if backend.TYPE == backend_type: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend: + yield backend + + +@asynccontextmanager +async def lease_project_backend_by_id( + project: ProjectModel, backend_id: UUID +) -> AsyncGenerator[Optional[Backend], None]: + async def select(backends_with_models: List[BackendTuple]) -> Optional[BackendTuple]: + for backend_model, backend in backends_with_models: + if backend_model.id == backend_id: + return backend_model, backend + return None + + async with _lease_selected_project_backend(project, select) as backend: + yield backend + + +async def close_unused_backends(): + """ + Retires backends that are not leased for `_BACKEND_IDLE_TTL` + and closes retired backends that are not leased. + """ + idle_since = time.monotonic() - _BACKEND_IDLE_TTL + to_close: List[_CachedBackend] = [] + for project_id in list(_BACKENDS_CACHE): + async with _get_project_cache_lock(project_id): + project_backends = _BACKENDS_CACHE.get(project_id) + if project_backends is None: + continue + for backend_type, cached_backend in list(project_backends.active.items()): + if cached_backend.leases == 0 and cached_backend.last_used < idle_since: + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) + to_close.extend(c for c in project_backends.retired if c.leases == 0) + project_backends.retired = [c for c in project_backends.retired if c.leases > 0] + if not project_backends.active and not project_backends.retired: + del _BACKENDS_CACHE[project_id] + await _close_backends(to_close) + + +@asynccontextmanager +async def _lease_selected_project_backend( + project: ProjectModel, + select: Callable[[List[BackendTuple]], Coroutine[None, None, Optional[BackendTuple]]], +) -> AsyncGenerator[Optional[Backend], None]: + """ + Leases the backend chosen by `select` from the project backends. + + `select` may do I/O, so it runs outside of the cache lock: otherwise it would block + other leases of the project and could deadlock if it leased backends itself. Then the + selected backend could be retired and closed before it is leased, so all project backends + are leased while `select` runs, and the ones not selected are released after it. + They are leased without updating `last_used` so that only the selected backend + counts as used for idle retirement. + """ + cached_backends = await _acquire_project_backends(project, touch=False) + selected: Optional[_CachedBackend] = None + try: + backend_with_model = await select([(c.model, c.backend) for c in cached_backends]) + if backend_with_model is not None: + selected = next( + (c for c in cached_backends if c.backend is backend_with_model[1]), None + ) + if selected is None: + raise ValueError("`select` must return one of the given backends") + finally: + _release_backends([c for c in cached_backends if c is not selected], touch=False) + if selected is None: + yield None + return + selected.last_used = time.monotonic() + try: + yield selected.backend + finally: + _release_backends([selected]) + + +async def _acquire_project_backends( + project: ProjectModel, touch: bool = True +) -> List[_CachedBackend]: + async with _get_project_cache_lock(project.id): + project_backends = _BACKENDS_CACHE.setdefault(project.id, _ProjectBackends()) + backend_models = {backend_model.type: backend_model for backend_model in project.backends} + for backend_type, cached_backend in list(project_backends.active.items()): + backend_model = backend_models.get(backend_type) + if ( + backend_model is None + or cached_backend.model.config != backend_model.config + or cached_backend.model.auth != backend_model.auth + ): + del project_backends.active[backend_type] + project_backends.retired.append(cached_backend) + to_init: List[Tuple[BackendModel, Configurator, StoredBackendRecord]] = [] + for backend_model in project.backends: + if backend_model.type in project_backends.active: + continue + configurator = get_configurator(backend_model.type) + if configurator is None: + logger.warning( + "Missing dependencies for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + if not backend_model.auth.decrypted: + logger.warning( + "Failed to decrypt creds for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + backend_record = get_stored_backend_record(backend_model) + to_init.append((backend_model, configurator, backend_record)) + + if to_init: + t0 = time.time() + tasks = [ + _get_backend_tracked(configurator, backend_record) + for _, configurator, backend_record in to_init + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + initialized_results = [] + for (backend_model, _, _), result in zip(to_init, results): + if isinstance(result, BaseException): + if isinstance(result, (BackendInvalidCredentialsError, BackendAuthError)): + logger.warning( + "Credentials for %s backend are invalid. Backend will be ignored.", + backend_model.type.value, + ) + else: + logger.error( + "Failed to initialize %s backend. Backend will be ignored.", + backend_model.type.value, + exc_info=result, + ) + else: + backend, duration = result + project_backends.active[backend_model.type] = _CachedBackend( + model=backend_model, backend=backend + ) + initialized_results.append(f"{backend_model.type.value}={duration:.1f}s") + logger.debug( + "Initialized %d backends in %.1fs: %s", + len(initialized_results), + time.time() - t0, + ", ".join(initialized_results), + ) + + cached_backends = list(project_backends.active.values()) + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases += 1 + if touch: + cached_backend.last_used = now + return cached_backends + + +def _release_backends(cached_backends: List[_CachedBackend], touch: bool = True): + # Releasing does not await, so it cannot be interrupted by cancellation + now = time.monotonic() + for cached_backend in cached_backends: + cached_backend.leases -= 1 + if touch: + cached_backend.last_used = now + + +async def _close_backends(cached_backends: List[_CachedBackend]): + for cached_backend, result in zip( + cached_backends, + await asyncio.gather( + *(run_async(c.backend.close) for c in cached_backends), return_exceptions=True + ), + ): + if isinstance(result, BaseException): + logger.error( + "Failed to close %s backend", + cached_backend.model.type.value, + exc_info=result, + ) + + +async def _get_backend_tracked( + configurator: Configurator, backend_record: StoredBackendRecord +) -> Tuple[Backend, float]: + t = time.time() + backend = await run_async(configurator.get_backend, backend_record) + return backend, time.time() - t diff --git a/src/dstack/_internal/server/services/backends/handlers.py b/src/dstack/_internal/server/services/backends/handlers.py index f3f5bab68c..0c1778bba7 100644 --- a/src/dstack/_internal/server/services/backends/handlers.py +++ b/src/dstack/_internal/server/services/backends/handlers.py @@ -5,7 +5,7 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.server.models import ProjectModel -from dstack._internal.server.services.backends import delete_backends +from dstack._internal.server.services.backends.store import delete_backends from dstack._internal.server.services.fleets import list_project_fleet_models from dstack._internal.server.services.volumes import list_project_volumes from dstack._internal.utils.logging import get_logger diff --git a/src/dstack/_internal/server/services/backends/store.py b/src/dstack/_internal/server/services/backends/store.py new file mode 100644 index 0000000000..9726401a4c --- /dev/null +++ b/src/dstack/_internal/server/services/backends/store.py @@ -0,0 +1,262 @@ +import json +from typing import List, Optional, Tuple + +from pydantic import ValidationError +from sqlalchemy import delete, update +from sqlalchemy.ext.asyncio import AsyncSession + +from dstack._internal.core.backends.base.configurator import ( + Configurator, + StoredBackendRecord, +) +from dstack._internal.core.backends.configurators import ( + get_configurator, +) +from dstack._internal.core.backends.models import ( + AnyBackendConfigWithCreds, + AnyBackendConfigWithoutCreds, + BackendConfigWithCreds, +) +from dstack._internal.core.errors import ( + BackendNotAvailable, + ResourceExistsError, + ResourceNotExistsError, + ServerClientError, +) +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server import settings +from dstack._internal.server.models import BackendModel, DecryptedString, ProjectModel +from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +def serialize_source_backend_config( + config: AnyBackendConfigWithCreds, +) -> Tuple[str, Optional[str]]: + """Split user-intent backend config into non-sensitive and sensitive JSON blobs.""" + source_config_dict = config.model_dump() + source_auth = source_config_dict.pop("creds", None) + source_auth_json = None if source_auth is None else json.dumps(source_auth) + return json.dumps(source_config_dict), source_auth_json + + +def get_backend_config_with_creds_from_backend_model( + configurator: Configurator, + backend_model: BackendModel, +) -> AnyBackendConfigWithCreds: + backend_record = get_stored_backend_record(backend_model) + backend_config = configurator.get_backend_config_with_creds(backend_record) + return backend_config + + +def get_backend_config_without_creds_from_backend_model( + configurator: Configurator, + backend_model: BackendModel, +) -> AnyBackendConfigWithoutCreds: + backend_record = get_stored_backend_record(backend_model) + backend_config = configurator.get_backend_config_without_creds(backend_record) + return backend_config + + +def get_source_backend_config_from_backend_model( + backend_model: BackendModel, +) -> Optional[AnyBackendConfigWithCreds]: + """Reconstruct user-intent backend config from `source_config`/`source_auth`.""" + + if backend_model.source_config is None: + return None + try: + source_config_dict = json.loads(backend_model.source_config) + except ValueError: + logger.warning( + "Failed to parse source config for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + if backend_model.source_auth is not None: + if not backend_model.source_auth.decrypted: + logger.warning( + "Failed to decrypt source creds for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + try: + source_config_dict["creds"] = json.loads( + backend_model.source_auth.get_plaintext_or_error() + ) + except ValueError: + logger.warning( + "Failed to parse source creds for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + try: + return BackendConfigWithCreds.model_validate(source_config_dict).root + except ValidationError: + logger.warning( + "Failed to validate source config for %s backend. Falling back to stored config.", + backend_model.type.value, + ) + return None + + +def get_stored_backend_record(backend_model: BackendModel) -> StoredBackendRecord: + return StoredBackendRecord( + config=backend_model.config, + auth=backend_model.auth.get_plaintext_or_error(), + project_id=backend_model.project_id, + backend_id=backend_model.id, + ) + + +async def get_project_backend_model_by_type( + project: ProjectModel, backend_type: BackendType +) -> Optional[BackendModel]: + for backend in project.backends: + if backend.type == backend_type: + return backend + return None + + +async def get_project_backend_model_by_type_or_error( + project: ProjectModel, backend_type: BackendType +) -> BackendModel: + backend_model = await get_project_backend_model_by_type( + project=project, backend_type=backend_type + ) + if backend_model is None: + raise BackendNotAvailable() + return backend_model + + +async def validate_and_create_backend_model( + project: ProjectModel, + configurator: Configurator, + config: AnyBackendConfigWithCreds, +) -> BackendModel: + # Configurators may mutate `config` while building the effective stored backend config, + # so capture the user-intent payload before validation/create_backend runs. + source_config, source_auth = serialize_source_backend_config(config) + await run_async( + configurator.validate_config, config, default_creds_enabled=settings.DEFAULT_CREDS_ENABLED + ) + backend_record = await run_async( + configurator.create_backend, + project_name=project.name, + config=config, + ) + return BackendModel( + project_id=project.id, + type=configurator.TYPE, + config=backend_record.config, + auth=DecryptedString(plaintext=backend_record.auth), + source_config=source_config, + source_auth=None if source_auth is None else DecryptedString(plaintext=source_auth), + ) + + +async def create_backend( + session: AsyncSession, + project: ProjectModel, + config: AnyBackendConfigWithCreds, +) -> AnyBackendConfigWithCreds: + configurator = get_configurator(config.type) + if configurator is None: + raise BackendNotAvailable() + backend_exists = any(configurator.TYPE == b.type for b in project.backends) + if backend_exists: + raise ResourceExistsError() + backend = await validate_and_create_backend_model( + project=project, configurator=configurator, config=config + ) + session.add(backend) + await session.commit() + return config + + +async def update_backend( + session: AsyncSession, + project: ProjectModel, + config: AnyBackendConfigWithCreds, +) -> AnyBackendConfigWithCreds: + configurator = get_configurator(config.type) + if configurator is None: + raise BackendNotAvailable() + backend_exists = any(configurator.TYPE == b.type for b in project.backends) + if not backend_exists: + raise ResourceNotExistsError() + backend = await validate_and_create_backend_model( + project=project, configurator=configurator, config=config + ) + # FIXME: potentially long write transaction + await session.execute( + update(BackendModel) + .where( + BackendModel.project_id == backend.project_id, + BackendModel.type == backend.type, + ) + .values( + config=backend.config, + auth=backend.auth, + source_config=backend.source_config, + source_auth=backend.source_auth, + ) + ) + return config + + +async def get_backend_config( + project: ProjectModel, + backend_type: BackendType, +) -> Optional[AnyBackendConfigWithCreds]: + configurator = get_configurator(backend_type) + if configurator is None: + raise BackendNotAvailable() + for backend_model in project.backends: + if not backend_model.auth.decrypted: + logger.warning( + "Failed to decrypt creds for %s backend. Backend will be ignored.", + backend_model.type.value, + ) + continue + if backend_model.type == backend_type: + return get_backend_config_with_creds_from_backend_model(configurator, backend_model) + return None + + +async def get_source_backend_config( + project: ProjectModel, + backend_type: BackendType, +) -> Optional[AnyBackendConfigWithCreds]: + backend_model = await get_project_backend_model_by_type(project, backend_type) + if backend_model is None: + return None + return get_source_backend_config_from_backend_model(backend_model) + + +async def delete_backends( + session: AsyncSession, + project: ProjectModel, + backends_types: List[BackendType], +): + if BackendType.DSTACK in backends_types: + raise ServerClientError("Cannot delete dstack backend") + current_backends_types = set(b.type for b in project.backends) + deleted_backends_types = current_backends_types.intersection(backends_types) + if len(deleted_backends_types) == 0: + return + # FIXME: potentially long write transaction + # Not urgent since backend deletion is a rare operation + await session.execute( + delete(BackendModel).where( + BackendModel.type.in_(deleted_backends_types), + BackendModel.project_id == project.id, + ) + ) + logger.info( + "Deleted backends %s in project %s", + [b.value for b in deleted_backends_types], + project.name, + ) diff --git a/src/dstack/_internal/server/services/config.py b/src/dstack/_internal/server/services/config.py index bba6e5d1fa..223523ffe0 100644 --- a/src/dstack/_internal/server/services/config.py +++ b/src/dstack/_internal/server/services/config.py @@ -21,9 +21,9 @@ from dstack._internal.core.models.common import CoreModel from dstack._internal.server import settings from dstack._internal.server.models import ProjectModel, UserModel -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import encryption as encryption_services from dstack._internal.server.services import projects as projects_services +from dstack._internal.server.services.backends import store as backends_store from dstack._internal.server.services.backends.handlers import delete_backends_safe from dstack._internal.server.services.encryption import AnyEncryptionKeyConfig from dstack._internal.server.services.permissions import ( @@ -145,7 +145,7 @@ async def _apply_project_config( backends_to_delete.difference_update([backend_type]) backend_exists = any(backend_type == b.type for b in project.backends) try: - current_backend_config = await backends_services.get_backend_config( + current_backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type, ) @@ -157,7 +157,7 @@ async def _apply_project_config( ) continue if current_backend_config is not None: - current_source_backend_config = await backends_services.get_source_backend_config( + current_source_backend_config = await backends_store.get_source_backend_config( project=project, backend_type=backend_type, ) @@ -170,10 +170,10 @@ async def _apply_project_config( # Update backend in this case. if current_backend_config is None and not backend_exists: apply_action = "create" - apply_func = backends_services.create_backend + apply_func = backends_store.create_backend else: apply_action = "update" - apply_func = backends_services.update_backend + apply_func = backends_store.update_backend try: await apply_func(session=session, project=project, config=backend_config) except Exception as e: @@ -204,7 +204,7 @@ async def _init_config(self, session: AsyncSession) -> Optional[ServerConfig]: for ( backend_type ) in dstack._internal.core.backends.configurators.list_available_backend_types(): - backend_config = await backends_services.get_backend_config( + backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type ) if backend_config is not None: @@ -232,7 +232,7 @@ def _save_config(self, config: ServerConfig): async def get_backend_config_yaml( project: ProjectModel, backend_type: BackendType ) -> BackendInfoYAML: - backend_config = await backends_services.get_backend_config( + backend_config = await backends_store.get_backend_config( project=project, backend_type=backend_type ) if backend_config is None: @@ -250,7 +250,7 @@ async def create_backend_config_yaml( config_yaml: str, ): config = config_yaml_to_backend_config(config_yaml) - await backends_services.create_backend(session=session, project=project, config=config) + await backends_store.create_backend(session=session, project=project, config=config) async def update_backend_config_yaml( @@ -259,7 +259,7 @@ async def update_backend_config_yaml( config_yaml: str, ): config = config_yaml_to_backend_config(config_yaml) - await backends_services.update_backend(session=session, project=project, config=config) + await backends_store.update_backend(session=session, project=project, config=config) def config_yaml_to_backend_config(config_yaml: str) -> AnyBackendConfigWithCreds: diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index ac677b4948..7be4bded4f 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -69,6 +69,7 @@ from dstack._internal.server.services import events from dstack._internal.server.services import instances as instances_services from dstack._internal.server.services import offers as offers_services +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import ( get_instance_remote_connection_info, is_placeholder_instance, @@ -460,16 +461,17 @@ async def get_plan( include_only_create_instance_supported_backends = True if nodes is not None: include_only_create_instance_supported_backends = nodes.target != 0 - offers_with_backends = await get_fleet_offers( - project=project, - profile=effective_spec.merged_profile, - requirements=requirements, - fleet_spec=effective_spec, - blocks=effective_spec.configuration.blocks, - include_only_create_instance_supported_backends=( - include_only_create_instance_supported_backends - ), - ) + async with backends_cache.lease_project_backends(project) as backends: + offers_with_backends = await get_fleet_offers( + backends=backends, + profile=effective_spec.merged_profile, + requirements=requirements, + fleet_spec=effective_spec, + blocks=effective_spec.configuration.blocks, + include_only_create_instance_supported_backends=( + include_only_create_instance_supported_backends + ), + ) offers = [offer for _, offer in offers_with_backends] _remove_fleet_spec_sensitive_info(effective_spec) @@ -490,7 +492,7 @@ async def get_plan( async def get_fleet_offers( - project: ProjectModel, + backends: List[Backend], profile: Profile, requirements: Requirements, placement_group: Optional[PlacementGroup] = None, @@ -528,7 +530,7 @@ async def get_fleet_offers( break offers = await offers_services.get_offers_by_requirements( - project=project, + backends=backends, profile=profile, requirements=requirements, exclude_not_available=exclude_not_available, diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index cb0edb0712..678649426d 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -17,11 +17,13 @@ from dstack._internal.core.backends.base.compute import ( get_dstack_gateway_install_command, ) +from dstack._internal.core.backends.configurators import check_backend_type_available from dstack._internal.core.backends.features import ( BACKENDS_WITH_GATEWAY_SUPPORT, BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT, ) from dstack._internal.core.errors import ( + BackendNotAvailable, GatewayError, ResourceNotExistsError, ServerClientError, @@ -64,9 +66,9 @@ UserModel, ) from dstack._internal.server.services import events -from dstack._internal.server.services.backends import ( - check_backend_type_available, - get_project_backend_with_model_by_type_or_error, +from dstack._internal.server.services.backends import cache as backends_cache +from dstack._internal.server.services.backends.store import ( + get_project_backend_model_by_type_or_error, ) from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.server.services.gateways.pool import gateway_connections_pool @@ -253,7 +255,13 @@ async def create_gateway( _validate_gateway_configuration(effective_configuration) configuration = effective_configuration - backend_model, _ = await get_project_backend_with_model_by_type_or_error( + # Leasing the backend also checks that it can be initialized, not just that it's configured. + async with backends_cache.lease_project_backend_by_type( + project=project, backend_type=configuration.backend + ) as backend: + if backend is None: + raise BackendNotAvailable() + backend_model = await get_project_backend_model_by_type_or_error( project=project, backend_type=configuration.backend ) diff --git a/src/dstack/_internal/server/services/gpus.py b/src/dstack/_internal/server/services/gpus.py index 4e9c526fad..78864da714 100644 --- a/src/dstack/_internal/server/services/gpus.py +++ b/src/dstack/_internal/server/services/gpus.py @@ -2,7 +2,6 @@ from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.gpus import BackendGpu, BackendGpus, CpuGroup, GpuGroup @@ -189,7 +188,7 @@ async def _get_gpu_offers( skip_backend_offers = profile.creation_policy == CreationPolicy.REUSE instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]] - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if profile.instances is not None: instance_offers = await get_targeted_instance_offers( session=session, @@ -218,7 +217,7 @@ async def _get_gpu_offers( full_offers=full_offers, unallocated_resources=unallocated_resources, ) - return [offer for _, offer in instance_offers] + [offer for _, offer in backend_offers] + return [offer for _, offer in instance_offers] + backend_offers def _process_offers_into_backend_gpus( diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index eb826e03ee..a0fd5e0f0f 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -1,5 +1,7 @@ +import asyncio import heapq import itertools +import time from collections.abc import Container, Iterable, Iterator from typing import List, Literal, Optional, Tuple, TypeVar, Union @@ -13,6 +15,9 @@ BACKENDS_WITH_PRIVILEGED_SUPPORT, BACKENDS_WITH_RESERVATION_SUPPORT, ) +from dstack._internal.core.errors import ( + BackendError, +) from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.instances import ( InstanceOfferWithAvailability, @@ -23,13 +28,14 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.runs import JobProvisioningData, Requirements from dstack._internal.core.models.volumes import Volume, VolumeConfigurationWithRegion -from dstack._internal.server.models import ProjectModel -from dstack._internal.server.services import backends as backends_services from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) async def get_offers_by_requirements( - project: ProjectModel, + backends: List[Backend], profile: Profile, requirements: Requirements, exclude_not_available=False, @@ -44,8 +50,11 @@ async def get_offers_by_requirements( full_offers: bool = False, unallocated_resources: bool = False, ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]: - backends: List[Backend] = await backends_services.get_project_backends(project=project) - + """ + Returns offers of `backends` matching the requirements. The caller must lease `backends` + (see `services.backends.cache.lease_project_backends()`) and must not use the returned backends + after the lease is released. + """ backend_types: Optional[list[BackendType]] = profile.backends regions: Optional[list[str]] = profile.regions availability_zones: Optional[list[str]] = profile.availability_zones @@ -99,7 +108,7 @@ async def get_offers_by_requirements( if backend_types is not None: backends = [b for b in backends if b.TYPE in backend_types or b.TYPE == BackendType.DSTACK] - offers = await backends_services.get_backend_offers( + offers = await get_backend_offers( backends=backends, requirements=requirements, full_offers=full_offers, @@ -289,3 +298,57 @@ def _get_shareable_offers( new_offer = offer.model_copy() new_offer.total_blocks = total_blocks yield (backend, new_offer) + + +async def get_backend_offers( + backends: List[Backend], + requirements: Requirements, + full_offers: bool, + unallocated_resources: bool, + exclude_not_available: bool = False, +) -> Iterable[Tuple[Backend, InstanceOfferWithAvailability]]: + """ + Returns a lazy iterable of backend offers satisfying `requirements` sorted by price. + """ + + def get_filtered_offers_with_backends( + backend: Backend, + offers: Iterable[InstanceOfferWithAvailability], + ) -> Iterator[Tuple[Backend, InstanceOfferWithAvailability]]: + for offer in offers: + if not exclude_not_available or offer.availability.is_available(): + yield (backend, offer) + + logger.debug("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends]) + tasks = [ + run_async(get_offers_tracked, backend, requirements, full_offers, unallocated_resources) + for backend in backends + ] + offers_by_backend: list[Iterable[tuple[Backend, InstanceOfferWithAvailability]]] = [] + for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)): + if isinstance(result, BackendError): + logger.warning( + "Failed to get offers from backend %s: %s", + backend.TYPE, + repr(result), + ) + continue + elif isinstance(result, BaseException): + logger.error( + "Got exception when requesting offers from backend %s", + backend.TYPE, + exc_info=result, + ) + continue + offers_by_backend.append(get_filtered_offers_with_backends(backend, result)) + return merge_offer_iterables(*offers_by_backend) + + +def get_offers_tracked( + backend: Backend, requirements: Requirements, full_offers: bool, unallocated_resources: bool +) -> Iterator[InstanceOfferWithAvailability]: + start = time.time() + res = backend.compute().get_offers(requirements, full_offers, unallocated_resources) + duration = time.time() - start + logger.debug("Got offers from %s in %.6fs", backend.TYPE.value, duration) + return res diff --git a/src/dstack/_internal/server/services/projects.py b/src/dstack/_internal/server/services/projects.py index afee5796a6..452ff93c2c 100644 --- a/src/dstack/_internal/server/services/projects.py +++ b/src/dstack/_internal/server/services/projects.py @@ -41,7 +41,7 @@ from dstack._internal.server.schemas.projects import MemberSetting from dstack._internal.server.services import events, users from dstack._internal.server.services import templates as templates_service -from dstack._internal.server.services.backends import ( +from dstack._internal.server.services.backends.store import ( get_backend_config_without_creds_from_backend_model, ) from dstack._internal.server.services.locking import ( diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index 41869b9538..28b6111ffc 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -1,17 +1,17 @@ +import heapq import itertools import math import uuid from collections.abc import Hashable, Mapping from dataclasses import dataclass from enum import Enum -from typing import Optional, Union +from typing import Optional from sqlalchemy import and_, exists, not_, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import contains_eager, raiseload from sqlalchemy.orm.attributes import set_committed_value -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.models.common import EntityReference from dstack._internal.core.models.fleets import FleetSpec, InstanceGroupPlacement from dstack._internal.core.models.instances import ( @@ -36,6 +36,7 @@ ProjectModel, RunModel, ) +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.fleets import ( check_can_create_new_cloud_instance_in_fleet, get_fleet_master_instance_provisioning_data, @@ -58,10 +59,7 @@ is_multinode_job, remove_job_spec_sensitive_info, ) -from dstack._internal.server.services.offers import ( - get_offers_by_requirements, - merge_offer_iterables, -) +from dstack._internal.server.services.offers import get_offers_by_requirements from dstack._internal.server.services.requirements.combine import ( combine_fleet_and_run_profiles, combine_fleet_and_run_requirements, @@ -362,7 +360,7 @@ class _FleetCandidate: @dataclass class _FleetCandidateWithBackendOffers: candidate: _FleetCandidate - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] sort_key: tuple[bool, float, float] @@ -382,7 +380,7 @@ async def find_optimal_fleet_with_offers( ) -> tuple[ Optional[FleetModel], list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Finds the optimal fleet for the run among the given fleets and returns @@ -447,9 +445,7 @@ async def find_optimal_fleet_with_offers( instance_offers=( available_instance_offers if exclude_not_available else all_instance_offers ), - min_instance_offer_price=_get_min_instance_or_backend_offer_price( - available_instance_offers - ), + min_instance_offer_price=_get_min_instance_offer_price(available_instance_offers), # Require at least one available instance so that fleets without matching # instances are not treated as having capacity when nodes_required_num is 0 # (e.g. a service scaling from zero replicas). @@ -470,7 +466,7 @@ async def find_optimal_fleet_with_offers( # Second step: gather backend offers unless skipped. candidates_with_backend_offers: list[_FleetCandidateWithBackendOffers] = [] for candidate in candidates: - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if _skip_backend_offers: backend_offers = [] else: @@ -494,7 +490,7 @@ async def find_optimal_fleet_with_offers( sort_key=( not candidate.has_pool_capacity, candidate.min_instance_offer_price, - _get_min_instance_or_backend_offer_price(available_backend_offers), + _get_min_backend_offer_price(available_backend_offers), ), ) ) @@ -686,8 +682,8 @@ def select_targeted_instance_offers( candidates.append( _TargetedInstanceOffersCandidate( lacks_capacity=not has_capacity, - available_price=_get_min_instance_or_backend_offer_price(available_offers), - selected_price=_get_min_instance_or_backend_offer_price(all_offers), + available_price=_get_min_instance_offer_price(available_offers), + selected_price=_get_min_instance_offer_price(all_offers), offers=all_offers, ) ) @@ -776,7 +772,7 @@ async def _get_backend_offers_in_fleet( max_offers: Optional[int] = None, full_offers: bool = False, unallocated_resources: bool = False, -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: +) -> list[InstanceOfferWithAvailability]: if fleet_spec is None: fleet_spec = get_fleet_spec(fleet_model) try: @@ -797,19 +793,21 @@ async def _get_backend_offers_in_fleet( # Handle multinode for old jobs that don't have requirements.multinode set. # TODO: Drop multinode param. multinode = requirements.multinode or is_multinode_job(job) - backend_offers = await get_offers_by_requirements( - project=project, - profile=profile, - requirements=requirements, - multinode=multinode, - master_job_provisioning_data=master_instance_provisioning_data, - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run_spec), - max_offers=max_offers, - full_offers=full_offers, - unallocated_resources=unallocated_resources, - ) + async with backends_cache.lease_project_backends(project) as backends: + backends_with_offers = await get_offers_by_requirements( + backends=backends, + profile=profile, + requirements=requirements, + multinode=multinode, + master_job_provisioning_data=master_instance_provisioning_data, + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run_spec), + max_offers=max_offers, + full_offers=full_offers, + unallocated_resources=unallocated_resources, + ) + backend_offers = [offer for _, offer in backends_with_offers] return backend_offers @@ -860,7 +858,7 @@ async def get_non_fleet_offers( unallocated_resources: bool = False, ) -> tuple[ list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Returns instance and backend offers for job irrespective of fleets or instances, @@ -873,22 +871,24 @@ async def get_non_fleet_offers( job=job, volumes=volumes, ) - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if skip_backend_offers: backend_offers = [] else: - backend_offers = await get_offers_by_requirements( - project=project, - profile=run_spec.merged_profile, - requirements=job.job_spec.requirements, - exclude_not_available=False, - multinode=is_multinode_job(job), - volumes=volumes, - privileged=job.job_spec.privileged, - instance_mounts=check_run_spec_requires_instance_mounts(run_spec), - full_offers=full_offers, - unallocated_resources=unallocated_resources, - ) + async with backends_cache.lease_project_backends(project) as backends: + backends_with_offers = await get_offers_by_requirements( + backends=backends, + profile=run_spec.merged_profile, + requirements=job.job_spec.requirements, + exclude_not_available=False, + multinode=is_multinode_job(job), + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run_spec), + full_offers=full_offers, + unallocated_resources=unallocated_resources, + ) + backend_offers = [offer for _, offer in backends_with_offers] return instance_offers, backend_offers @@ -901,7 +901,7 @@ async def get_backend_offers_in_run_candidate_fleets( max_offers_per_fleet: Optional[int] = None, full_offers: bool = False, unallocated_resources: bool = False, -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: +) -> list[InstanceOfferWithAvailability]: """ Returns backend offers across the run's selected candidate fleets. @@ -919,10 +919,10 @@ async def get_backend_offers_in_run_candidate_fleets( run_spec=run_spec, ) seen_offer_identities = set() - offers: list[tuple[Backend, InstanceOfferWithAvailability]] = [] + offers: list[InstanceOfferWithAvailability] = [] for candidate_fleet_model in candidate_fleet_models: offers_from_fleet = [] - for backend, offer in await _get_backend_offers_in_fleet( + for offer in await _get_backend_offers_in_fleet( project=project, fleet_model=candidate_fleet_model, run_spec=run_spec, @@ -934,9 +934,9 @@ async def get_backend_offers_in_run_candidate_fleets( ): offer_identity = _get_backend_offer_identity(offer) if offer_identity not in seen_offer_identities: - offers_from_fleet.append((backend, offer)) + offers_from_fleet.append(offer) seen_offer_identities.add(offer_identity) - offers = list(merge_offer_iterables(offers, offers_from_fleet)) + offers = list(heapq.merge(offers, offers_from_fleet, key=lambda o: o.price)) return offers @@ -951,7 +951,7 @@ async def get_offers_in_run_candidate_fleets( unallocated_resources: bool = False, ) -> tuple[ list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], + list[InstanceOfferWithAvailability], ]: """ Returns existing-instance and backend offers across the run's candidate fleets. @@ -980,7 +980,7 @@ async def get_offers_in_run_candidate_fleets( ) instance_offers.sort(key=lambda offer: offer[1].price or 0) - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]] + backend_offers: list[InstanceOfferWithAvailability] if skip_backend_offers: backend_offers = [] else: @@ -1040,14 +1040,14 @@ def _freeze_offer_identity_value(value: object) -> Hashable: def _get_job_plan( instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]], - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]], + backend_offers: list[InstanceOfferWithAvailability], job: Job, max_offers: Optional[int], reverse_offers: bool = False, ) -> JobPlan: job_offers: list[InstanceOfferWithAvailability] = [] job_offers.extend(offer for _, offer in instance_offers) - job_offers.extend(offer for _, offer in backend_offers) + job_offers.extend(backend_offers) if reverse_offers: job_offers.sort(key=lambda offer: (not offer.availability.is_available(), -offer.price)) else: @@ -1074,15 +1074,19 @@ def _get_offers_from_instances( return instances_with_offers -def _get_min_instance_or_backend_offer_price( - offers: Union[ - list[tuple[InstanceModel, InstanceOfferWithAvailability]], - list[tuple[Backend, InstanceOfferWithAvailability]], - ], +def _get_min_instance_offer_price( + instance_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]], ) -> float: min_offer_price = math.inf - if len(offers) > 0: - min_offer_price = offers[0][1].price + if len(instance_offers) > 0: + min_offer_price = instance_offers[0][1].price + return min_offer_price + + +def _get_min_backend_offer_price(backend_offers: list[InstanceOfferWithAvailability]) -> float: + min_offer_price = math.inf + if len(backend_offers) > 0: + min_offer_price = backend_offers[0].price return min_offer_price @@ -1097,8 +1101,6 @@ def _exclude_non_available_instance_offers( def _exclude_non_available_backend_offers( - backend_offers: list[tuple[Backend, InstanceOfferWithAvailability]], -) -> list[tuple[Backend, InstanceOfferWithAvailability]]: - return [ - (backend, offer) for backend, offer in backend_offers if offer.availability.is_available() - ] + backend_offers: list[InstanceOfferWithAvailability], +) -> list[InstanceOfferWithAvailability]: + return [offer for offer in backend_offers if offer.availability.is_available()] diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index 9d648eee46..d77d6f5ea1 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -8,9 +8,9 @@ from sqlalchemy.orm import joinedload, selectinload from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport +from dstack._internal.core.backends.configurators import check_backend_type_available from dstack._internal.core.backends.features import BACKENDS_WITH_VOLUMES_SUPPORT from dstack._internal.core.errors import ( - BackendNotAvailable, ResourceExistsError, ServerClientError, ) @@ -37,8 +37,8 @@ VolumeAttachmentModel, VolumeModel, ) -from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services.backends import cache as backends_cache from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.locking import ( get_locker, @@ -481,7 +481,7 @@ def _validate_volume_configuration(configuration: AnyVolumeConfiguration): and configuration.size is None ): raise ServerClientError("Volume must specify either existing identifier or size") - backends_services.check_backend_type_available(configuration.backend) + check_backend_type_available(configuration.backend) if configuration.backend not in BACKENDS_WITH_VOLUMES_SUPPORT: raise ServerClientError( f"Volumes are not supported for {configuration.backend.value} backend." @@ -518,23 +518,21 @@ async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_mo ) return - try: - backend = await backends_services.get_project_backend_by_type_or_error( - project=volume_model.project, - backend_type=volume.provisioning_data.backend, - ) - except BackendNotAvailable: - logger.error( - f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + async with backends_cache.lease_project_backend_by_type( + project=volume_model.project, + backend_type=volume.provisioning_data.backend, + ) as backend: + if backend is None: + logger.error( + f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available." + ) + return + compute = backend.compute() + assert isinstance(compute, ComputeWithVolumeSupport) + await common.run_async( + compute.delete_volume, + volume=volume, ) - return - - compute = backend.compute() - assert isinstance(compute, ComputeWithVolumeSupport) - await common.run_async( - compute.delete_volume, - volume=volume, - ) # Clouds charge volumes assuming 30-day months, e.g. https://aws.amazon.com/ebs/pricing/ diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 29f2eac87b..73b27b70a1 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -1,10 +1,11 @@ import json import uuid from collections.abc import Callable -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager from datetime import datetime, timezone from http import HTTPStatus from typing import Any, Dict, List, Literal, Optional, Union +from unittest.mock import Mock from uuid import UUID import gpuhunt @@ -147,6 +148,19 @@ from dstack._internal.server.services.users import get_token_hash +def mock_lease(value: Any) -> Mock: + """ + Returns a mock to patch a `lease_*()` function from `services.backends.cache` with. + The lease yields `value`. + """ + + @asynccontextmanager + async def lease(*args, **kwargs): + yield value + + return Mock(side_effect=lease) + + def get_auth_headers(token: Union[DecryptedString, str]) -> Dict: if isinstance(token, DecryptedString): token = token.get_plaintext_or_error() diff --git a/src/dstack/_internal/utils/event_loop.py b/src/dstack/_internal/utils/event_loop.py deleted file mode 100644 index 29de015c01..0000000000 --- a/src/dstack/_internal/utils/event_loop.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import threading -from collections.abc import Awaitable -from typing import TypeVar - -T = TypeVar("T") - - -class DaemonEventLoop: - """ - A wrapper around asyncio.EventLoop that runs the loop in a daemon thread. - The thread is started with the first `await_` call. - """ - - def __init__(self) -> None: - self._loop = asyncio.new_event_loop() - self._start_lock = threading.Lock() - self._started = False - - def await_(self, awaitable: Awaitable[T]) -> T: - with self._start_lock: - if not self._started: - threading.Thread(target=self._loop.run_forever, daemon=True).start() - self._started = True - future = asyncio.run_coroutine_threadsafe(_coroutine(awaitable), self._loop) - return future.result() - - -async def _coroutine(awaitable: Awaitable[T]) -> T: - return await awaitable diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py index ab1441434a..c016d4b139 100644 --- a/src/tests/_internal/core/backends/nebius/test_compute.py +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -486,3 +486,27 @@ def test_waits_for_pricing_policy_being_created(self, resources): make_compute().delete_resource_group(group) resources.delete_pricing_policy.assert_not_called() + + +class TestClose: + def test_closes_created_sdk_once(self, mocker): + sdk = Mock() + mocker.patch.object(compute_module.resources, "make_sdk", return_value=sdk) + compute = make_compute() + assert compute._sdk is sdk + compute.close() + compute.close() + sdk.sync_close.assert_called_once_with(timeout=compute_module.resources.SDK_CLOSE_TIMEOUT) + + def test_does_not_create_sdk(self, mocker): + make_sdk = mocker.patch.object(compute_module.resources, "make_sdk") + make_compute().close() + make_sdk.assert_not_called() + + def test_does_not_raise_if_sdk_fails_to_close(self, mocker): + sdk = Mock() + sdk.sync_close.side_effect = TimeoutError() + mocker.patch.object(compute_module.resources, "make_sdk", return_value=sdk) + compute = make_compute() + assert compute._sdk is sdk + compute.close() diff --git a/src/tests/_internal/core/backends/nebius/test_configurator.py b/src/tests/_internal/core/backends/nebius/test_configurator.py new file mode 100644 index 0000000000..86949d73d6 --- /dev/null +++ b/src/tests/_internal/core/backends/nebius/test_configurator.py @@ -0,0 +1,48 @@ +from unittest.mock import Mock + +import pytest + +from dstack._internal.core.backends.nebius import configurator as configurator_module +from dstack._internal.core.backends.nebius.configurator import NebiusConfigurator +from dstack._internal.core.backends.nebius.models import ( + NebiusBackendConfigWithCreds, + NebiusServiceAccountCreds, +) +from dstack._internal.core.errors import BackendInvalidCredentialsError + + +def make_config() -> NebiusBackendConfigWithCreds: + return NebiusBackendConfigWithCreds( + creds=NebiusServiceAccountCreds( + service_account_id="service-account-id", + public_key_id="public-key-id", + private_key_content="private-key", + ) + ) + + +class TestValidateConfig: + @pytest.fixture(autouse=True) + def _mock_fabrics(self, mocker): + mocker.patch.object(configurator_module, "get_all_infiniband_fabrics", return_value=set()) + + def test_closes_sdk_if_valid(self, mocker): + sdk = Mock() + mocker.patch.object(configurator_module.resources, "make_sdk", return_value=sdk) + mocker.patch.object( + configurator_module.resources, "get_region_to_project_id_map", return_value={} + ) + NebiusConfigurator().validate_config(make_config(), default_creds_enabled=False) + sdk.sync_close.assert_called_once() + + def test_closes_sdk_if_invalid(self, mocker): + sdk = Mock() + mocker.patch.object(configurator_module.resources, "make_sdk", return_value=sdk) + mocker.patch.object( + configurator_module.resources, + "get_region_to_project_id_map", + side_effect=ValueError("Expected to find 1 tenant, found 0"), + ) + with pytest.raises(BackendInvalidCredentialsError): + NebiusConfigurator().validate_config(make_config(), default_creds_enabled=False) + sdk.sync_close.assert_called_once() diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py index a1b61979bf..47f6cc7a41 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_check.py @@ -1,7 +1,7 @@ import datetime as dt import logging from typing import ClassVar, Optional -from unittest.mock import AsyncMock, Mock +from unittest.mock import Mock import pytest import pytest_asyncio @@ -54,6 +54,7 @@ get_job_provisioning_data, get_remote_connection_info, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime from tests._internal.server.background.pipeline_tasks.instances.helpers import ( @@ -165,9 +166,8 @@ async def test_cloudrift_waits_for_backend_within_timeout( await session.commit() backend = Mock() monkeypatch.setattr( - instances_check, - "_get_backend_for_provisioning_wait", - AsyncMock(return_value=backend), + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend), ) await process_instance(session, worker, instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py index 804f81e0fe..816613d344 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_cloud_provisioning.py @@ -35,6 +35,7 @@ get_instance_offer_with_availability, get_job_provisioning_data, get_placement_group_provisioning_data, + mock_lease, ) from tests._internal.server.background.pipeline_tasks.instances.helpers import ( instance_to_pipeline_item, @@ -86,9 +87,11 @@ async def test_creates_instance( total_blocks=requested_blocks, busy_blocks=0, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS gpu = Gpu(name="T4", memory_mib=16384, vendor=gpuhunt.AcceleratorVendor.NVIDIA) offer = InstanceOfferWithAvailability( @@ -161,8 +164,10 @@ async def test_tries_second_offer_if_first_fails( region=offer.region, price=offer.price, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [aws_mock, gcp_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([aws_mock, gcp_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -190,8 +195,10 @@ async def test_fails_if_all_offers_fail( aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.get_offers.return_value = [offer] aws_mock.compute.return_value.create_instance.side_effect = err - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [aws_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([aws_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -210,8 +217,10 @@ async def test_fails_if_no_offers( project=project, status=InstanceStatus.PENDING, ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -248,8 +257,10 @@ async def test_waits_when_fleet_has_no_current_master( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -296,8 +307,10 @@ async def test_waits_for_current_master_to_determine_cluster_placement( backend_mock = Mock() backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(master_instance) @@ -349,8 +362,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi master_item = instance_to_pipeline_item(master_instance) sibling_item = instance_to_pipeline_item(sibling_instance) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([]), + ): await worker.process(master_item) await session.refresh(master_instance) @@ -383,8 +398,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [gcp_mock, aws_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([gcp_mock, aws_mock]), + ): await worker.process(sibling_item) await session.refresh(sibling_instance) @@ -409,8 +426,10 @@ async def test_failed_master_does_not_provision_stale_sibling_until_fleet_reassi region="us-east-1", ) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [promoted_backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([promoted_backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -478,8 +497,10 @@ async def test_follows_current_master_backend_and_region_constraints( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [gcp_mock, aws_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([gcp_mock, aws_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -540,8 +561,10 @@ async def test_non_master_does_not_create_new_placement_group_without_master_pg( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -605,8 +628,10 @@ async def test_non_master_reuses_existing_current_master_placement_group( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, sibling_instance) await session.refresh(sibling_instance) @@ -677,8 +702,10 @@ async def test_allows_parallel_processing_after_master_is_provisioned( backend=BackendType.AWS, region="us-east-1", ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, later_instance) assert backend_mock.compute.return_value.create_instance.call_count == 1 await process_instance(session, worker, earlier_instance) @@ -736,8 +763,10 @@ async def test_create_placement_group_if_placement_cluster( backend_mock.compute.return_value.create_placement_group.return_value = ( get_placement_group_provisioning_data() ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -790,8 +819,10 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.create_instance.return_value = get_job_provisioning_data() compute_mock.create_placement_group.return_value = get_placement_group_provisioning_data() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -859,8 +890,10 @@ def create_instance_method( get_placement_group_provisioning_data() ) backend_mock.compute.return_value.is_suitable_placement_group.return_value = can_reuse - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) @@ -918,8 +951,10 @@ async def test_master_reuses_existing_placement_group( backend_mock.compute.return_value.create_instance.return_value = ( get_job_provisioning_data() ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, master_instance) await session.refresh(master_instance) @@ -979,8 +1014,10 @@ def create_placement_group_method( raise err backend_mock.compute.return_value.create_placement_group = create_placement_group_method - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): await process_instance(session, worker, instance) await session.refresh(instance) diff --git a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py index 414518d0d1..18e4050244 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py +++ b/src/tests/_internal/server/background/pipeline_tasks/instances/test_termination.py @@ -1,7 +1,7 @@ import datetime as dt from contextlib import contextmanager from typing import Optional -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import Mock, patch import pytest import time_machine @@ -21,6 +21,7 @@ create_instance, create_project, get_job_provisioning_data, + mock_lease, ) from tests._internal.server.background.pipeline_tasks.instances.helpers import ( instance_to_pipeline_item, @@ -40,10 +41,9 @@ def mock_terminate_in_backend(error: Optional[Exception] = None): terminate_instance = backend.compute.return_value.terminate_instance if error is not None: terminate_instance.side_effect = error - with patch.object( - instances_termination.backends_services, - "get_project_backend_by_type", - AsyncMock(return_value=backend), + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend), ): yield terminate_instance diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py index 66296ba844..15acef062b 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_compute_groups.py @@ -21,6 +21,7 @@ create_compute_group, create_fleet, create_project, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -164,11 +165,13 @@ async def test_terminates_compute_group( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock backend_mock.TYPE = BackendType.RUNPOD await worker.process(_compute_group_to_pipeline_item(compute_group)) compute_mock.terminate_compute_group.assert_called_once() @@ -190,11 +193,13 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) @@ -210,11 +215,13 @@ async def test_retries_compute_group_termination( compute_group.lock_token = uuid.uuid4() compute_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = backend_mock backend_mock.TYPE = BackendType.RUNPOD compute_mock.terminate_compute_group.side_effect = BackendError() await worker.process(_compute_group_to_pipeline_item(compute_group)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index 1c1ca8bb4f..5b2d0241b3 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -49,6 +49,7 @@ get_job_provisioning_data, get_run_spec, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -412,11 +413,11 @@ async def test_submitted_to_provisioning( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as m: - aws = Mock() - m.return_value = [(backend, aws)] + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.return_value = ( GatewayReplicaProvisioningData( @@ -458,11 +459,11 @@ async def test_submitted_backend_error_marks_terminated( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as m: - aws = Mock() - m.return_value = [(backend, aws)] + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = BackendError( "Some error" @@ -499,9 +500,9 @@ async def test_submitted_backend_not_available_marks_terminated( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as m: - m.return_value = [] + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(None), + ): await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -535,7 +536,7 @@ async def test_submitted_skips_provisioning_if_gateway_to_be_deleted( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -570,7 +571,7 @@ async def test_submitted_skips_provisioning_if_gateway_failed( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -604,11 +605,11 @@ async def test_submitted_unexpected_error_marks_terminated( _lock_replica(replica) await session.commit() + aws = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as m: - aws = Mock() - m.return_value = [(backend, aws)] + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(aws), + ): aws.compute.return_value = Mock(spec=ComputeMockSpec) aws.compute.return_value.create_gateway_replica.side_effect = RuntimeError( "Unexpected!" @@ -647,7 +648,7 @@ async def test_submitted_to_terminated_when_scaled_in( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as m: await worker.process(_replica_to_pipeline_item(replica)) m.assert_not_called() @@ -2208,19 +2209,19 @@ async def test_provisioning_to_running_registers_with_load_balancer( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2262,7 +2263,7 @@ async def test_provisioning_skips_load_balancer_registration_without_hostname( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id" ) as get_backends_mock, ): pool_add.return_value = MagicMock() @@ -2299,22 +2300,22 @@ async def test_provisioning_to_terminating_when_load_balancer_registration_fails _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.register_gateway_replica_with_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2346,19 +2347,19 @@ async def test_provisioning_to_terminating_when_backend_does_not_support_load_ba _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( "dstack._internal.server.services.gateways.gateway_connections_pool.get_or_add" ) as pool_add, patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), ): pool_add.return_value = MagicMock() pool_add.return_value.client.return_value = MagicMock(AsyncContextManager()) - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2588,17 +2589,17 @@ async def test_terminating_to_terminated( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2666,17 +2667,17 @@ async def test_terminating_to_terminated_deletes_only_own_registration_records( _lock_replica(replica_to_terminate) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica_to_terminate)) @@ -2717,17 +2718,17 @@ async def test_terminating_deregisters_from_load_balancer_before_terminating( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2770,20 +2771,20 @@ async def test_terminating_proceeds_when_load_balancer_deregistration_raises( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.deregister_gateway_replica_from_load_balancer.side_effect = Exception( "boom" ) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2821,17 +2822,17 @@ async def test_terminating_skips_deregistration_when_gateway_has_no_hostname( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ), ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2866,12 +2867,12 @@ async def test_terminating_waits_for_pending_acm_gateway_migration( original_last_processed_at = replica.last_processed_at await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -2915,9 +2916,9 @@ async def test_terminating_to_terminated_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - get_backends_mock.return_value = [] + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(None), + ): await worker.process(_replica_to_pipeline_item(replica)) await session.refresh(replica) @@ -2958,17 +2959,17 @@ async def test_terminating_to_terminated_with_no_instance_id( _lock_replica(replica) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) @@ -3012,20 +3013,20 @@ async def test_terminating_retries_if_terminate_fails( original_last_processed_at = replica.last_processed_at await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_id", + mock_lease(backend_mock), ) as get_backends_mock, patch( "dstack._internal.server.background.pipeline_tasks.gateway_replicas.gateway_connections_pool.remove" ) as remove_mock, ): - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.terminate_gateway_replica.side_effect = Exception( "Terminate failed" ) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_replica_to_pipeline_item(replica)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py index 2953744be4..47aa3c0694 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateways.py @@ -36,6 +36,7 @@ create_project, get_gateway_replica_configuration, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -534,10 +535,11 @@ async def test_submitted_to_provisioning_creates_load_balancer( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_gateway_load_balancer.return_value = ( @@ -545,7 +547,6 @@ async def test_submitted_to_provisioning_creates_load_balancer( hostname="gateway-lb.example.com", backend_data="lb-backend-data" ) ) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) @@ -575,7 +576,7 @@ async def test_submitted_skips_load_balancer_creation_for_non_acm_gateway( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -601,13 +602,13 @@ async def test_submitted_to_failed_when_backend_does_not_support_load_balancer( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -642,14 +643,14 @@ async def test_submitted_to_failed_when_load_balancer_creation_raises( gateway.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_gateway_load_balancer.side_effect = exception - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) await session.refresh(gateway) @@ -1697,13 +1698,13 @@ async def test_deletes_gateway_and_terminates_load_balancer_when_hostname_set( gateway.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1742,7 +1743,7 @@ async def test_delete_skips_load_balancer_termination_when_hostname_not_set( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) get_backends_mock.assert_not_called() @@ -1777,16 +1778,16 @@ async def test_delete_deferred_when_load_balancer_termination_fails( original_last_processed_at = gateway.last_processed_at await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.terminate_gateway_load_balancer.side_effect = ( Exception("boom") ) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1826,13 +1827,13 @@ async def test_delete_deferred_when_backend_does_not_support_load_balancer( gateway.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" - ) as get_backends_mock: - backend_mock = Mock() + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeWithGatewaySupport) - get_backends_mock.return_value = [(backend, backend_mock)] await worker.process(_gateway_to_pipeline_item(gateway)) @@ -1867,7 +1868,7 @@ async def test_delete_migrates_hostname_before_evaluating_termination( await session.commit() with patch( - "dstack._internal.server.services.backends.get_project_backends_with_models" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backends_mock: await worker.process(_gateway_to_pipeline_item(gateway)) # Migration happens before the load balancer would be evaluated for termination, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py index 8294e86238..e3360dd575 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py @@ -88,6 +88,7 @@ get_run_spec, get_ssh_fleet_configuration, get_volume_provisioning_data, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime, get_or_error @@ -428,9 +429,11 @@ async def test_provisions_new_capacity_for_assigned_job_with_placeholder( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -511,12 +514,14 @@ async def test_multinode_master_reuses_placeholder_when_provisioning_falls_back_ ) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( dockerized=True, @@ -608,9 +613,11 @@ async def test_provisioning_master_job_respects_cluster_placement_in_non_empty_f fleet.lock_owner = "OtherPipeline:cluster-master" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS offer_1 = get_instance_offer_with_availability( backend=BackendType.AWS, @@ -674,7 +681,7 @@ async def test_defers_new_capacity_provisioning_until_fleet_master_is_elected( ) previous_last_processed_at = job.last_processed_at - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -732,9 +739,11 @@ async def test_provisioning_non_master_job_ignores_cluster_master_fleet_lock( fleet.lock_owner = "OtherPipeline:cluster-master" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS offer_1 = get_instance_offer_with_availability( backend=BackendType.AWS, @@ -783,12 +792,14 @@ async def test_creates_placement_group_for_cluster_fleet( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( backend=BackendType.AWS, @@ -836,12 +847,14 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible compute_mock.run_job.return_value = get_job_provisioning_data( @@ -905,12 +918,14 @@ async def test_marks_unused_existing_placement_groups_for_cleanup( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.is_suitable_placement_group.side_effect = ( lambda placement_group, _: placement_group.name == selected_pg.name @@ -963,12 +978,14 @@ async def test_marks_new_and_existing_placement_groups_for_cleanup_on_failed_pro job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.is_suitable_placement_group.return_value = False compute_mock.create_placement_group.return_value = ( @@ -1019,7 +1036,7 @@ async def test_resets_lock_for_retry_when_cluster_master_fleet_lock_is_unavailab previous_last_processed_at = job.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) m.assert_not_called() @@ -1058,9 +1075,11 @@ async def test_reclaims_stale_related_cluster_master_fleet_lock( fleet.lock_owner = f"{JobSubmittedPipeline.__name__}:{job.id}" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.AWS) @@ -1854,9 +1873,11 @@ async def test_assignment_creates_placeholder_instance_for_new_capacity( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] @@ -1932,7 +1953,7 @@ async def test_terminates_job_when_no_reusable_instances_with_reuse_creation_pol ) job = await create_job(session=session, run=run) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: + with patch("dstack._internal.server.services.backends.cache.lease_project_backends") as m: await _process_job(session=session, worker=worker, job_model=job) # Backend offers must not be requested with the reuse policy. @@ -1974,9 +1995,11 @@ async def test_job_fails_when_fleet_is_full( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] @@ -2027,12 +2050,14 @@ async def test_leaves_placeholder_for_terminating_pipeline_on_failed_new_capacit await session.commit() offer = get_instance_offer_with_availability(backend=BackendType.AWS) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.side_effect = BackendError("boom") @@ -2058,12 +2083,14 @@ async def test_reports_tried_offers_when_new_capacity_provisioning_fails( job = await create_job(session=session, run=run) offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us-east-1") - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] compute_mock.get_offers.return_value = [offer] compute_mock.run_job.side_effect = BackendError("InsufficientInstanceCapacity") @@ -2111,11 +2138,13 @@ async def test_provisions_compute_group( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_jobs.return_value = get_compute_group_provisioning_data( @@ -2181,11 +2210,13 @@ async def test_provisions_one_job_node_group_via_run_job( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_job.return_value = get_job_provisioning_data( @@ -2457,9 +2488,11 @@ async def test_attaches_volume_on_existing_instance( instance_assigned=True, ) - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData() @@ -2528,9 +2561,11 @@ async def test_terminates_job_when_volume_is_locked_for_processing( ) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) @@ -2547,6 +2582,56 @@ async def test_terminates_job_when_volume_is_locked_for_processing( assert volume.lock_expires_at is not None backend_mock.compute.return_value.attach_volume.assert_not_called() + async def test_terminates_job_when_volume_backend_is_not_available( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + volume = await create_volume( + session=session, + project=project, + user=user, + status=VolumeStatus.ACTIVE, + volume_provisioning_data=get_volume_provisioning_data(), + backend=BackendType.AWS, + region="us-east-1", + ) + fleet = await create_fleet(session=session, project=project) + instance = await create_instance( + session=session, + project=project, + fleet=fleet, + status=InstanceStatus.BUSY, + busy_blocks=1, + backend=BackendType.AWS, + region="us-east-1", + ) + run_spec = get_run_spec(repo_id=repo.name) + run_spec.configuration.volumes = [VolumeMountPoint(name=volume.name, path="/volume")] + run = await create_run( + session=session, project=project, repo=repo, user=user, run_spec=run_spec + ) + job = await create_job( + session=session, + run=run, + instance=instance, + instance_assigned=True, + ) + + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), + ): + await _process_job(session=session, worker=worker, job_model=job) + + await session.refresh(job) + await session.refresh(volume) + assert job.status == JobStatus.TERMINATING + assert job.termination_reason == JobTerminationReason.VOLUME_ERROR + assert job.termination_reason_message == "Failed to attach volume: backend not available" + assert volume.lock_owner is None + async def test_reclaims_stale_related_volume_lock( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): @@ -2588,9 +2673,11 @@ async def test_reclaims_stale_related_volume_lock( volume.lock_owner = f"{JobSubmittedPipeline.__name__}:{job.id}" await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - backend_mock = Mock() - m.return_value = backend_mock + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData() @@ -2639,9 +2726,11 @@ async def test_run_job_uses_server_default_registry( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -2704,11 +2793,13 @@ async def test_run_jobs_uses_server_default_registry( ) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): compute_mock = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value = compute_mock - m.return_value = [backend_mock] backend_mock.TYPE = BackendType.RUNPOD compute_mock.get_offers.return_value = [offer] compute_mock.run_jobs.return_value = get_compute_group_provisioning_data( @@ -2771,9 +2862,11 @@ async def test_interpolates_secrets_when_provisioning_new_capacity( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( @@ -2829,9 +2922,11 @@ async def test_terminates_job_when_secret_is_missing( job = await create_job(session=session, run=run, instance_assigned=True) offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index 978a2720a1..5a64fd62a0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -46,6 +46,7 @@ get_volume_configuration, get_volume_provisioning_data, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -809,15 +810,15 @@ async def test_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as m: - backend_mock = Mock() - m.return_value = backend_mock backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) - m.assert_awaited_once() + m.assert_called_once() backend_mock.compute.return_value.detach_volume.assert_called_once() backend_mock.compute.return_value.is_volume_detached.assert_called_once() @@ -860,11 +861,11 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.return_value = backend_mock + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -878,10 +879,12 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with ( patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" - ) as m, + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ), patch( "dstack._internal.server.background.pipeline_tasks.jobs_terminating.get_current_datetime" ) as datetime_mock, @@ -889,8 +892,6 @@ async def test_force_detaches_job_volumes( datetime_mock.return_value = job.volumes_detached_at.replace( tzinfo=timezone.utc ) + timedelta(minutes=30) - backend_mock = Mock() - m.return_value = backend_mock backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = False await worker.process(_job_to_pipeline_item(job)) @@ -908,11 +909,11 @@ async def test_force_detaches_job_volumes( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.return_value = backend_mock + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True await worker.process(_job_to_pipeline_item(job)) @@ -1016,11 +1017,11 @@ async def test_detaches_job_volumes_on_shared_instance( _lock_job(job) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.jobs_terminating.backends_services.get_project_backend_by_type" - ) as m: - backend_mock = Mock() - m.return_value = backend_mock + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), + ): backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.is_volume_detached.return_value = True diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py index ccd2513230..8b7f4892e0 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime, timedelta, timezone from typing import Optional -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import Mock, patch import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -50,6 +50,7 @@ create_user, get_job_provisioning_data, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -402,10 +403,9 @@ async def _lock(session: AsyncSession, orphan_cleanup: OrphanCleanupModel) -> Pi def _patch_compute(compute: _ListingCompute): backend = Mock() backend.compute.return_value = compute - return patch.object( - orphan_cleanup_pipeline.backends_services, - "get_project_backend_by_type", - AsyncMock(return_value=backend), + return patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend), ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py index 9b3f7fbef1..3cf058dbff 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_placement_groups.py @@ -19,6 +19,7 @@ create_fleet, create_placement_group, create_project, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -196,9 +197,11 @@ async def test_deletes_placement_group( placement_group.lock_token = uuid.uuid4() placement_group.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - aws_mock = Mock() - m.return_value = aws_mock + aws_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), + ): aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await worker.process(_placement_group_to_pipeline_item(placement_group)) aws_mock.compute.return_value.delete_placement_group.assert_called_once() @@ -225,9 +228,11 @@ async def test_retries_placement_group_deletion_if_still_in_use( placement_group.lock_owner = "PlacementGroupPipeline" original_last_processed_at = placement_group.last_processed_at await session.commit() - with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: - aws_mock = Mock() - m.return_value = aws_mock + aws_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), + ): aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) aws_mock.compute.return_value.delete_placement_group.side_effect = ( PlacementGroupInUseError() diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py index 2487dc4cfe..3cac7f3f05 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_volumes.py @@ -6,7 +6,7 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.errors import BackendError, BackendNotAvailable +from dstack._internal.core.errors import BackendError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.volumes import VolumeProvisioningData, VolumeStatus from dstack._internal.server.background.pipeline_tasks import volumes as volumes_pipeline @@ -25,6 +25,7 @@ get_volume_configuration, get_volume_provisioning_data, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -214,17 +215,17 @@ async def test_submitted_to_active(self, test_db, session: AsyncSession, worker: volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_volume.return_value = VolumeProvisioningData( backend=BackendType.AWS, volume_id="vol-1234", size_gb=100, ) - get_backend_mock.return_value = backend_mock await worker.process(_volume_to_pipeline_item(volume)) @@ -255,10 +256,11 @@ async def test_registers_external_volume( volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.register_volume.return_value = ( VolumeProvisioningData( @@ -267,7 +269,6 @@ async def test_registers_external_volume( size_gb=100, ) ) - get_backend_mock.return_value = backend_mock await worker.process(_volume_to_pipeline_item(volume)) @@ -297,9 +298,9 @@ async def test_marks_volume_failed_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), ) as get_backend_mock: - get_backend_mock.side_effect = BackendNotAvailable() await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -328,15 +329,15 @@ async def test_marks_volume_failed_if_backend_returns_error( volume.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.create_volume.side_effect = BackendError( "Some error" ) - get_backend_mock.return_value = backend_mock await worker.process(_volume_to_pipeline_item(volume)) @@ -437,12 +438,12 @@ async def test_marks_volume_deleted( volume.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) - get_backend_mock.return_value = backend_mock await worker.process(_volume_to_pipeline_item(volume)) @@ -474,7 +475,7 @@ async def test_marks_external_volume_deleted_without_backend_call( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type" ) as get_backend_mock: await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_not_called() @@ -504,9 +505,9 @@ async def test_marks_volume_deleted_if_backend_not_available( await session.commit() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(None), ) as get_backend_mock: - get_backend_mock.side_effect = BackendNotAvailable() await worker.process(_volume_to_pipeline_item(volume)) get_backend_mock.assert_called_once() @@ -534,15 +535,15 @@ async def test_marks_volume_deleted_if_backend_delete_errors( volume.to_be_deleted = True await session.commit() + backend_mock = Mock() with patch( - "dstack._internal.server.background.pipeline_tasks.volumes.backends_services.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(backend_mock), ) as get_backend_mock: - backend_mock = Mock() backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) backend_mock.compute.return_value.delete_volume.side_effect = BackendError( "Delete failed" ) - get_backend_mock.return_value = backend_mock await worker.process(_volume_to_pipeline_item(volume)) diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py index 3b6659729a..b2f63c9e42 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_idle_volumes.py @@ -21,6 +21,7 @@ get_volume_configuration, get_volume_provisioning_data, list_events, + mock_lease, ) from dstack._internal.utils.common import get_current_datetime @@ -64,11 +65,11 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): ) await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), ) as m: - aws_mock = Mock() - m.return_value = aws_mock aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() @@ -110,11 +111,11 @@ async def test_deletes_idle_volume_with_null_auto_cleanup_enabled( volume.auto_cleanup_enabled = None await session.commit() + aws_mock = Mock() with patch( - "dstack._internal.server.services.backends.get_project_backend_by_type_or_error" + "dstack._internal.server.services.backends.cache.lease_project_backend_by_type", + mock_lease(aws_mock), ) as m: - aws_mock = Mock() - m.return_value = aws_mock aws_mock.compute.return_value = Mock(spec=ComputeMockSpec) await process_idle_volumes() m.assert_not_called() diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index 90c1c9bd3f..f21c27f153 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -54,6 +54,7 @@ get_private_key_string, get_remote_connection_info, get_ssh_fleet_configuration, + mock_lease, ) from dstack._internal.server.testing.matchers import SomeUUID4Str @@ -2248,9 +2249,11 @@ async def test_returns_create_plan_for_new_fleet( ) ] spec = get_fleet_spec() - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = offers response = await client.post( @@ -2291,9 +2294,11 @@ async def test_returns_offers_for_elastic_container_backend_fleet( spec = get_fleet_spec( conf=get_fleet_configuration(nodes=FleetNodesSpec(min=0, target=0, max=1)) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] response = await client.post( @@ -2327,9 +2332,11 @@ async def test_returns_no_offers_for_non_elastic_container_backend_fleet( spec = get_fleet_spec( conf=get_fleet_configuration(nodes=FleetNodesSpec(min=0, target=1, max=1)) ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() - m.return_value = [backend_mock] + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.RUNPOD backend_mock.compute.return_value.get_offers.return_value = [offer] response = await client.post( diff --git a/src/tests/_internal/server/routers/test_gpus.py b/src/tests/_internal/server/routers/test_gpus.py index 638f133101..8847b8031a 100644 --- a/src/tests/_internal/server/routers/test_gpus.py +++ b/src/tests/_internal/server/routers/test_gpus.py @@ -33,6 +33,7 @@ get_fleet_spec, get_instance_offer_with_availability, get_run_spec, + mock_lease, ) pytestmark = pytest.mark.usefixtures("image_config_mock") @@ -482,8 +483,10 @@ async def test_price_order( BackendType.AWS, "T4", 16384, 100, availability=InstanceAvailability.NO_QUOTA ), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = create_mock_backends_with_offers({BackendType.AWS: offers}) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(create_mock_backends_with_offers({BackendType.AWS: offers})), + ): response = await call_gpus_api( client, project.name, user.token, run_spec, group_by=group_by, reverse=reverse ) @@ -550,10 +553,13 @@ async def test_cpu_groups( availability=InstanceAvailability.NO_QUOTA, ) unavailable.instance.resources.cpu_arch = gpuhunt.CPUArchitecture.X86 - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = create_mock_backends_with_offers( - {BackendType.AWS: aws, BackendType.AZURE: [unavailable]} - ) + mocked_backends = create_mock_backends_with_offers( + {BackendType.AWS: aws, BackendType.AZURE: [unavailable]} + ) + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -625,8 +631,10 @@ async def test_returns_gpus_without_group_by( offers_by_backend = {BackendType.AWS: [offer_aws], BackendType.RUNPOD: [offer_runpod]} mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api(client, project.name, user.token, run_spec) assert response.status_code == 200 @@ -657,8 +665,10 @@ async def test_forwards_full_offers_to_compute_get_offers( offer = create_gpu_offer(BackendType.AWS, "T4", 16384, 0.50) mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, user.token, run_spec, full_offers=body_full_offers ) @@ -694,8 +704,10 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( offer = create_gpu_offer(BackendType.AWS, "T4", 16384, 0.50) mocked_backends = create_mock_backends_with_offers({BackendType.AWS: [offer]}) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -763,8 +775,10 @@ async def test_filters_gpus_by_multiple_specified_fleets( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api( client, project.name, @@ -791,8 +805,10 @@ async def test_includes_backend_offers_when_creation_policy_reuse_or_create( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) response = await call_gpus_api(client, project.name, user.token, run_spec) @@ -813,8 +829,10 @@ async def test_skips_backend_offers_when_creation_policy_reuse( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): # reuse skips backend offers, keeping only the existing instance. run_spec = get_run_spec( run_name="test-run", @@ -850,8 +868,10 @@ async def test_uses_targeted_instance_offers_when_instances_specified( } mocked_backends = create_mock_backends_with_offers(offers_by_backend) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = mocked_backends + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease(mocked_backends), + ): response = await call_gpus_api(client, project.name, user.token, run_spec) assert response.status_code == 200 @@ -871,11 +891,13 @@ async def test_returns_empty_gpus_when_no_offers( repo = await create_repo(session=session, project_id=project.id) run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [] - m.return_value = [backend_mock_aws] response = await client.post( f"/api/project/{project.name}/gpus/list", @@ -1020,8 +1042,12 @@ async def test_exact_aggregation_values( BackendType.RUNPOD, "T4", 16384, 0.25, spot=True, region="us-east-1" ) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ offer_t4_spot, @@ -1030,7 +1056,6 @@ async def test_exact_aggregation_values( offer_t4_multi, ] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [ offer_runpod_rtx_east, @@ -1038,8 +1063,6 @@ async def test_exact_aggregation_values( offer_runpod_t4_east, ] - m.return_value = [backend_mock_aws, backend_mock_runpod] - response = await client.post( f"/api/project/{project.name}/gpus/list", headers=get_auth_headers(user.token), diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 70cd675155..ef888668a0 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -75,6 +75,7 @@ get_run_spec, get_ssh_fleet_configuration, list_events, + mock_lease, ) from dstack._internal.server.testing.matchers import SomeUUID4Str from dstack._internal.utils.common import render_datetime_as_api @@ -1581,8 +1582,10 @@ async def test_sorts_offers_before_limit( } if reverse_offers is not None: body["reverse_offers"] = reverse_offers - with patch("dstack._internal.server.services.backends.get_project_backends") as mock: - mock.return_value = [backend] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1655,14 +1658,16 @@ async def test_returns_run_plan_privileged_false( privileged=False, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1706,12 +1711,14 @@ async def test_forwards_full_offers_to_compute_get_offers( if body_full_offers is not None: body["full_offers"] = body_full_offers - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS get_offers_mock = backend_mock.compute.return_value.get_offers get_offers_mock.return_value = [] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1761,12 +1768,14 @@ async def test_forwards_unallocated_resources_to_compute_get_offers( if body_unallocated_resources is not None: body["unallocated_resources"] = body_unallocated_resources - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS get_offers_mock = backend_mock.compute.return_value.get_offers get_offers_mock.return_value = [] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1828,14 +1837,16 @@ async def test_returns_run_plan_privileged_true( privileged=True, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1874,11 +1885,13 @@ async def test_task_with_two_nodes_returns_two_job_plans( configuration=TaskConfiguration(commands=["echo hi"], nodes=2), ) body = {"run_spec": json.loads(run_spec.model_dump_json())} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -1965,11 +1978,13 @@ def offers_by_requirements( return [gpu_offer] return [cpu_offer] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock = Mock() + backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.side_effect = offers_by_requirements - m.return_value = [backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2015,18 +2030,20 @@ async def test_service_reservation_group_filters_backends_by_reservation_support ) body = {"run_spec": json.loads(run_spec.model_dump_json())} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() + aws_backend_mock = Mock() + verda_backend_mock = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([aws_backend_mock, verda_backend_mock]), + ): aws_backend_mock.TYPE = BackendType.AWS aws_backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.AWS, price=2) ] - verda_backend_mock = Mock() verda_backend_mock.TYPE = BackendType.VERDA verda_backend_mock.compute.return_value.get_offers.return_value = [ get_instance_offer_with_availability(backend=BackendType.VERDA, price=1) ] - m.return_value = [aws_backend_mock, verda_backend_mock] response = await client.post( f"/api/project/{project.name}/runs/get_plan", @@ -2091,14 +2108,16 @@ async def test_returns_run_plan_docker_true( docker=True, ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2154,14 +2173,16 @@ async def test_returns_run_plan_instance_volumes( volumes=[InstanceMountPoint.parse("/data:/data")], ) body = {"run_spec": run_plan_dict["run_spec"]} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [offer_aws] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [offer_runpod] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2482,8 +2503,10 @@ async def test_preserves_backend_specific_offer_order( get_instance_offer_with_availability(backend=BackendType.VASTAI, price=2.0), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock_aws, backend_mock_vastai] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_vastai]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2553,8 +2576,10 @@ async def test_offer_cli_preserves_backend_specific_offer_order_across_fleets( get_instance_offer_with_availability(backend=BackendType.VASTAI, price=2.0), ] - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - m.return_value = [backend_mock_aws, backend_mock_vastai] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_vastai]), + ): response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), @@ -2687,8 +2712,11 @@ async def test_offer_cli_deduplicates_identical_backend_offers_across_specified_ ) body = {"run_spec": run_spec.model_dump(), "for_offers_only": True} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2704,7 +2732,6 @@ async def test_offer_cli_deduplicates_identical_backend_offers_across_specified_ availability_zones=["us-a"], ) ] - m.return_value = [backend_mock_aws] response = await client.post( f"/api/project/{project.name}/runs/get_plan", @@ -2808,8 +2835,12 @@ async def test_offer_cli_without_fleet_keeps_global_offers( ), ) body = {"run_spec": run_spec.model_dump(), "for_offers_only": True} - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - backend_mock_aws = Mock() + backend_mock_aws = Mock() + backend_mock_runpod = Mock() + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock_aws, backend_mock_runpod]), + ): backend_mock_aws.TYPE = BackendType.AWS backend_mock_aws.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2823,7 +2854,6 @@ async def test_offer_cli_without_fleet_keeps_global_offers( availability=InstanceAvailability.AVAILABLE, ) ] - backend_mock_runpod = Mock() backend_mock_runpod.TYPE = BackendType.RUNPOD backend_mock_runpod.compute.return_value.get_offers.return_value = [ InstanceOfferWithAvailability( @@ -2837,7 +2867,6 @@ async def test_offer_cli_without_fleet_keeps_global_offers( availability=InstanceAvailability.AVAILABLE, ) ] - m.return_value = [backend_mock_aws, backend_mock_runpod] response = await client.post( f"/api/project/{project.name}/runs/get_plan", headers=get_auth_headers(user.token), diff --git a/src/tests/_internal/server/services/backends/test_cache.py b/src/tests/_internal/server/services/backends/test_cache.py new file mode 100644 index 0000000000..ddec5f7e6c --- /dev/null +++ b/src/tests/_internal/server/services/backends/test_cache.py @@ -0,0 +1,242 @@ +import asyncio +import uuid +from types import SimpleNamespace +from typing import cast +from unittest.mock import Mock + +import pytest + +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.models import BackendModel, ProjectModel +from dstack._internal.server.services.backends import cache as backends_cache + + +def make_backend_model(backend_type: BackendType, config: str = "{}") -> BackendModel: + backend_model = SimpleNamespace( + id=uuid.uuid4(), + type=backend_type, + config=config, + auth=SimpleNamespace(decrypted=True), + ) + return cast(BackendModel, backend_model) + + +def make_project(*backend_models: BackendModel) -> ProjectModel: + project = SimpleNamespace(id=uuid.uuid4(), backends=list(backend_models)) + return cast(ProjectModel, project) + + +def close_mock(backend: Backend) -> Mock: + return cast(Mock, backend.close) + + +async def close_unused_backends_now(monkeypatch: pytest.MonkeyPatch): + """ + Closes all backends that are not leased, as if they all were idle. + """ + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", -1) + await backends_cache.close_unused_backends() + + +class TestBackendsCache: + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE", {}) + monkeypatch.setattr(backends_cache, "_BACKENDS_CACHE_LOCKS", {}) + monkeypatch.setattr(backends_cache, "get_stored_backend_record", lambda m: m) + # Backends are created concurrently, so their order is not deterministic + self.backends: list[Mock] = [] + + def get_backend(backend_model): + backend = Mock(TYPE=backend_model.type) + self.backends.append(backend) + return backend + + monkeypatch.setattr( + backends_cache, + "get_configurator", + lambda backend_type: Mock(get_backend=Mock(side_effect=get_backend)), + ) + + def get_created_backend(self, backend_type: BackendType) -> Mock: + return next(b for b in self.backends if b.TYPE == backend_type) + + @pytest.mark.asyncio + async def test_reuses_backends_between_leases(self): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_cache.lease_project_backends(project) as first: + pass + async with backends_cache.lease_project_backends(project) as second: + pass + assert first == second == self.backends + self.backends[0].close.assert_not_called() + + @pytest.mark.asyncio + async def test_closes_backend_with_changed_config_after_release(self): + project = make_project(make_backend_model(BackendType.AWS, config="old")) + async with backends_cache.lease_project_backends(project) as [old_backend]: + project.backends = [make_backend_model(BackendType.AWS, config="new")] + async with backends_cache.lease_project_backends(project) as [new_backend]: + assert new_backend is not old_backend + await backends_cache.close_unused_backends() + close_mock(old_backend).assert_not_called() + await backends_cache.close_unused_backends() + close_mock(old_backend).assert_called_once() + close_mock(new_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_closes_deleted_backend(self): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_cache.lease_project_backends(project): + pass + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) + project.backends = project.backends[:1] + async with backends_cache.lease_project_backends(project) as backends: + assert backends == [aws_backend] + await backends_cache.close_unused_backends() + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_closes_idle_backends(self, monkeypatch): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_cache.lease_project_backends(project): + pass + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) + async with backends_cache.lease_project_backend_by_type( + project, BackendType.AWS + ) as backend: + assert backend is aws_backend + # Only GCP is idle: AWS is leased and was used just now + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", 0) + await backends_cache.close_unused_backends() + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_selecting_backend_does_not_refresh_other_backends(self, monkeypatch): + project = make_project( + make_backend_model(BackendType.AWS), make_backend_model(BackendType.GCP) + ) + async with backends_cache.lease_project_backends(project): + pass + aws_backend = self.get_created_backend(BackendType.AWS) + gcp_backend = self.get_created_backend(BackendType.GCP) + monkeypatch.setattr(backends_cache, "_BACKEND_IDLE_TTL", 0) + async with backends_cache.lease_project_backend_by_type(project, BackendType.AWS): + await backends_cache.close_unused_backends() + close_mock(gcp_backend).assert_called_once() + close_mock(aws_backend).assert_not_called() + + @pytest.mark.asyncio + async def test_does_not_close_leased_backends(self, monkeypatch): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_cache.lease_project_backends(project) as [backend]: + await close_unused_backends_now(monkeypatch) + close_mock(backend).assert_not_called() + await close_unused_backends_now(monkeypatch) + close_mock(backend).assert_called_once() + + @pytest.mark.asyncio + async def test_releases_lease_if_cancelled(self, monkeypatch): + project = make_project(make_backend_model(BackendType.AWS)) + leased = asyncio.Event() + + async def hold_lease(): + async with backends_cache.lease_project_backends(project): + leased.set() + await asyncio.Event().wait() + + task = asyncio.create_task(hold_lease()) + await leased.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await close_unused_backends_now(monkeypatch) + close_mock(self.backends[0]).assert_called_once() + + @pytest.mark.asyncio + async def test_does_not_raise_if_backend_fails_to_close(self, monkeypatch): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_cache.lease_project_backends(project) as [backend]: + close_mock(backend).side_effect = RuntimeError("close failed") + await close_unused_backends_now(monkeypatch) + close_mock(backend).assert_called_once() + + @pytest.mark.asyncio + async def test_leases_backend_by_id(self): + aws_model = make_backend_model(BackendType.AWS) + project = make_project(aws_model, make_backend_model(BackendType.GCP)) + async with backends_cache.lease_project_backend_by_id(project, aws_model.id) as backend: + assert backend is self.get_created_backend(BackendType.AWS) + + @pytest.mark.asyncio + async def test_returns_none_and_releases_leases_if_backend_not_found(self, monkeypatch): + project = make_project(make_backend_model(BackendType.AWS)) + async with backends_cache.lease_project_backend_by_type( + project, BackendType.GCP + ) as backend: + assert backend is None + async with backends_cache.lease_project_backend_by_id(project, uuid.uuid4()) as backend: + assert backend is None + [aws_backend] = self.backends + await close_unused_backends_now(monkeypatch) + close_mock(aws_backend).assert_called_once() + + @pytest.mark.asyncio + async def test_uses_selector_with_overrides(self, monkeypatch): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + return backends_with_models[0] + + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) + async with backends_cache.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ) as backend: + assert backend is self.backends[0] + async with backends_cache.lease_project_backend_by_type( + project, BackendType.AWS + ) as backend: + assert backend is None + + @pytest.mark.asyncio + async def test_releases_leases_if_selector_fails(self, monkeypatch): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + raise RuntimeError("select failed") + + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) + with pytest.raises(RuntimeError): + async with backends_cache.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ): + pass + await close_unused_backends_now(monkeypatch) + close_mock(self.backends[0]).assert_called_once() + + @pytest.mark.asyncio + async def test_raises_and_releases_leases_if_selector_returns_unknown_backend( + self, monkeypatch + ): + project = make_project(make_backend_model(BackendType.DSTACK)) + + async def select(project, backend_type, backends_with_models): + return backends_with_models[0][0], Mock() + + monkeypatch.setattr(backends_cache, "_select_project_backend_with_model_by_type", select) + with pytest.raises(ValueError): + async with backends_cache.lease_project_backend_by_type( + project, BackendType.AWS, overrides=True + ): + pass + await close_unused_backends_now(monkeypatch) + close_mock(self.backends[0]).assert_called_once() diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index a4c9cf2621..3a75af2575 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -46,6 +46,7 @@ get_job_provisioning_data, get_remote_connection_info, get_run_spec, + mock_lease, ) pytestmark = pytest.mark.usefixtures("image_config_mock") @@ -229,8 +230,8 @@ async def test_plans_each_node_group_with_its_own_requirements( async def find_optimal_fleet_with_offers_side_effect(*, job, **kwargs): gpu = job.job_spec.requirements.resources.gpu if gpu is not None and gpu.name: - return Mock(), [], [(Mock(), gpu_offer)] - return Mock(), [], [(Mock(), cpu_offer)] + return Mock(), [], [gpu_offer] + return Mock(), [], [cpu_offer] monkeypatch.setattr( "dstack._internal.server.services.runs.plan._select_candidate_fleet_models", @@ -310,8 +311,8 @@ async def test_plans_each_replica_group_with_its_own_requirements( async def find_optimal_fleet_with_offers_side_effect(*, job, **kwargs): gpu = job.job_spec.requirements.resources.gpu if gpu is not None and gpu.name: - return Mock(), [], [(Mock(), gpu_offer)] - return Mock(), [], [(Mock(), cpu_offer)] + return Mock(), [], [gpu_offer] + return Mock(), [], [cpu_offer] monkeypatch.setattr( "dstack._internal.server.services.runs.plan._select_candidate_fleet_models", @@ -839,6 +840,10 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el ) offer = get_instance_offer_with_availability() backend = AsyncMock() + monkeypatch.setattr( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend]), + ) get_offers_by_requirements_mock.return_value = [(backend, offer)] offers = await _get_backend_offers_in_fleet( @@ -849,8 +854,9 @@ async def test_keeps_unconstrained_offers_for_non_empty_cluster_fleet_without_el volumes=None, ) - assert offers == [(backend, offer)] + assert offers == [offer] get_offers_by_requirements_mock.assert_awaited_once() await_args = get_offers_by_requirements_mock.await_args assert await_args is not None + assert await_args.kwargs["backends"] == [backend] assert await_args.kwargs["master_job_provisioning_data"] is None diff --git a/src/tests/_internal/server/services/test_config.py b/src/tests/_internal/server/services/test_config.py index bc07da3b57..b9869646fc 100644 --- a/src/tests/_internal/server/services/test_config.py +++ b/src/tests/_internal/server/services/test_config.py @@ -118,7 +118,7 @@ async def test_skips_update_when_source_config_matches( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): @@ -174,7 +174,7 @@ async def test_populates_source_config_for_legacy_backend( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): @@ -212,12 +212,12 @@ async def test_forces_update_when_current_backend_config_is_unavailable( with ( patch.object(settings, "SERVER_DIR_PATH", tmp_path), patch( - "dstack._internal.server.services.backends.get_backend_config", + "dstack._internal.server.services.backends.store.get_backend_config", new_callable=AsyncMock, return_value=None, ), patch( - "dstack._internal.server.services.backends.update_backend", + "dstack._internal.server.services.backends.store.update_backend", new_callable=AsyncMock, ) as update_backend, ): diff --git a/src/tests/_internal/server/services/test_fleets.py b/src/tests/_internal/server/services/test_fleets.py index 19ae011d65..ce207d2421 100644 --- a/src/tests/_internal/server/services/test_fleets.py +++ b/src/tests/_internal/server/services/test_fleets.py @@ -4,7 +4,6 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.fleets import ( @@ -17,7 +16,6 @@ ) from dstack._internal.core.models.instances import RemoteConnectionInfo from dstack._internal.server.models import FleetModel, ProjectModel -from dstack._internal.server.services.backends import get_project_backends from dstack._internal.server.services.fleets import ( get_fleet_master_instance_provisioning_data, get_plan, @@ -30,14 +28,17 @@ get_fleet_spec, get_job_provisioning_data, get_ssh_key, + mock_lease, ) class TestGetPlanSSHFleetHostsValidation: @pytest.fixture - def get_project_backends_mock(self, monkeypatch: pytest.MonkeyPatch) -> list[Backend]: - mock = Mock(spec_set=get_project_backends, return_value=[]) - monkeypatch.setattr("dstack._internal.server.services.backends.get_project_backends", mock) + def lease_project_backends_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = mock_lease([]) + monkeypatch.setattr( + "dstack._internal.server.services.backends.cache.lease_project_backends", mock + ) return mock def get_ssh_fleet_spec( @@ -74,7 +75,7 @@ async def create_fleet( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_same_fleet_update(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -88,7 +89,7 @@ async def test_ok_same_fleet_update(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_deleted_instances_ignored(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -106,7 +107,7 @@ async def test_ok_deleted_instances_ignored(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_ok_no_common_hosts_with_another_fleet(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -120,7 +121,7 @@ async def test_ok_no_common_hosts_with_another_fleet(self, session: AsyncSession @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_another_fleet_same_project(self, session: AsyncSession): user = await create_user(session=session) project = await create_project(session=session, owner=user) @@ -138,7 +139,7 @@ async def test_error_another_fleet_same_project(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_another_fleet_another_project(self, session: AsyncSession): another_user = await create_user(session=session, name="another-user") another_project = await create_project( @@ -160,7 +161,7 @@ async def test_error_another_fleet_another_project(self, session: AsyncSession): @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - @pytest.mark.usefixtures("test_db", "get_project_backends_mock") + @pytest.mark.usefixtures("test_db", "lease_project_backends_mock") async def test_error_fleet_spec_without_name(self, session: AsyncSession): # Even if the user apply the same configuration again, we cannot be sure if it is the same # fleet or a brand new fleet, as we identify fleets by name. diff --git a/src/tests/_internal/server/services/test_offers.py b/src/tests/_internal/server/services/test_offers.py index 7ff0b4fe4a..02658c50e4 100644 --- a/src/tests/_internal/server/services/test_offers.py +++ b/src/tests/_internal/server/services/test_offers.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest @@ -21,85 +21,76 @@ class TestGetOffersByRequirements: async def test_returns_all_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer), (runpod_backend_mock, runpod_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) + runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, aws_offer), (runpod_backend_mock, runpod_offer)] @pytest.mark.asyncio async def test_returns_multinode_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - vastai_backend_mock = Mock() - vastai_backend_mock.TYPE = BackendType.VASTAI - vastai_offer = get_instance_offer_with_availability(backend=BackendType.VASTAI) - vastai_backend_mock.compute.return_value.get_offers.return_value = [vastai_offer] - m.return_value = [aws_backend_mock, vastai_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - multinode=True, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + vastai_backend_mock = Mock() + vastai_backend_mock.TYPE = BackendType.VASTAI + vastai_offer = get_instance_offer_with_availability(backend=BackendType.VASTAI) + vastai_backend_mock.compute.return_value.get_offers.return_value = [vastai_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, vastai_backend_mock], + profile=profile, + requirements=requirements, + multinode=True, + ) + assert res == [(aws_backend_mock, aws_offer)] @pytest.mark.asyncio async def test_returns_volume_offers(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer1 = get_instance_offer_with_availability( - backend=BackendType.RUNPOD, region="eu" - ) - runpod_offer2 = get_instance_offer_with_availability( - backend=BackendType.RUNPOD, region="us" - ) - runpod_backend_mock.compute.return_value.get_offers.return_value = [ - runpod_offer1, - runpod_offer2, - ] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - volumes=[ - [ - get_volume( - configuration=get_volume_configuration( - backend=BackendType.RUNPOD, region="us" - ) + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer1 = get_instance_offer_with_availability( + backend=BackendType.RUNPOD, region="eu" + ) + runpod_offer2 = get_instance_offer_with_availability( + backend=BackendType.RUNPOD, region="us" + ) + runpod_backend_mock.compute.return_value.get_offers.return_value = [ + runpod_offer1, + runpod_offer2, + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + volumes=[ + [ + get_volume( + configuration=get_volume_configuration( + backend=BackendType.RUNPOD, region="us" ) - ] - ], - ) - m.assert_awaited_once() - assert res == [(runpod_backend_mock, runpod_offer2)] + ) + ] + ], + ) + assert res == [(runpod_backend_mock, runpod_offer2)] @pytest.mark.asyncio async def test_global_volume_matches_cpu_and_gpu_regions(self): @@ -113,16 +104,12 @@ async def test_global_volume_matches_cpu_and_gpu_regions(self): aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us") aws_backend.compute.return_value.get_offers.return_value = [aws_offer] - with patch( - "dstack._internal.server.services.backends.get_project_backends", - return_value=[daytona_backend, aws_backend], - ): - offers = await get_offers_by_requirements( - project=Mock(), - profile=Profile(), - requirements=Requirements(resources=ResourcesSpec()), - volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], - ) + offers = await get_offers_by_requirements( + backends=[daytona_backend, aws_backend], + profile=Profile(), + requirements=Requirements(resources=ResourcesSpec()), + volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], + ) assert offers == [(daytona_backend, cpu_offer), (daytona_backend, gpu_offer)] @@ -130,113 +117,102 @@ async def test_global_volume_matches_cpu_and_gpu_regions(self): async def test_returns_volume_offers_without_region(self): profile = Profile(name="test") requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - kubernetes_backend_mock = Mock() - kubernetes_backend_mock.TYPE = BackendType.KUBERNETES - kubernetes_offer = get_instance_offer_with_availability( - backend=BackendType.KUBERNETES, - region="", - availability_zones=None, - ) - kubernetes_backend_mock.compute.return_value.get_offers.return_value = [ - kubernetes_offer, - get_instance_offer_with_availability( - backend=BackendType.KUBERNETES, region="other-cluster" - ), - ] - m.return_value = [aws_backend_mock, kubernetes_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - volumes=[[get_volume(configuration=get_kubernetes_volume_configuration())]], - ) - m.assert_awaited_once() - assert res == [(kubernetes_backend_mock, kubernetes_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + kubernetes_backend_mock = Mock() + kubernetes_backend_mock.TYPE = BackendType.KUBERNETES + kubernetes_offer = get_instance_offer_with_availability( + backend=BackendType.KUBERNETES, + region="", + availability_zones=None, + ) + kubernetes_backend_mock.compute.return_value.get_offers.return_value = [ + kubernetes_offer, + get_instance_offer_with_availability( + backend=BackendType.KUBERNETES, region="other-cluster" + ), + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, kubernetes_backend_mock], + profile=profile, + requirements=requirements, + volumes=[[get_volume(configuration=get_kubernetes_volume_configuration())]], + ) + assert res == [(kubernetes_backend_mock, kubernetes_offer)] @pytest.mark.asyncio async def test_returns_az_offers(self): profile = Profile(name="test", availability_zones=["az1", "az3"]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer1 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az1"] - ) - aws_offer2 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az2"] - ) - aws_offer3 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az2", "az3"] - ) - expected_aws_offer3 = aws_offer3.model_copy() - expected_aws_offer3.availability_zones = ["az3"] - aws_offer4 = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=None - ) - aws_backend_mock.compute.return_value.get_offers.return_value = [ - aws_offer1, - aws_offer2, - aws_offer3, - aws_offer4, - ] - m.return_value = [aws_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - m.assert_awaited_once() - assert res == [(aws_backend_mock, aws_offer1), (aws_backend_mock, expected_aws_offer3)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer1 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az1"] + ) + aws_offer2 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az2"] + ) + aws_offer3 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az2", "az3"] + ) + expected_aws_offer3 = aws_offer3.model_copy() + expected_aws_offer3.availability_zones = ["az3"] + aws_offer4 = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=None + ) + aws_backend_mock.compute.return_value.get_offers.return_value = [ + aws_offer1, + aws_offer2, + aws_offer3, + aws_offer4, + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, aws_offer1), (aws_backend_mock, expected_aws_offer3)] @pytest.mark.asyncio async def test_returns_az_offers_ignoring_case(self): profile = Profile(name="test", availability_zones=["AZ1"]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability( - backend=BackendType.AWS, availability_zones=["az1", "az2"] - ) - # The offer keeps the zone spelling reported by the backend. - expected_aws_offer = aws_offer.model_copy() - expected_aws_offer.availability_zones = ["az1"] - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - m.return_value = [aws_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - ) - assert res == [(aws_backend_mock, expected_aws_offer)] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability( + backend=BackendType.AWS, availability_zones=["az1", "az2"] + ) + # The offer keeps the zone spelling reported by the backend. + expected_aws_offer = aws_offer.model_copy() + expected_aws_offer.availability_zones = ["az1"] + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock], + profile=profile, + requirements=requirements, + ) + assert res == [(aws_backend_mock, expected_aws_offer)] @pytest.mark.asyncio async def test_returns_no_offers_for_multinode_instance_mounts_and_non_multinode_backend(self): # Regression test for https://github.com/dstackai/dstack/issues/2211 profile = Profile(name="test", backends=[BackendType.RUNPOD]) requirements = Requirements(resources=ResourcesSpec()) - with patch("dstack._internal.server.services.backends.get_project_backends") as m: - aws_backend_mock = Mock() - aws_backend_mock.TYPE = BackendType.AWS - aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) - aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] - runpod_backend_mock = Mock() - runpod_backend_mock.TYPE = BackendType.RUNPOD - runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) - runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] - m.return_value = [aws_backend_mock, runpod_backend_mock] - res = await get_offers_by_requirements( - project=Mock(), - profile=profile, - requirements=requirements, - multinode=True, - instance_mounts=True, - ) - m.assert_awaited_once() - assert res == [] + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD) + runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock], + profile=profile, + requirements=requirements, + multinode=True, + instance_mounts=True, + ) + assert res == [] diff --git a/src/tests/_internal/utils/test_event_loop.py b/src/tests/_internal/utils/test_event_loop.py deleted file mode 100644 index 98812b52ac..0000000000 --- a/src/tests/_internal/utils/test_event_loop.py +++ /dev/null @@ -1,18 +0,0 @@ -import asyncio - -from dstack._internal.utils.event_loop import DaemonEventLoop - - -def test_daemon_event_loop(): - q = asyncio.Queue() - - async def worker(i): - await q.put(i) - - async def all_workers(): - await asyncio.gather(*[worker(i) for i in range(3)]) - - loop = DaemonEventLoop() - loop.await_(all_workers()) - assert q.qsize() == 3 - assert {loop.await_(q.get()) for _ in range(3)} == {0, 1, 2}