diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index bf04205a87..26d4b6d1db 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -75,6 +75,9 @@ jobs: run: uv sync --all-extras - name: Run pyright uses: jakebailey/pyright-action@v3 + env: + # The default Node heap on the macOS runners (~2 GB) is not enough for pyright + NODE_OPTIONS: --max-old-space-size=4096 with: pylance-version: latest-release - name: Download frontend build diff --git a/pyproject.toml b/pyproject.toml index 7f0845c3c8..0b72fa2323 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,16 +109,16 @@ include = [ "src/dstack/plugins", "src/dstack/_internal/server", "src/dstack/_internal/core/services", - "src/dstack/_internal/core/backends/aws", - "src/dstack/_internal/core/backends/daytona", - "src/dstack/_internal/core/backends/kubernetes", - "src/dstack/_internal/core/backends/lium", - "src/dstack/_internal/core/backends/runpod", - "src/dstack/_internal/core/backends/slurm", + "src/dstack/_internal/core/backends", "src/dstack/_internal/cli/services/configurators", "src/dstack/_internal/cli/commands", "src/tests/_internal/server/background/pipeline_tasks", ] +exclude = [ + # OCI has pyright errors. Excluded rather than ignored because checking it makes pyright analyze + # the large OCI SDK, which is slow and takes a lot of memory. Editors still show its errors + "src/dstack/_internal/core/backends/oci", +] ignore = [ "src/dstack/_internal/server/migrations/versions", ] diff --git a/src/dstack/_internal/core/backends/azure/auth.py b/src/dstack/_internal/core/backends/azure/auth.py index e81105d530..c6c2bd1e5a 100644 --- a/src/dstack/_internal/core/backends/azure/auth.py +++ b/src/dstack/_internal/core/backends/azure/auth.py @@ -1,4 +1,4 @@ -from typing import Tuple, Union +from typing import Union from azure.core.exceptions import ClientAuthenticationError from azure.identity import ClientSecretCredential, DefaultAzureCredential @@ -9,26 +9,26 @@ AzureClientCreds, ) from dstack._internal.core.errors import BackendAuthError +from dstack._internal.utils.common import get_or_error AzureCredential = Union[ClientSecretCredential, DefaultAzureCredential] -def authenticate(creds: AnyAzureCreds) -> Tuple[AzureCredential, str]: - credential, tenant_id = get_credential(creds) +def authenticate(creds: AnyAzureCreds) -> AzureCredential: + credential = get_credential(creds) check_credential(credential) - return credential, tenant_id + return credential -def get_credential(creds: AnyAzureCreds) -> Tuple[AzureCredential, str]: +def get_credential(creds: AnyAzureCreds) -> AzureCredential: if isinstance(creds, AzureClientCreds): - credential = ClientSecretCredential( - tenant_id=creds.tenant_id, + return ClientSecretCredential( + # The configurator populates `tenant_id` from the backend config if it's missing + tenant_id=get_or_error(creds.tenant_id), client_id=creds.client_id, client_secret=creds.client_secret, ) - return credential, creds.tenant_id - - return DefaultAzureCredential(), None + return DefaultAzureCredential() def check_credential(credential: AzureCredential): diff --git a/src/dstack/_internal/core/backends/azure/backend.py b/src/dstack/_internal/core/backends/azure/backend.py index ec61b80530..6eac3fba65 100644 --- a/src/dstack/_internal/core/backends/azure/backend.py +++ b/src/dstack/_internal/core/backends/azure/backend.py @@ -11,7 +11,7 @@ class AzureBackend(Backend): def __init__(self, config: AzureConfig): self.config = config - self.credential, _ = auth.authenticate(self.config.creds) + self.credential = auth.authenticate(self.config.creds) self._compute = AzureCompute( config=self.config, credential=self.credential, diff --git a/src/dstack/_internal/core/backends/azure/compute.py b/src/dstack/_internal/core/backends/azure/compute.py index 089ffc1229..a6f66be392 100644 --- a/src/dstack/_internal/core/backends/azure/compute.py +++ b/src/dstack/_internal/core/backends/azure/compute.py @@ -154,14 +154,14 @@ def create_instance( allocate_public_ip = self.config.allocate_public_ips network_resource_group, network, subnet = get_resource_group_network_subnet_or_error( network_client=self._network_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), vpc_ids=self.config.vpc_ids, subnet_ids=self.config.subnet_ids, location=location, allocate_public_ip=allocate_public_ip, ) network_security_group = azure_utils.get_default_network_security_group_name( - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), location=location, ) @@ -190,7 +190,7 @@ def create_instance( compute_client=self._compute_client, subscription_id=self.config.subscription_id, location=location, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), network_security_group=network_security_group, network=network, subnet=subnet, @@ -214,7 +214,7 @@ def create_instance( logger.info("Request succeeded") public_ip, private_ip = _get_vm_public_private_ips( network_client=self._network_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), vm=vm, ) hostname = public_ip @@ -223,7 +223,7 @@ def create_instance( return JobProvisioningData( backend=instance_offer.backend, instance_type=instance_offer.instance, - instance_id=vm.name, + instance_id=get_or_error(vm.name), hostname=hostname, internal_ip=private_ip, region=location, @@ -240,7 +240,7 @@ def terminate_instance( ): _terminate_instance( compute_client=self._compute_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), instance_name=instance_id, ) @@ -256,7 +256,7 @@ def list_resource_groups(self) -> ResourceListing: try: vms = list( self._compute_client.virtual_machines.list( - resource_group_name=self.config.resource_group + resource_group_name=self.config.get_resource_group() ) ) except AzureError as e: @@ -290,7 +290,7 @@ def delete_resource_group(self, group: ListedResourceGroup) -> None: try: _terminate_instance( compute_client=self._compute_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), instance_name=get_or_error(group.primary_id), ) except ResourceNotFoundError: @@ -320,14 +320,14 @@ def create_gateway_replica( ) network_resource_group, network, subnet = get_resource_group_network_subnet_or_error( network_client=self._network_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), vpc_ids=self.config.vpc_ids, subnet_ids=self.config.subnet_ids, location=configuration.region, allocate_public_ip=True, ) network_security_group = azure_utils.get_default_network_security_group_name( - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), location=configuration.region, ) @@ -349,7 +349,7 @@ def create_gateway_replica( compute_client=self._compute_client, subscription_id=self.config.subscription_id, location=configuration.region, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), network_security_group=network_security_group, network=network, subnet=subnet, @@ -369,12 +369,12 @@ def create_gateway_replica( logger.info("Request succeeded") public_ip, _ = _get_vm_public_private_ips( network_client=self._network_client, - resource_group=self.config.resource_group, + resource_group=self.config.get_resource_group(), vm=vm, ) return GatewayReplicaProvisioningData( - instance_id=vm.name, - ip_address=public_ip, + instance_id=get_or_error(vm.name), + ip_address=get_or_error(public_ip), region=configuration.region, ) @@ -549,10 +549,11 @@ def _supported_instances(offer: InstanceOffer) -> bool: def _get_offers_with_availability( compute_client: compute_mgmt.ComputeManagementClient, - config_locations: List[str], + config_locations: Optional[List[str]], offers: List[InstanceOffer], ) -> List[InstanceOfferWithAvailability]: - offers = [offer for offer in offers if offer.region in config_locations] + if config_locations is not None: + offers = [offer for offer in offers if offer.region in config_locations] locations = set(offer.region for offer in offers) def get_location_quotas(location: str) -> List[str]: @@ -583,11 +584,12 @@ def get_location_quotas(location: str) -> List[str]: def _vm_type_available(vm_resource: ResourceSku) -> bool: - if len(vm_resource.restrictions) == 0: + restrictions = vm_resource.restrictions + if not restrictions: return True # If a VM type is restricted in "Zone", it is still available in other zone. # Otherwise the restriction type is "Location" - if vm_resource.restrictions[0].type == "Zone": + if restrictions[0].type == "Zone": return True return False @@ -671,10 +673,13 @@ def _begin_create_instance( }, ) try: + # The typed overloads of the TypeSpec-generated models (azure-mgmt-compute 38+) only + # accept nested `properties`. The flattened kwargs used here are supported at runtime + # and are the only form accepted by older SDK versions. poller = compute_client.virtual_machines.begin_create_or_update( resource_group, instance_name, - VirtualMachine( + VirtualMachine( # pyright: ignore[reportCallIssue] location=location, hardware_profile=HardwareProfile(vm_size=vm_size), storage_profile=StorageProfile( @@ -708,7 +713,7 @@ def _begin_create_instance( # Azure deletes the NIC and public IP created from these configurations # together with the VM. network_interface_configurations=[ - VirtualMachineNetworkInterfaceConfiguration( + VirtualMachineNetworkInterfaceConfiguration( # pyright: ignore[reportCallIssue] name="nic_config", network_security_group=SubResource( id=azure_utils.get_network_security_group_id( @@ -718,7 +723,7 @@ def _begin_create_instance( ) ), ip_configurations=[ - VirtualMachineNetworkInterfaceIPConfiguration( + VirtualMachineNetworkInterfaceIPConfiguration( # pyright: ignore[reportCallIssue] name="ip_config", subnet=SubResource( id=azure_utils.get_subnet_id( @@ -855,11 +860,12 @@ def _get_vm_public_private_ips( network_interface_name=nic_name, ) - private_ip = nic.ip_configurations[0].private_ip_address - if nic.ip_configurations[0].public_ip_address is None: + ip_configuration = get_or_error(nic.ip_configurations)[0] + private_ip = get_or_error(ip_configuration.private_ip_address) + if ip_configuration.public_ip_address is None: return None, private_ip - public_ip_id = nic.ip_configurations[0].public_ip_address.id + public_ip_id = get_or_error(ip_configuration.public_ip_address.id) public_ip_name = azure_utils.get_resource_name_from_resource_id(public_ip_id) public_ip = network_client.public_ip_addresses.get(resource_group, public_ip_name) return public_ip.ip_address, private_ip diff --git a/src/dstack/_internal/core/backends/azure/configurator.py b/src/dstack/_internal/core/backends/azure/configurator.py index bea1b3bad3..063ba21079 100644 --- a/src/dstack/_internal/core/backends/azure/configurator.py +++ b/src/dstack/_internal/core/backends/azure/configurator.py @@ -47,6 +47,7 @@ BackendType, ) from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore +from dstack._internal.utils.common import get_or_error LOCATIONS = [ ("(US) Central US", "centralus"), @@ -86,7 +87,7 @@ def validate_config(self, config: AzureBackendConfigWithCreds, default_creds_ena if isinstance(config.creds, AzureClientCreds): self._set_client_creds_tenant_id(config.creds, config.tenant_id) try: - credential, _ = auth.authenticate(config.creds) + credential = auth.authenticate(config.creds) except BackendAuthError: if isinstance(config.creds, AzureClientCreds): raise_invalid_credentials_error( @@ -113,7 +114,7 @@ def create_backend( config.regions = DEFAULT_LOCATIONS if isinstance(config.creds, AzureClientCreds): self._set_client_creds_tenant_id(config.creds, config.tenant_id) - credential, _ = auth.authenticate(config.creds) + credential = auth.authenticate(config.creds) if config.resource_group is None: config.resource_group = self._create_resource_group( credential=credential, @@ -393,7 +394,7 @@ def create_resource_group( location=location, ), ) - return resource_group.name + return get_or_error(resource_group.name) def resource_group_exists( self, @@ -435,7 +436,7 @@ def create_virtual_network( ], ), ).result() - return network.name, subnet_name + return get_or_error(network.name), subnet_name def create_network_security_group( self, diff --git a/src/dstack/_internal/core/backends/azure/models.py b/src/dstack/_internal/core/backends/azure/models.py index dc6c174c34..086d1bd016 100644 --- a/src/dstack/_internal/core/backends/azure/models.py +++ b/src/dstack/_internal/core/backends/azure/models.py @@ -1,8 +1,10 @@ from typing import Annotated, Dict, List, Literal, Optional, Union -from pydantic import Field, RootModel +from pydantic import Field, RootModel, model_validator +from typing_extensions import Self from dstack._internal.core.models.common import CoreModel +from dstack._internal.utils.common import get_or_error class AzureClientCreds(CoreModel): @@ -95,7 +97,11 @@ class AzureBackendConfigWithCreds(AzureBackendConfig): class AzureStoredConfig(AzureBackendConfig): - resource_group: str = "" + @model_validator(mode="after") + def validate_resource_group(self) -> Self: + if self.resource_group is None: + raise ValueError("resource_group must be set in the stored config") + return self class AzureConfig(AzureStoredConfig): @@ -106,3 +112,7 @@ def allocate_public_ips(self) -> bool: if self.public_ips is not None: return self.public_ips return True + + def get_resource_group(self) -> str: + # Guaranteed by `AzureStoredConfig.validate_resource_group()` + return get_or_error(self.resource_group) diff --git a/src/dstack/_internal/core/backends/base/offers.py b/src/dstack/_internal/core/backends/base/offers.py index f82df795a4..cad3b0b6b6 100644 --- a/src/dstack/_internal/core/backends/base/offers.py +++ b/src/dstack/_internal/core/backends/base/offers.py @@ -53,6 +53,10 @@ def get_catalog_offers( ) -> List[InstanceOffer]: """ Args: + configurable_disk_size: disk sizes the backend can provision. Used for catalog items + without a fixed disk size: the offer's disk is set to the smallest size within both + this range and the required range (`DEFAULT_DISK` if `requirements` don't specify disk). + `min` must be set, since it is the lower bound when the required range has none. catalog_item_filter: applied to raw catalog items before the conversion to `InstanceOffer` models. Use it for filtering that can be done on raw catalog fields to avoid expensive model construction for items that will be discarded. @@ -93,6 +97,8 @@ def catalog_item_to_offer( # The rest use construct() to skip redundant validation — data comes from already validated CatalogItem. gpus = [] if item.gpu_count > 0: + # gpuhunt requires GPU name and memory for items with GPUs + assert item.gpu_name is not None and item.gpu_memory is not None gpu = Gpu( vendor=item.gpu_vendor, name=item.gpu_name, memory_mib=round(item.gpu_memory * 1024) ) @@ -181,8 +187,9 @@ def requirements_to_query_filter( q.min_memory = res.memory.min q.max_memory = res.memory.max if res.disk and not ignore_disk: - q.min_disk_size = res.disk.size.min - q.max_disk_size = res.disk.size.max + # `QueryFilter` disk size bounds are typed as `int` until the next gpuhunt release + q.min_disk_size = res.disk.size.min # pyright: ignore[reportAttributeAccessIssue] + q.max_disk_size = res.disk.size.max # pyright: ignore[reportAttributeAccessIssue] if res.gpu: q.gpu_vendor = res.gpu.vendor @@ -229,7 +236,7 @@ def choose_disk_size_mib( disk_size_range = disk_size_range.intersect(configurable_disk_size) if disk_size_range is None: return None - disk_size_gib = disk_size_range.min + disk_size_gib = get_or_error(disk_size_range.min) return round(disk_size_gib * 1024) diff --git a/src/dstack/_internal/core/backends/cloudrift/compute.py b/src/dstack/_internal/core/backends/cloudrift/compute.py index 6c2f4437bf..8a505ee0db 100644 --- a/src/dstack/_internal/core/backends/cloudrift/compute.py +++ b/src/dstack/_internal/core/backends/cloudrift/compute.py @@ -94,7 +94,9 @@ def create_instance( gpu_vendor = None if instance_offer.instance.resources.gpus: - gpu_vendor = instance_offer.instance.resources.gpus[0].vendor.value + vendor = instance_offer.instance.resources.gpus[0].vendor + assert vendor is not None + gpu_vendor = vendor.value instance_ids = self.client.deploy_instance( instance_type=instance_offer.instance.name, diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index 273e8bd637..5877776dfe 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -8,6 +8,9 @@ from typing import Callable, Dict, List, Literal, Optional, Tuple import google.api_core.exceptions + +# Pyright infers compute_v1 enum members as ints in the huge generated types module, +# so accessing `.name` on them needs a pyright ignore. import google.cloud.compute_v1 as compute_v1 import gpuhunt from cachetools import TTLCache, cachedmethod @@ -834,7 +837,7 @@ def create_gateway_load_balancer( logger.debug("Creating health check for gateway %s...", configuration.gateway_name) health_check = compute_v1.HealthCheck() health_check.name = health_check_name - health_check.type_ = compute_v1.HealthCheck.Type.HTTP.name + health_check.type_ = compute_v1.HealthCheck.Type.HTTP.name # pyright: ignore[reportAttributeAccessIssue] health_check.http_health_check = compute_v1.HTTPHealthCheck(port=80) operation = self.region_health_checks_client.insert( project=self.config.project_id, @@ -846,20 +849,20 @@ def create_gateway_load_balancer( logger.debug("Creating backend service for gateway %s...", configuration.gateway_name) load_balancing_scheme = ( - compute_v1.BackendService.LoadBalancingScheme.EXTERNAL_MANAGED.name + compute_v1.BackendService.LoadBalancingScheme.EXTERNAL_MANAGED.name # pyright: ignore[reportAttributeAccessIssue] if configuration.public_ip - else compute_v1.BackendService.LoadBalancingScheme.INTERNAL_MANAGED.name + else compute_v1.BackendService.LoadBalancingScheme.INTERNAL_MANAGED.name # pyright: ignore[reportAttributeAccessIssue] ) backend_service = compute_v1.BackendService() backend_service.name = backend_service_name backend_service.load_balancing_scheme = load_balancing_scheme - backend_service.protocol = compute_v1.BackendService.Protocol.HTTP.name + backend_service.protocol = compute_v1.BackendService.Protocol.HTTP.name # pyright: ignore[reportAttributeAccessIssue] backend_service.port_name = "http" backend_service.health_checks = [health_check_resource_name] backend_service.backends = [ compute_v1.Backend( group=instance_group_resource_name, - balancing_mode=compute_v1.Backend.BalancingMode.UTILIZATION.name, + balancing_mode=compute_v1.Backend.BalancingMode.UTILIZATION.name, # pyright: ignore[reportAttributeAccessIssue] capacity_scaler=1.0, ) ] @@ -922,7 +925,7 @@ def create_gateway_load_balancer( forwarding_rule = compute_v1.ForwardingRule() forwarding_rule.name = forwarding_rule_name forwarding_rule.load_balancing_scheme = load_balancing_scheme - forwarding_rule.I_p_protocol = compute_v1.ForwardingRule.IPProtocolEnum.TCP.name + forwarding_rule.I_p_protocol = compute_v1.ForwardingRule.IPProtocolEnum.TCP.name # pyright: ignore[reportAttributeAccessIssue] forwarding_rule.port_range = "80" if configuration.certificate is None else "443" forwarding_rule.target = target_proxy_resource_name forwarding_rule.network = self.config.vpc_resource_name diff --git a/src/dstack/_internal/core/backends/gcp/resources.py b/src/dstack/_internal/core/backends/gcp/resources.py index a9f1d23541..70112e95d4 100644 --- a/src/dstack/_internal/core/backends/gcp/resources.py +++ b/src/dstack/_internal/core/backends/gcp/resources.py @@ -4,6 +4,9 @@ from typing import Dict, List, Optional, Tuple import google.api_core.exceptions + +# Pyright infers compute_v1 enum members as ints in the huge generated types module, +# so accessing `.name` on them needs a pyright ignore. import google.cloud.compute_v1 as compute_v1 from google.api_core.extended_operation import ExtendedOperation from google.api_core.operation import Operation @@ -218,7 +221,7 @@ def create_instance_struct( if spot: instance.scheduling = compute_v1.Scheduling() - instance.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.SPOT.name + instance.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.SPOT.name # pyright: ignore[reportAttributeAccessIssue] instance.scheduling.instance_termination_action = "STOP" # TODO? metadata_items = [ @@ -244,7 +247,7 @@ def create_instance_struct( reservation_project = get_resource_project(reservation.self_link) instance.reservation_affinity = compute_v1.ReservationAffinity() instance.reservation_affinity.consume_reservation_type = ( - compute_v1.ReservationAffinity.ConsumeReservationType.SPECIFIC_RESERVATION.name + compute_v1.ReservationAffinity.ConsumeReservationType.SPECIFIC_RESERVATION.name # pyright: ignore[reportAttributeAccessIssue] ) instance.reservation_affinity.key = "compute.googleapis.com/reservation-name" instance.reservation_affinity.values = [ @@ -267,16 +270,16 @@ def _get_network_interfaces( network_interface.subnetwork = subnetwork if allocate_public_ip: access = compute_v1.AccessConfig() - access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name + access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name # pyright: ignore[reportAttributeAccessIssue] access.name = "External NAT" - access.network_tier = access.NetworkTier.PREMIUM.name + access.network_tier = access.NetworkTier.PREMIUM.name # pyright: ignore[reportAttributeAccessIssue] network_interface.access_configs = [access] else: network_interface.access_configs = [] if extra_subnetworks: # Multiple interfaces are set only for GPU VM that require gVNIC for best performance - network_interface.nic_type = compute_v1.NetworkInterface.NicType.GVNIC.name + network_interface.nic_type = compute_v1.NetworkInterface.NicType.GVNIC.name # pyright: ignore[reportAttributeAccessIssue] network_interfaces = [network_interface] for network, subnetwork in extra_subnetworks or []: @@ -284,7 +287,7 @@ def _get_network_interfaces( compute_v1.NetworkInterface( network=network, subnetwork=subnetwork, - nic_type=compute_v1.NetworkInterface.NicType.GVNIC.name, + nic_type=compute_v1.NetworkInterface.NicType.GVNIC.name, # pyright: ignore[reportAttributeAccessIssue] ) ) for network, subnetwork in roce_subnetworks or []: @@ -292,7 +295,7 @@ def _get_network_interfaces( compute_v1.NetworkInterface( network=network, subnetwork=subnetwork, - nic_type=compute_v1.NetworkInterface.NicType.MRDMA.name, + nic_type=compute_v1.NetworkInterface.NicType.MRDMA.name, # pyright: ignore[reportAttributeAccessIssue] ) ) return network_interfaces @@ -499,8 +502,8 @@ def get_proxy_only_subnet_cidr_or_error( network_name = network.split("/")[-1] for subnet in subnetworks_client.list(project=project_id, region=region): if ( - subnet.purpose == compute_v1.Subnetwork.Purpose.REGIONAL_MANAGED_PROXY.name - and subnet.role == compute_v1.Subnetwork.Role.ACTIVE.name + subnet.purpose == compute_v1.Subnetwork.Purpose.REGIONAL_MANAGED_PROXY.name # pyright: ignore[reportAttributeAccessIssue] + and subnet.role == compute_v1.Subnetwork.Role.ACTIVE.name # pyright: ignore[reportAttributeAccessIssue] and subnet.network.split("/")[-1] == network_name ): return subnet.ip_cidr_range diff --git a/src/dstack/_internal/core/backends/lambdalabs/api_client.py b/src/dstack/_internal/core/backends/lambdalabs/api_client.py index 0da70ff5fc..44874e9c76 100644 --- a/src/dstack/_internal/core/backends/lambdalabs/api_client.py +++ b/src/dstack/_internal/core/backends/lambdalabs/api_client.py @@ -18,11 +18,10 @@ def validate_api_key(self) -> bool: raise e return True - def list_instance_types(self): + def list_instance_types(self) -> Dict[str, Any]: resp = self._make_request("GET", "/instance-types") - if resp.ok: - return resp.json()["data"] resp.raise_for_status() + return resp.json()["data"] def list_instances(self) -> List[dict]: resp = self._make_request("GET", "/instances") diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index 7acd3a5707..89b5795a69 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -7,10 +7,10 @@ from functools import cached_property from typing import List, Optional -from nebius.aio.operation import Operation as SDKOperation -from nebius.aio.service_error import RequestError, StatusCode +from grpc import StatusCode +from nebius.aio.service_error import RequestError from nebius.api.nebius.billing.v1 import PricingPolicy, PricingPolicyStatus -from nebius.api.nebius.common.v1 import Operation, ResourceMetadata +from nebius.api.nebius.common.v1 import ResourceMetadata from nebius.api.nebius.compute.v1 import Disk, DiskStatus, Instance, InstanceStatus from nebius.sdk import SDK @@ -223,10 +223,9 @@ def create_instance( logger.debug("Blocking until disk %s is created", create_disk_op.resource_id) resources.wait_for_operation(create_disk_op, timeout=WAIT_FOR_DISK_TIMEOUT) if not create_disk_op.successful(): - raw_op = create_disk_op.raw() raise ProvisioningError( - f"Create disk operation failed. Message: {raw_op.status.message}." - f" Details: {raw_op.status.details}" + "Create disk operation failed." + f" {resources.get_operation_error(create_disk_op)}" ) if ( instance_offer.instance.resources.spot @@ -244,10 +243,9 @@ def create_instance( create_pricing_policy_op, timeout=WAIT_FOR_PRICING_POLICY_TIMEOUT ) if not create_pricing_policy_op.successful(): - raw_op = create_pricing_policy_op.raw() raise ProvisioningError( - f"Create pricing policy operation failed. Message: {raw_op.status.message}." - f" Details: {raw_op.status.details}" + "Create pricing policy operation failed." + f" {resources.get_operation_error(create_pricing_policy_op)}" ) create_instance_op = resources.create_instance( sdk=self._sdk, @@ -473,13 +471,12 @@ def load(cls, raw: Optional[str]) -> "NebiusPlacementGroupBackendData": return validate_json_extra_ignore(cls, raw) -def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None: +def _wait_for_instance(sdk: SDK, op: resources.NebiusOperation) -> None: start = time.monotonic() while True: if op.done() and not op.successful(): raise ProvisioningError( - f"Create instance operation failed. Message: {op.raw().status.message}." - f" Details: {op.raw().status.details}" + f"Create instance operation failed. {resources.get_operation_error(op)}" ) instance = resources.get_instance(sdk, op.resource_id) if instance.status.state in [ diff --git a/src/dstack/_internal/core/backends/nebius/resources.py b/src/dstack/_internal/core/backends/nebius/resources.py index 4fe3417126..2928cfc7ab 100644 --- a/src/dstack/_internal/core/backends/nebius/resources.py +++ b/src/dstack/_internal/core/backends/nebius/resources.py @@ -8,9 +8,10 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict, Optional +from grpc import StatusCode from nebius.aio.operation import Operation as SDKOperation -from nebius.aio.service_error import RequestError, StatusCode -from nebius.aio.token.renewable import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS +from nebius.aio.service_error import RequestError +from nebius.aio.token.options import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS from nebius.api.nebius.billing.v1 import ( ComputeInstanceSpec, ComputeInstanceSpecV1, @@ -76,6 +77,7 @@ from dstack._internal.core.errors import BackendError, NoCapacityError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import validate_extra_ignore +from dstack._internal.utils.common import get_or_error from dstack._internal.utils.event_loop import DaemonEventLoop from dstack._internal.utils.logging import get_logger from dstack.version import __version__ @@ -94,6 +96,9 @@ OPTION_RENEW_REQUEST_TIMEOUT: "5", } LIST_PAGE_SIZE = 100 +# The SDK's own `Operation` does not satisfy the `OperationMessage` bound of `SDKOperation`: +# the protocol declares a mutable `id: str`, while the `Operation` setter accepts `str | None` +NebiusOperation = SDKOperation[Operation] # pyright: ignore[reportInvalidTypeArguments] # disables log messages about errors such as invalid creds or expired timeouts logging.getLogger("nebius").setLevel(logging.CRITICAL) @@ -132,7 +137,7 @@ def make_sdk(creds: NebiusServiceAccountCreds) -> SDK: def wait_for_operation( - op: SDKOperation[Operation], + op: NebiusOperation, timeout: float, interval: float = 1, ) -> None: @@ -147,6 +152,11 @@ def wait_for_operation( ) +def get_operation_error(op: NebiusOperation) -> str: + status = get_or_error(op.raw().status) + return f"Message: {status.message}. Details: {status.details}" + + def get_region_to_project_id_map( sdk: SDK, configured_regions: Optional[list[str]], configured_project_ids: Optional[list[str]] ) -> dict[str, str]: @@ -216,7 +226,7 @@ def _get_region_to_default_project_id_map( ] if len(region_projects) != 1: raise_invalid_credentials_error( - ["regions"], + [["regions"]], ( f"Could not find the default project in region {region}." " Consider setting the `projects` property in backend settings" @@ -237,13 +247,13 @@ def _get_region_to_configured_project_id_map( project = project_id_to_project.get(project_id) if project is None: raise_invalid_credentials_error( - ["projects"], + [["projects"]], f"Configured project ID {project_id!r} not found in this Nebius tenancy", ) duplicate_project_id = region_to_project_id.get(project.status.region) if duplicate_project_id: raise_invalid_credentials_error( - ["projects"], + [["projects"]], ( f"Configured projects {project_id} and {duplicate_project_id}" f" both belong to the same region {project.status.region}." @@ -286,7 +296,7 @@ def get_all_infiniband_fabrics() -> set[str]: def create_disk( sdk: SDK, name: str, project_id: str, size_mib: int, image_family: str, labels: Dict[str, str] -) -> SDKOperation[Operation]: +) -> NebiusOperation: client = DiskServiceClient(sdk) request = CreateDiskRequest( metadata=ResourceMetadata( @@ -333,7 +343,7 @@ def list_disks(sdk: SDK, project_id: str) -> list[Disk]: def create_pricing_policy( sdk: SDK, name: str, project_id: str, platform: str, max_price: str -) -> SDKOperation[Operation]: +) -> NebiusOperation: client = PricingPolicyServiceClient(sdk) request = CreatePricingPolicyRequest( metadata=ResourceMetadata(name=name, parent_id=project_id), @@ -395,7 +405,7 @@ def create_instance( preemptible: bool, pricing_policy_id: Optional[str], labels: Dict[str, str], -) -> SDKOperation[Operation]: +) -> NebiusOperation: client = InstanceServiceClient(sdk) request = CreateInstanceRequest( metadata=ResourceMetadata( @@ -453,7 +463,7 @@ def get_instance(sdk: SDK, instance_id: str) -> Instance: ) -def delete_instance(sdk: SDK, instance_id: str) -> SDKOperation[Operation]: +def delete_instance(sdk: SDK, instance_id: str) -> NebiusOperation: return LOOP.await_( InstanceServiceClient(sdk).delete( DeleteInstanceRequest(id=instance_id), @@ -476,7 +486,7 @@ def list_instances(sdk: SDK, project_id: str) -> list[Instance]: ) -def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> SDKOperation[Operation]: +def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> NebiusOperation: with wrap_capacity_errors(): return LOOP.await_( GpuClusterServiceClient(sdk).create( diff --git a/src/dstack/_internal/core/backends/vultr/compute.py b/src/dstack/_internal/core/backends/vultr/compute.py index 9b85e71eb0..1c3923dbb6 100644 --- a/src/dstack/_internal/core/backends/vultr/compute.py +++ b/src/dstack/_internal/core/backends/vultr/compute.py @@ -122,6 +122,7 @@ def create_instance( def terminate_instance( self, instance_id: str, region: str, backend_data: Optional[str] = None ) -> None: + assert backend_data is not None plan_type = json.loads(backend_data)["plan_type"] try: self.api_client.terminate_instance(instance_id=instance_id, plan_type=plan_type) @@ -134,6 +135,7 @@ def update_provisioning_data( project_ssh_public_key: str, project_ssh_private_key: str, ): + assert provisioning_data.backend_data is not None plan_type = json.loads(provisioning_data.backend_data)["plan_type"] instance_data = self.api_client.get_instance(provisioning_data.instance_id, plan_type) # Access specific fields diff --git a/src/tests/_internal/core/backends/azure/test_configurator.py b/src/tests/_internal/core/backends/azure/test_configurator.py index 1a329e0cd5..ae62700aac 100644 --- a/src/tests/_internal/core/backends/azure/test_configurator.py +++ b/src/tests/_internal/core/backends/azure/test_configurator.py @@ -30,7 +30,7 @@ def test_validate_config_valid(self): patch("dstack._internal.core.backends.azure.auth.authenticate") as authenticate_mock, patch("azure.mgmt.subscription.SubscriptionClient") as SubscriptionClientMock, ): - authenticate_mock.return_value = Mock(), Mock() + authenticate_mock.return_value = Mock() subcription_client_mock = SubscriptionClientMock.return_value subcription_client_mock.tenants.list.return_value = [Mock(tenant_id="ten1")] subcription_client_mock.subscriptions.list.return_value = [ diff --git a/src/tests/_internal/server/routers/test_backends.py b/src/tests/_internal/server/routers/test_backends.py index 665f1fff67..fcaf726e7a 100644 --- a/src/tests/_internal/server/routers/test_backends.py +++ b/src/tests/_internal/server/routers/test_backends.py @@ -505,7 +505,7 @@ async def test_create_azure_backend(self, test_db, session: AsyncSession, client "dstack._internal.core.backends.azure.configurator.network_mgmt.NetworkManagementClient" ) as NetworkManagementClientMock, ): - authenticate_mock.return_value = None, "test_tenant" + authenticate_mock.return_value = None subscription_client_mock = SubscriptionClientMock.return_value tenant_mock = Mock() tenant_mock.tenant_id = "test_tenant"