dstack 0.19.7__py3-none-any.whl → 0.19.9__py3-none-any.whl

dstack/_internal/core/backends/base/offers.py

@@ -2,6 +2,7 @@ from dataclasses import asdict
 from typing import Callable, List, Optional
 
 import gpuhunt
+from pydantic import parse_obj_as
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
@@ -11,13 +12,14 @@ from dstack._internal.core.models.instances import (
     InstanceType,
     Resources,
 )
-from dstack._internal.core.models.resources import DEFAULT_DISK, Memory, Range
+from dstack._internal.core.models.resources import DEFAULT_DISK, CPUSpec, Memory, Range
 from dstack._internal.core.models.runs import Requirements
 
 # Offers not supported by all dstack versions are hidden behind one or more flags.
 # This list enables the flags that are currently supported.
 SUPPORTED_GPUHUNT_FLAGS = [
     "oci-spot",
+    "lambda-arm",
 ]
 
 
@@ -71,6 +73,7 @@ def catalog_item_to_offer(
     if disk_size_mib is None:
         return None
     resources = Resources(
+        cpu_arch=item.cpu_arch,
         cpus=item.cpu,
         memory_mib=round(item.memory * 1024),
         gpus=gpus,
@@ -90,6 +93,9 @@ def catalog_item_to_offer(
 
 
 def offer_to_catalog_item(offer: InstanceOffer) -> gpuhunt.CatalogItem:
+    cpu_arch = offer.instance.resources.cpu_arch
+    if cpu_arch is None:
+        cpu_arch = gpuhunt.CPUArchitecture.X86
     gpu_count = len(offer.instance.resources.gpus)
     gpu_vendor = None
     gpu_name = None
@@ -104,6 +110,7 @@ def offer_to_catalog_item(offer: InstanceOffer) -> gpuhunt.CatalogItem:
         instance_name=offer.instance.name,
         location=offer.region,
         price=offer.price,
+        cpu_arch=cpu_arch,
         cpu=offer.instance.resources.cpus,
         memory=offer.instance.resources.memory_mib / 1024,
         gpu_count=gpu_count,
@@ -125,8 +132,11 @@ def requirements_to_query_filter(req: Optional[Requirements]) -> gpuhunt.QueryFi
 
     res = req.resources
     if res.cpu:
-        q.min_cpu = res.cpu.min
-        q.max_cpu = res.cpu.max
+        # TODO: Remove in 0.20. Use res.cpu directly
+        cpu = parse_obj_as(CPUSpec, res.cpu)
+        q.cpu_arch = cpu.arch
+        q.min_cpu = cpu.count.min
+        q.max_cpu = cpu.count.max
     if res.memory:
         q.min_memory = res.memory.min
         q.max_memory = res.memory.max

dstack/_internal/core/backends/cudo/compute.py

@@ -18,6 +18,7 @@ from dstack._internal.core.models.instances import (
     InstanceConfiguration,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
 
@@ -58,6 +59,7 @@ class CudoCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         vm_id = generate_unique_instance_name(instance_config, max_length=MAX_RESOURCE_NAME_LEN)
         public_keys = instance_config.get_public_keys()
@@ -145,7 +147,7 @@ class CudoCompute(
 
 
 def _get_image_id(cuda: bool) -> str:
-    image_name = "ubuntu-2204-nvidia-535-docker-v20240214" if cuda else "ubuntu-2204"
+    image_name = "ubuntu-2204-nvidia-535-docker-v20241017" if cuda else "ubuntu-2204"
     return image_name
 
 

dstack/_internal/core/backends/datacrunch/compute.py

@@ -20,6 +20,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
@@ -85,6 +86,7 @@ class DataCrunchCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=MAX_INSTANCE_NAME_LEN

dstack/_internal/core/backends/gcp/auth.py

@@ -19,7 +19,7 @@ def authenticate(creds: AnyGCPCreds, project_id: Optional[str] = None) -> Tuple[
     credentials, credentials_project_id = get_credentials(creds)
     if project_id is None:
         # If project_id is not specified explicitly, try using credentials' project_id.
-        # Explicit project_id takes precedence bacause credentials' project_id may be irrelevant.
+        # Explicit project_id takes precedence because credentials' project_id may be irrelevant.
         # For example, with Workload Identity Federation for GKE, it's cluster project_id.
         project_id = credentials_project_id
     if project_id is None:

dstack/_internal/core/backends/gcp/compute.py

@@ -1,10 +1,12 @@
 import concurrent.futures
 import json
+import threading
 from collections import defaultdict
 from typing import Callable, Dict, List, Literal, Optional, Tuple
 
 import google.api_core.exceptions
 import google.cloud.compute_v1 as compute_v1
+from cachetools import TTLCache, cachedmethod
 from google.cloud import tpu_v2
 from gpuhunt import KNOWN_TPUS
 
@@ -98,6 +100,8 @@ class GCPCompute(
         self.resource_policies_client = compute_v1.ResourcePoliciesClient(
             credentials=self.credentials
         )
+        self._extra_subnets_cache_lock = threading.Lock()
+        self._extra_subnets_cache = TTLCache(maxsize=30, ttl=60)
 
     def get_offers(
         self, requirements: Optional[Requirements] = None
@@ -166,6 +170,7 @@ class GCPCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=gcp_resources.MAX_RESOURCE_NAME_LEN
@@ -192,18 +197,16 @@ class GCPCompute(
             config=self.config,
             region=instance_offer.region,
         )
-        extra_subnets = _get_extra_subnets(
-            subnetworks_client=self.subnetworks_client,
-            config=self.config,
+        extra_subnets = self._get_extra_subnets(
             region=instance_offer.region,
             instance_type_name=instance_offer.instance.name,
         )
         placement_policy = None
-        if instance_config.placement_group_name is not None:
+        if placement_group is not None:
             placement_policy = gcp_resources.get_placement_policy_resource_name(
                 project_id=self.config.project_id,
                 region=instance_offer.region,
-                placement_policy=instance_config.placement_group_name,
+                placement_policy=placement_group.name,
             )
         labels = {
             "owner": "dstack",
@@ -406,6 +409,7 @@ class GCPCompute(
     def create_placement_group(
         self,
         placement_group: PlacementGroup,
+        master_instance_offer: InstanceOffer,
     ) -> PlacementGroupProvisioningData:
         policy = compute_v1.ResourcePolicy(
             name=placement_group.name,
@@ -440,6 +444,16 @@ class GCPCompute(
                 raise PlacementGroupInUseError()
             raise
 
+    def is_suitable_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        instance_offer: InstanceOffer,
+    ) -> bool:
+        return (
+            placement_group.configuration.backend == BackendType.GCP
+            and placement_group.configuration.region == instance_offer.region
+        )
+
     def create_gateway(
         self,
         configuration: GatewayComputeConfiguration,
@@ -757,6 +771,38 @@ class GCPCompute(
             instance_id,
         )
 
+    @cachedmethod(
+        cache=lambda self: self._extra_subnets_cache,
+        lock=lambda self: self._extra_subnets_cache_lock,
+    )
+    def _get_extra_subnets(
+        self,
+        region: str,
+        instance_type_name: str,
+    ) -> List[Tuple[str, str]]:
+        if self.config.extra_vpcs is None:
+            return []
+        if instance_type_name == "a3-megagpu-8g":
+            subnets_num = 8
+        elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+            subnets_num = 4
+        else:
+            return []
+        extra_subnets = []
+        for vpc_name in self.config.extra_vpcs[:subnets_num]:
+            subnet = gcp_resources.get_vpc_subnet_or_error(
+                subnetworks_client=self.subnetworks_client,
+                vpc_project_id=self.config.vpc_project_id or self.config.project_id,
+                vpc_name=vpc_name,
+                region=region,
+            )
+            vpc_resource_name = gcp_resources.vpc_name_to_vpc_resource_name(
+                project_id=self.config.vpc_project_id or self.config.project_id,
+                vpc_name=vpc_name,
+            )
+            extra_subnets.append((vpc_resource_name, subnet))
+        return extra_subnets
+
 
 def _supported_instances_and_zones(
     regions: List[str],
@@ -831,36 +877,6 @@ def _get_vpc_subnet(
     )
 
 
-def _get_extra_subnets(
-    subnetworks_client: compute_v1.SubnetworksClient,
-    config: GCPConfig,
-    region: str,
-    instance_type_name: str,
-) -> List[Tuple[str, str]]:
-    if config.extra_vpcs is None:
-        return []
-    if instance_type_name == "a3-megagpu-8g":
-        subnets_num = 8
-    elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
-        subnets_num = 4
-    else:
-        return []
-    extra_subnets = []
-    for vpc_name in config.extra_vpcs[:subnets_num]:
-        subnet = gcp_resources.get_vpc_subnet_or_error(
-            subnetworks_client=subnetworks_client,
-            vpc_project_id=config.vpc_project_id or config.project_id,
-            vpc_name=vpc_name,
-            region=region,
-        )
-        vpc_resource_name = gcp_resources.vpc_name_to_vpc_resource_name(
-            project_id=config.vpc_project_id or config.project_id,
-            vpc_name=vpc_name,
-        )
-        extra_subnets.append((vpc_resource_name, subnet))
-    return extra_subnets
-
-
 def _get_image_id(instance_type_name: str, cuda: bool) -> str:
     if instance_type_name == "a3-megagpu-8g":
         image_name = "dstack-a3mega-5"

dstack/_internal/core/backends/lambdalabs/compute.py

@@ -20,6 +20,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 
 MAX_INSTANCE_NAME_LEN = 60
@@ -46,7 +47,10 @@ class LambdaCompute(
         return offers_with_availability
 
     def create_instance(
-        self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=MAX_INSTANCE_NAME_LEN
@@ -89,7 +93,10 @@ class LambdaCompute(
         instance_info = _get_instance_info(self.api_client, provisioning_data.instance_id)
         if instance_info is not None and instance_info["status"] != "booting":
             provisioning_data.hostname = instance_info["ip"]
-            commands = get_shim_commands(authorized_keys=[project_ssh_public_key])
+            commands = get_shim_commands(
+                authorized_keys=[project_ssh_public_key],
+                arch=provisioning_data.instance_type.resources.cpu_arch,
+            )
             # shim is assumed to be run under root
             launch_command = "sudo sh -c '" + "&& ".join(commands) + "'"
             thread = Thread(
@@ -179,13 +186,18 @@ def _setup_instance(
     ssh_private_key: str,
 ):
     setup_commands = (
-        "mkdir /home/ubuntu/.dstack && "
-        "sudo apt-get update && "
-        "sudo apt-get install -y --no-install-recommends nvidia-container-toolkit && "
-        "sudo nvidia-ctk runtime configure --runtime=docker && "
-        "sudo pkill -SIGHUP dockerd"
+        "mkdir /home/ubuntu/.dstack",
+        "sudo apt-get update",
+        "sudo apt-get install -y --no-install-recommends nvidia-container-toolkit",
+        "sudo install -d -m 0755 /etc/docker",
+        # Workaround for https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
+        """echo '{"exec-opts":["native.cgroupdriver=cgroupfs"]}' | sudo tee /etc/docker/daemon.json""",
+        "sudo nvidia-ctk runtime configure --runtime=docker",
+        "sudo systemctl restart docker.service",  # `systemctl reload` (`kill -HUP`) won't work
+    )
+    _run_ssh_command(
+        hostname=hostname, ssh_private_key=ssh_private_key, command=" && ".join(setup_commands)
     )
-    _run_ssh_command(hostname=hostname, ssh_private_key=ssh_private_key, command=setup_commands)
 
 
 def _launch_runner(

dstack/_internal/core/backends/local/compute.py

@@ -15,6 +15,7 @@ from dstack._internal.core.models.instances import (
     InstanceType,
     Resources,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run
 from dstack._internal.core.models.volumes import Volume, VolumeProvisioningData
 from dstack._internal.utils.logging import get_logger
@@ -53,6 +54,7 @@ class LocalCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         return JobProvisioningData(
             backend=instance_offer.backend,

dstack/_internal/core/backends/nebius/compute.py

@@ -1,4 +1,5 @@
 import json
+import random
 import shlex
 import time
 from functools import cached_property
@@ -13,13 +14,19 @@ from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     generate_unique_instance_name,
     get_user_data,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.nebius import resources
+from dstack._internal.core.backends.nebius.fabrics import get_suitable_infiniband_fabrics
 from dstack._internal.core.backends.nebius.models import NebiusConfig, NebiusServiceAccountCreds
-from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
+from dstack._internal.core.errors import (
+    BackendError,
+    NotYetTerminated,
+    ProvisioningError,
+)
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel
 from dstack._internal.core.models.instances import (
@@ -28,6 +35,11 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import (
+    PlacementGroup,
+    PlacementGroupProvisioningData,
+    PlacementStrategy,
+)
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
@@ -72,6 +84,7 @@ SUPPORTED_PLATFORMS = [
 class NebiusCompute(
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     Compute,
 ):
     def __init__(self, config: NebiusConfig):
@@ -121,6 +134,7 @@ class NebiusCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         # NOTE: This method can block for a long time as it waits for the boot disk to be created
         # and the instance to enter the STARTING state. This has to be done in create_instance so
@@ -128,6 +142,14 @@ class NebiusCompute(
         # instance.
         instance_name = generate_unique_instance_name(instance_config)
         platform, preset = instance_offer.instance.name.split()
+        cluster_id = None
+        if placement_group:
+            assert placement_group.provisioning_data is not None
+            backend_data = NebiusPlacementGroupBackendData.load(
+                placement_group.provisioning_data.backend_data
+            )
+            if backend_data.cluster is not None:
+                cluster_id = backend_data.cluster.id
         create_disk_op = resources.create_disk(
             sdk=self._sdk,
             name=instance_name,
@@ -155,6 +177,7 @@ class NebiusCompute(
                 ),
                 platform=platform,
                 preset=preset,
+                cluster_id=cluster_id,
                 disk_id=create_disk_op.resource_id,
                 subnet_id=self._get_subnet_id(instance_offer.region),
             )
@@ -230,6 +253,63 @@ class NebiusCompute(
         with resources.ignore_errors([StatusCode.NOT_FOUND]):
             resources.delete_disk(self._sdk, backend_data_parsed.boot_disk_id)
 
+    def create_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        master_instance_offer: InstanceOffer,
+    ) -> PlacementGroupProvisioningData:
+        assert placement_group.configuration.placement_strategy == PlacementStrategy.CLUSTER
+        backend_data = NebiusPlacementGroupBackendData(cluster=None)
+        # Only create a Nebius cluster if the instance supports it.
+        # For other instances, return dummy PlacementGroupProvisioningData.
+        if fabrics := get_suitable_infiniband_fabrics(
+            master_instance_offer, allowed_fabrics=self.config.fabrics
+        ):
+            fabric = random.choice(fabrics)
+            op = resources.create_cluster(
+                self._sdk,
+                name=placement_group.name,
+                project_id=self._region_to_project_id[placement_group.configuration.region],
+                fabric=fabric,
+            )
+            backend_data.cluster = NebiusClusterBackendData(id=op.resource_id, fabric=fabric)
+        return PlacementGroupProvisioningData(
+            backend=BackendType.NEBIUS,
+            backend_data=backend_data.json(),
+        )
+
+    def delete_placement_group(self, placement_group: PlacementGroup) -> None:
+        assert placement_group.provisioning_data is not None
+        backend_data = NebiusPlacementGroupBackendData.load(
+            placement_group.provisioning_data.backend_data
+        )
+        if backend_data.cluster is not None:
+            with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                resources.delete_cluster(self._sdk, backend_data.cluster.id)
+
+    def is_suitable_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        instance_offer: InstanceOffer,
+    ) -> bool:
+        if not (
+            placement_group.configuration.backend == BackendType.NEBIUS
+            and placement_group.configuration.region == instance_offer.region
+        ):
+            return False
+        assert placement_group.provisioning_data is not None
+        backend_data = NebiusPlacementGroupBackendData.load(
+            placement_group.provisioning_data.backend_data
+        )
+        return (
+            backend_data.cluster is None
+            or backend_data.cluster.fabric
+            in get_suitable_infiniband_fabrics(
+                instance_offer,
+                allowed_fabrics=None,  # enforced at cluster creation time, no need to enforce here
+            )
+        )
+
 
 class NebiusInstanceBackendData(CoreModel):
     boot_disk_id: str
@@ -240,6 +320,20 @@ class NebiusInstanceBackendData(CoreModel):
         return cls.__response__.parse_raw(raw)
 
 
+class NebiusClusterBackendData(CoreModel):
+    id: str
+    fabric: str
+
+
+class NebiusPlacementGroupBackendData(CoreModel):
+    cluster: Optional[NebiusClusterBackendData]
+
+    @classmethod
+    def load(cls, raw: Optional[str]) -> "NebiusPlacementGroupBackendData":
+        assert raw is not None
+        return cls.__response__.parse_raw(raw)
+
+
 def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None:
     start = time.monotonic()
     while True:

dstack/_internal/core/backends/nebius/configurator.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.configurator import (
 )
 from dstack._internal.core.backends.nebius import resources
 from dstack._internal.core.backends.nebius.backend import NebiusBackend
+from dstack._internal.core.backends.nebius.fabrics import get_all_infiniband_fabrics
 from dstack._internal.core.backends.nebius.models import (
     AnyNebiusBackendConfig,
     NebiusBackendConfig,
@@ -38,6 +39,16 @@ class NebiusConfigurator(Configurator):
                 fields=[["creds"]],
                 details=str(e),
             )
+        valid_fabrics = get_all_infiniband_fabrics()
+        if invalid_fabrics := set(config.fabrics or []) - valid_fabrics:
+            raise_invalid_credentials_error(
+                fields=[["fabrics"]],
+                details=(
+                    "These InfiniBand fabrics do not exist or are not known to dstack:"
+                    f" {sorted(invalid_fabrics)}. Omit `fabrics` to allow all fabrics or select"
+                    f" some of the valid options: {sorted(valid_fabrics)}"
+                ),
+            )
 
     def create_backend(
         self, project_name: str, config: NebiusBackendConfigWithCreds

dstack/_internal/core/backends/nebius/fabrics.py

@@ -0,0 +1,48 @@
+from collections.abc import Container
+from dataclasses import dataclass
+from typing import Optional
+
+from dstack._internal.core.models.instances import InstanceOffer
+
+
+@dataclass(frozen=True)
+class InfinibandFabric:
+    name: str
+    platform: str
+    region: str
+
+
+# https://docs.nebius.com/compute/clusters/gpu#fabrics
+INFINIBAND_FABRICS = [
+    InfinibandFabric("fabric-2", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-3", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-4", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-5", "gpu-h200-sxm", "eu-west1"),
+    InfinibandFabric("fabric-6", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-7", "gpu-h200-sxm", "eu-north1"),
+    InfinibandFabric("us-central1-a", "gpu-h200-sxm", "us-central1"),
+]
+
+
+def get_suitable_infiniband_fabrics(
+    offer: InstanceOffer, allowed_fabrics: Optional[Container[str]]
+) -> list[str]:
+    if len(offer.instance.resources.gpus) < 8:
+        # From the create VM page in the Nebius Console:
+        # > Only virtual machines with at least 8 NVIDIA® Hopper® H100 or H200 GPUs
+        # > can be added to the cluster
+        return []
+    platform, _ = offer.instance.name.split()
+    return [
+        f.name
+        for f in INFINIBAND_FABRICS
+        if (
+            f.platform == platform
+            and f.region == offer.region
+            and (allowed_fabrics is None or f.name in allowed_fabrics)
+        )
+    ]
+
+
+def get_all_infiniband_fabrics() -> set[str]:
+    return {f.name for f in INFINIBAND_FABRICS}

dstack/_internal/core/backends/nebius/models.py

@@ -5,7 +5,7 @@ from pydantic import Field, root_validator
 from dstack._internal.core.backends.base.models import fill_data
 from dstack._internal.core.models.common import CoreModel
 
-DEFAULT_PROJECT_NAME_PREFIX = "default-project"
+DEFAULT_PROJECT_NAME_PREFIX = "default"
 
 
 class NebiusServiceAccountCreds(CoreModel):
@@ -87,6 +87,14 @@ class NebiusBackendConfig(CoreModel):
         Optional[list[str]],
         Field(description="The list of allowed Nebius regions. Omit to allow all regions"),
     ] = None
+    fabrics: Annotated[
+        Optional[list[str]],
+        Field(
+            description=(
+                "The list of allowed fabrics for InfiniBand clusters. Omit to allow all fabrics"
+            )
+        ),
+    ] = None
 
 
 class NebiusBackendConfigWithCreds(NebiusBackendConfig):

dstack/_internal/core/backends/nebius/resources.py

@@ -15,14 +15,19 @@ from nebius.api.nebius.common.v1 import Operation, ResourceMetadata
 from nebius.api.nebius.compute.v1 import (
     AttachedDiskSpec,
     CreateDiskRequest,
+    CreateGpuClusterRequest,
     CreateInstanceRequest,
     DeleteDiskRequest,
+    DeleteGpuClusterRequest,
     DeleteInstanceRequest,
     DiskServiceClient,
     DiskSpec,
     ExistingDisk,
     GetInstanceRequest,
+    GpuClusterServiceClient,
+    GpuClusterSpec,
     Instance,
+    InstanceGpuClusterSpec,
     InstanceServiceClient,
     InstanceSpec,
     IPAddress,
@@ -275,6 +280,7 @@ def create_instance(
     user_data: str,
     platform: str,
     preset: str,
+    cluster_id: Optional[str],
     disk_id: str,
     subnet_id: str,
 ) -> SDKOperation[Operation]:
@@ -287,6 +293,7 @@ def create_instance(
         spec=InstanceSpec(
             cloud_init_user_data=user_data,
             resources=ResourcesSpec(platform=platform, preset=preset),
+            gpu_cluster=InstanceGpuClusterSpec(id=cluster_id) if cluster_id is not None else None,
             boot_disk=AttachedDiskSpec(
                 attach_mode=AttachedDiskSpec.AttachMode.READ_WRITE,
                 existing_disk=ExistingDisk(id=disk_id),
@@ -319,3 +326,25 @@ def delete_instance(sdk: SDK, instance_id: str) -> SDKOperation[Operation]:
             DeleteInstanceRequest(id=instance_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
         )
     )
+
+
+def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> SDKOperation[Operation]:
+    with wrap_capacity_errors():
+        return LOOP.await_(
+            GpuClusterServiceClient(sdk).create(
+                CreateGpuClusterRequest(
+                    metadata=ResourceMetadata(name=name, parent_id=project_id),
+                    spec=GpuClusterSpec(infiniband_fabric=fabric),
+                ),
+                timeout=REQUEST_TIMEOUT,
+                metadata=REQUEST_MD,
+            )
+        )
+
+
+def delete_cluster(sdk: SDK, cluster_id: str) -> None:
+    return LOOP.await_(
+        GpuClusterServiceClient(sdk).delete(
+            DeleteGpuClusterRequest(id=cluster_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )

dstack/_internal/core/backends/oci/compute.py

@@ -23,6 +23,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 
@@ -105,6 +106,7 @@ class OCICompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         region = self.regions[instance_offer.region]