dstack 0.19.7__py3-none-any.whl → 0.19.8__py3-none-any.whl

dstack/_internal/core/backends/gcp/compute.py

@@ -1,10 +1,12 @@
 import concurrent.futures
 import json
+import threading
 from collections import defaultdict
 from typing import Callable, Dict, List, Literal, Optional, Tuple
 
 import google.api_core.exceptions
 import google.cloud.compute_v1 as compute_v1
+from cachetools import TTLCache, cachedmethod
 from google.cloud import tpu_v2
 from gpuhunt import KNOWN_TPUS
 
@@ -98,6 +100,8 @@ class GCPCompute(
         self.resource_policies_client = compute_v1.ResourcePoliciesClient(
             credentials=self.credentials
         )
+        self._extra_subnets_cache_lock = threading.Lock()
+        self._extra_subnets_cache = TTLCache(maxsize=30, ttl=60)
 
     def get_offers(
         self, requirements: Optional[Requirements] = None
@@ -166,6 +170,7 @@ class GCPCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=gcp_resources.MAX_RESOURCE_NAME_LEN
@@ -192,18 +197,16 @@ class GCPCompute(
             config=self.config,
             region=instance_offer.region,
         )
-        extra_subnets = _get_extra_subnets(
-            subnetworks_client=self.subnetworks_client,
-            config=self.config,
+        extra_subnets = self._get_extra_subnets(
             region=instance_offer.region,
             instance_type_name=instance_offer.instance.name,
         )
         placement_policy = None
-        if instance_config.placement_group_name is not None:
+        if placement_group is not None:
             placement_policy = gcp_resources.get_placement_policy_resource_name(
                 project_id=self.config.project_id,
                 region=instance_offer.region,
-                placement_policy=instance_config.placement_group_name,
+                placement_policy=placement_group.name,
             )
         labels = {
             "owner": "dstack",
@@ -406,6 +409,7 @@ class GCPCompute(
     def create_placement_group(
         self,
         placement_group: PlacementGroup,
+        master_instance_offer: InstanceOffer,
     ) -> PlacementGroupProvisioningData:
         policy = compute_v1.ResourcePolicy(
             name=placement_group.name,
@@ -440,6 +444,16 @@ class GCPCompute(
                 raise PlacementGroupInUseError()
             raise
 
+    def is_suitable_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        instance_offer: InstanceOffer,
+    ) -> bool:
+        return (
+            placement_group.configuration.backend == BackendType.GCP
+            and placement_group.configuration.region == instance_offer.region
+        )
+
     def create_gateway(
         self,
         configuration: GatewayComputeConfiguration,
@@ -757,6 +771,38 @@ class GCPCompute(
             instance_id,
         )
 
+    @cachedmethod(
+        cache=lambda self: self._extra_subnets_cache,
+        lock=lambda self: self._extra_subnets_cache_lock,
+    )
+    def _get_extra_subnets(
+        self,
+        region: str,
+        instance_type_name: str,
+    ) -> List[Tuple[str, str]]:
+        if self.config.extra_vpcs is None:
+            return []
+        if instance_type_name == "a3-megagpu-8g":
+            subnets_num = 8
+        elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+            subnets_num = 4
+        else:
+            return []
+        extra_subnets = []
+        for vpc_name in self.config.extra_vpcs[:subnets_num]:
+            subnet = gcp_resources.get_vpc_subnet_or_error(
+                subnetworks_client=self.subnetworks_client,
+                vpc_project_id=self.config.vpc_project_id or self.config.project_id,
+                vpc_name=vpc_name,
+                region=region,
+            )
+            vpc_resource_name = gcp_resources.vpc_name_to_vpc_resource_name(
+                project_id=self.config.vpc_project_id or self.config.project_id,
+                vpc_name=vpc_name,
+            )
+            extra_subnets.append((vpc_resource_name, subnet))
+        return extra_subnets
+
 
 def _supported_instances_and_zones(
     regions: List[str],
@@ -831,36 +877,6 @@ def _get_vpc_subnet(
     )
 
 
-def _get_extra_subnets(
-    subnetworks_client: compute_v1.SubnetworksClient,
-    config: GCPConfig,
-    region: str,
-    instance_type_name: str,
-) -> List[Tuple[str, str]]:
-    if config.extra_vpcs is None:
-        return []
-    if instance_type_name == "a3-megagpu-8g":
-        subnets_num = 8
-    elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
-        subnets_num = 4
-    else:
-        return []
-    extra_subnets = []
-    for vpc_name in config.extra_vpcs[:subnets_num]:
-        subnet = gcp_resources.get_vpc_subnet_or_error(
-            subnetworks_client=subnetworks_client,
-            vpc_project_id=config.vpc_project_id or config.project_id,
-            vpc_name=vpc_name,
-            region=region,
-        )
-        vpc_resource_name = gcp_resources.vpc_name_to_vpc_resource_name(
-            project_id=config.vpc_project_id or config.project_id,
-            vpc_name=vpc_name,
-        )
-        extra_subnets.append((vpc_resource_name, subnet))
-    return extra_subnets
-
-
 def _get_image_id(instance_type_name: str, cuda: bool) -> str:
     if instance_type_name == "a3-megagpu-8g":
         image_name = "dstack-a3mega-5"

dstack/_internal/core/backends/lambdalabs/compute.py

@@ -20,6 +20,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 
 MAX_INSTANCE_NAME_LEN = 60
@@ -46,7 +47,10 @@ class LambdaCompute(
         return offers_with_availability
 
     def create_instance(
-        self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=MAX_INSTANCE_NAME_LEN
@@ -89,7 +93,10 @@ class LambdaCompute(
         instance_info = _get_instance_info(self.api_client, provisioning_data.instance_id)
         if instance_info is not None and instance_info["status"] != "booting":
             provisioning_data.hostname = instance_info["ip"]
-            commands = get_shim_commands(authorized_keys=[project_ssh_public_key])
+            commands = get_shim_commands(
+                authorized_keys=[project_ssh_public_key],
+                arch=provisioning_data.instance_type.resources.cpu_arch,
+            )
             # shim is assumed to be run under root
             launch_command = "sudo sh -c '" + "&& ".join(commands) + "'"
             thread = Thread(
@@ -179,13 +186,18 @@ def _setup_instance(
     ssh_private_key: str,
 ):
     setup_commands = (
-        "mkdir /home/ubuntu/.dstack && "
-        "sudo apt-get update && "
-        "sudo apt-get install -y --no-install-recommends nvidia-container-toolkit && "
-        "sudo nvidia-ctk runtime configure --runtime=docker && "
-        "sudo pkill -SIGHUP dockerd"
+        "mkdir /home/ubuntu/.dstack",
+        "sudo apt-get update",
+        "sudo apt-get install -y --no-install-recommends nvidia-container-toolkit",
+        "sudo install -d -m 0755 /etc/docker",
+        # Workaround for https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
+        """echo '{"exec-opts":["native.cgroupdriver=cgroupfs"]}' | sudo tee /etc/docker/daemon.json""",
+        "sudo nvidia-ctk runtime configure --runtime=docker",
+        "sudo systemctl restart docker.service",  # `systemctl reload` (`kill -HUP`) won't work
+    )
+    _run_ssh_command(
+        hostname=hostname, ssh_private_key=ssh_private_key, command=" && ".join(setup_commands)
     )
-    _run_ssh_command(hostname=hostname, ssh_private_key=ssh_private_key, command=setup_commands)
 
 
 def _launch_runner(

dstack/_internal/core/backends/local/compute.py

@@ -15,6 +15,7 @@ from dstack._internal.core.models.instances import (
     InstanceType,
     Resources,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run
 from dstack._internal.core.models.volumes import Volume, VolumeProvisioningData
 from dstack._internal.utils.logging import get_logger
@@ -53,6 +54,7 @@ class LocalCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         return JobProvisioningData(
             backend=instance_offer.backend,

dstack/_internal/core/backends/nebius/compute.py

@@ -1,4 +1,5 @@
 import json
+import random
 import shlex
 import time
 from functools import cached_property
@@ -13,13 +14,19 @@ from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     generate_unique_instance_name,
     get_user_data,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.nebius import resources
+from dstack._internal.core.backends.nebius.fabrics import get_suitable_infiniband_fabrics
 from dstack._internal.core.backends.nebius.models import NebiusConfig, NebiusServiceAccountCreds
-from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
+from dstack._internal.core.errors import (
+    BackendError,
+    NotYetTerminated,
+    ProvisioningError,
+)
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel
 from dstack._internal.core.models.instances import (
@@ -28,6 +35,11 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import (
+    PlacementGroup,
+    PlacementGroupProvisioningData,
+    PlacementStrategy,
+)
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
@@ -72,6 +84,7 @@ SUPPORTED_PLATFORMS = [
 class NebiusCompute(
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     Compute,
 ):
     def __init__(self, config: NebiusConfig):
@@ -121,6 +134,7 @@ class NebiusCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         # NOTE: This method can block for a long time as it waits for the boot disk to be created
         # and the instance to enter the STARTING state. This has to be done in create_instance so
@@ -128,6 +142,14 @@ class NebiusCompute(
         # instance.
         instance_name = generate_unique_instance_name(instance_config)
         platform, preset = instance_offer.instance.name.split()
+        cluster_id = None
+        if placement_group:
+            assert placement_group.provisioning_data is not None
+            backend_data = NebiusPlacementGroupBackendData.load(
+                placement_group.provisioning_data.backend_data
+            )
+            if backend_data.cluster is not None:
+                cluster_id = backend_data.cluster.id
         create_disk_op = resources.create_disk(
             sdk=self._sdk,
             name=instance_name,
@@ -155,6 +177,7 @@ class NebiusCompute(
                 ),
                 platform=platform,
                 preset=preset,
+                cluster_id=cluster_id,
                 disk_id=create_disk_op.resource_id,
                 subnet_id=self._get_subnet_id(instance_offer.region),
             )
@@ -230,6 +253,63 @@ class NebiusCompute(
         with resources.ignore_errors([StatusCode.NOT_FOUND]):
             resources.delete_disk(self._sdk, backend_data_parsed.boot_disk_id)
 
+    def create_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        master_instance_offer: InstanceOffer,
+    ) -> PlacementGroupProvisioningData:
+        assert placement_group.configuration.placement_strategy == PlacementStrategy.CLUSTER
+        backend_data = NebiusPlacementGroupBackendData(cluster=None)
+        # Only create a Nebius cluster if the instance supports it.
+        # For other instances, return dummy PlacementGroupProvisioningData.
+        if fabrics := get_suitable_infiniband_fabrics(
+            master_instance_offer, allowed_fabrics=self.config.fabrics
+        ):
+            fabric = random.choice(fabrics)
+            op = resources.create_cluster(
+                self._sdk,
+                name=placement_group.name,
+                project_id=self._region_to_project_id[placement_group.configuration.region],
+                fabric=fabric,
+            )
+            backend_data.cluster = NebiusClusterBackendData(id=op.resource_id, fabric=fabric)
+        return PlacementGroupProvisioningData(
+            backend=BackendType.NEBIUS,
+            backend_data=backend_data.json(),
+        )
+
+    def delete_placement_group(self, placement_group: PlacementGroup) -> None:
+        assert placement_group.provisioning_data is not None
+        backend_data = NebiusPlacementGroupBackendData.load(
+            placement_group.provisioning_data.backend_data
+        )
+        if backend_data.cluster is not None:
+            with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                resources.delete_cluster(self._sdk, backend_data.cluster.id)
+
+    def is_suitable_placement_group(
+        self,
+        placement_group: PlacementGroup,
+        instance_offer: InstanceOffer,
+    ) -> bool:
+        if not (
+            placement_group.configuration.backend == BackendType.NEBIUS
+            and placement_group.configuration.region == instance_offer.region
+        ):
+            return False
+        assert placement_group.provisioning_data is not None
+        backend_data = NebiusPlacementGroupBackendData.load(
+            placement_group.provisioning_data.backend_data
+        )
+        return (
+            backend_data.cluster is None
+            or backend_data.cluster.fabric
+            in get_suitable_infiniband_fabrics(
+                instance_offer,
+                allowed_fabrics=None,  # enforced at cluster creation time, no need to enforce here
+            )
+        )
+
 
 class NebiusInstanceBackendData(CoreModel):
     boot_disk_id: str
@@ -240,6 +320,20 @@ class NebiusInstanceBackendData(CoreModel):
         return cls.__response__.parse_raw(raw)
 
 
+class NebiusClusterBackendData(CoreModel):
+    id: str
+    fabric: str
+
+
+class NebiusPlacementGroupBackendData(CoreModel):
+    cluster: Optional[NebiusClusterBackendData]
+
+    @classmethod
+    def load(cls, raw: Optional[str]) -> "NebiusPlacementGroupBackendData":
+        assert raw is not None
+        return cls.__response__.parse_raw(raw)
+
+
 def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None:
     start = time.monotonic()
     while True:

dstack/_internal/core/backends/nebius/configurator.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.configurator import (
 )
 from dstack._internal.core.backends.nebius import resources
 from dstack._internal.core.backends.nebius.backend import NebiusBackend
+from dstack._internal.core.backends.nebius.fabrics import get_all_infiniband_fabrics
 from dstack._internal.core.backends.nebius.models import (
     AnyNebiusBackendConfig,
     NebiusBackendConfig,
@@ -38,6 +39,16 @@ class NebiusConfigurator(Configurator):
                 fields=[["creds"]],
                 details=str(e),
             )
+        valid_fabrics = get_all_infiniband_fabrics()
+        if invalid_fabrics := set(config.fabrics or []) - valid_fabrics:
+            raise_invalid_credentials_error(
+                fields=[["fabrics"]],
+                details=(
+                    "These InfiniBand fabrics do not exist or are not known to dstack:"
+                    f" {sorted(invalid_fabrics)}. Omit `fabrics` to allow all fabrics or select"
+                    f" some of the valid options: {sorted(valid_fabrics)}"
+                ),
+            )
 
     def create_backend(
         self, project_name: str, config: NebiusBackendConfigWithCreds

dstack/_internal/core/backends/nebius/fabrics.py

@@ -0,0 +1,47 @@
+from collections.abc import Container
+from dataclasses import dataclass
+from typing import Optional
+
+from dstack._internal.core.models.instances import InstanceOffer
+
+
+@dataclass(frozen=True)
+class InfinibandFabric:
+    name: str
+    platform: str
+    region: str
+
+
+# https://docs.nebius.com/compute/clusters/gpu#fabrics
+INFINIBAND_FABRICS = [
+    InfinibandFabric("fabric-2", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-3", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-4", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-5", "gpu-h200-sxm", "eu-west1"),
+    InfinibandFabric("fabric-6", "gpu-h100-sxm", "eu-north1"),
+    InfinibandFabric("fabric-7", "gpu-h200-sxm", "eu-north1"),
+]
+
+
+def get_suitable_infiniband_fabrics(
+    offer: InstanceOffer, allowed_fabrics: Optional[Container[str]]
+) -> list[str]:
+    if len(offer.instance.resources.gpus) < 8:
+        # From the create VM page in the Nebius Console:
+        # > Only virtual machines with at least 8 NVIDIA® Hopper® H100 or H200 GPUs
+        # > can be added to the cluster
+        return []
+    platform, _ = offer.instance.name.split()
+    return [
+        f.name
+        for f in INFINIBAND_FABRICS
+        if (
+            f.platform == platform
+            and f.region == offer.region
+            and (allowed_fabrics is None or f.name in allowed_fabrics)
+        )
+    ]
+
+
+def get_all_infiniband_fabrics() -> set[str]:
+    return {f.name for f in INFINIBAND_FABRICS}

dstack/_internal/core/backends/nebius/models.py

@@ -87,6 +87,14 @@ class NebiusBackendConfig(CoreModel):
         Optional[list[str]],
         Field(description="The list of allowed Nebius regions. Omit to allow all regions"),
     ] = None
+    fabrics: Annotated[
+        Optional[list[str]],
+        Field(
+            description=(
+                "The list of allowed fabrics for InfiniBand clusters. Omit to allow all fabrics"
+            )
+        ),
+    ] = None
 
 
 class NebiusBackendConfigWithCreds(NebiusBackendConfig):

dstack/_internal/core/backends/nebius/resources.py

@@ -15,14 +15,19 @@ from nebius.api.nebius.common.v1 import Operation, ResourceMetadata
 from nebius.api.nebius.compute.v1 import (
     AttachedDiskSpec,
     CreateDiskRequest,
+    CreateGpuClusterRequest,
     CreateInstanceRequest,
     DeleteDiskRequest,
+    DeleteGpuClusterRequest,
     DeleteInstanceRequest,
     DiskServiceClient,
     DiskSpec,
     ExistingDisk,
     GetInstanceRequest,
+    GpuClusterServiceClient,
+    GpuClusterSpec,
     Instance,
+    InstanceGpuClusterSpec,
     InstanceServiceClient,
     InstanceSpec,
     IPAddress,
@@ -275,6 +280,7 @@ def create_instance(
     user_data: str,
     platform: str,
     preset: str,
+    cluster_id: Optional[str],
     disk_id: str,
     subnet_id: str,
 ) -> SDKOperation[Operation]:
@@ -287,6 +293,7 @@ def create_instance(
         spec=InstanceSpec(
             cloud_init_user_data=user_data,
             resources=ResourcesSpec(platform=platform, preset=preset),
+            gpu_cluster=InstanceGpuClusterSpec(id=cluster_id) if cluster_id is not None else None,
             boot_disk=AttachedDiskSpec(
                 attach_mode=AttachedDiskSpec.AttachMode.READ_WRITE,
                 existing_disk=ExistingDisk(id=disk_id),
@@ -319,3 +326,25 @@ def delete_instance(sdk: SDK, instance_id: str) -> SDKOperation[Operation]:
             DeleteInstanceRequest(id=instance_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
         )
     )
+
+
+def create_cluster(sdk: SDK, name: str, project_id: str, fabric: str) -> SDKOperation[Operation]:
+    with wrap_capacity_errors():
+        return LOOP.await_(
+            GpuClusterServiceClient(sdk).create(
+                CreateGpuClusterRequest(
+                    metadata=ResourceMetadata(name=name, parent_id=project_id),
+                    spec=GpuClusterSpec(infiniband_fabric=fabric),
+                ),
+                timeout=REQUEST_TIMEOUT,
+                metadata=REQUEST_MD,
+            )
+        )
+
+
+def delete_cluster(sdk: SDK, cluster_id: str) -> None:
+    return LOOP.await_(
+        GpuClusterServiceClient(sdk).delete(
+            DeleteGpuClusterRequest(id=cluster_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )

dstack/_internal/core/backends/oci/compute.py

@@ -23,6 +23,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 
@@ -105,6 +106,7 @@ class OCICompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         region = self.regions[instance_offer.region]
 

dstack/_internal/core/backends/remote/provisioning.py

@@ -6,8 +6,9 @@ from textwrap import dedent
 from typing import Any, Dict, Generator, List, Optional
 
 import paramiko
-from gpuhunt import AcceleratorVendor, correct_gpu_memory_gib
+from gpuhunt import AcceleratorVendor, CPUArchitecture, correct_gpu_memory_gib
 
+from dstack._internal.core.backends.base.compute import GoArchType, normalize_arch
 from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT
 
 # FIXME: ProvisioningError is a subclass of ComputeError and should not be used outside of Compute
@@ -36,6 +37,22 @@ DSTACK_SHIM_ENV_FILE = "shim.env"
 HOST_INFO_FILE = "host_info.json"
 
 
+def detect_cpu_arch(client: paramiko.SSHClient) -> GoArchType:
+    cmd = "uname -m"
+    try:
+        _, stdout, stderr = client.exec_command(cmd, timeout=20)
+    except (paramiko.SSHException, OSError) as e:
+        raise ProvisioningError(f"detect_cpu_arch: {e}") from e
+    out = stdout.read().strip().decode()
+    err = stderr.read().strip().decode()
+    if err:
+        raise ProvisioningError(f"detect_cpu_arch: {cmd} failed, stdout: {out}, stderr: {err}")
+    try:
+        return normalize_arch(out)
+    except ValueError as e:
+        raise ProvisioningError(f"detect_cpu_arch: failed to normalize arch: {e}") from e
+
+
 def sftp_upload(client: paramiko.SSHClient, path: str, body: str) -> None:
     try:
         sftp = client.open_sftp()
@@ -226,7 +243,14 @@ def get_shim_healthcheck(client: paramiko.SSHClient) -> str:
             raise ProvisioningError(f"get_shim_healthcheck failed: {e}") from e
 
 
-def host_info_to_instance_type(host_info: Dict[str, Any]) -> InstanceType:
+def host_info_to_instance_type(host_info: Dict[str, Any], cpu_arch: GoArchType) -> InstanceType:
+    _cpu_arch: CPUArchitecture
+    if cpu_arch == "amd64":
+        _cpu_arch = CPUArchitecture.X86
+    elif cpu_arch == "arm64":
+        _cpu_arch = CPUArchitecture.ARM
+    else:
+        raise ValueError(f"Unexpected cpu_arch: {cpu_arch}")
     gpu_count = host_info.get("gpu_count", 0)
     if gpu_count > 0:
         gpu_vendor = AcceleratorVendor.cast(host_info.get("gpu_vendor", "nvidia"))
@@ -251,6 +275,7 @@ def host_info_to_instance_type(host_info: Dict[str, Any]) -> InstanceType:
     instance_type = InstanceType(
         name="instance",
         resources=Resources(
+            cpu_arch=_cpu_arch,
             cpus=host_info["cpus"],
             memory_mib=host_info["memory"] / 1024 / 1024,
             spot=False,

dstack/_internal/core/backends/template/compute.py.jinja

@@ -18,6 +18,7 @@ from dstack._internal.core.models.instances import (
     InstanceConfiguration,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run
 from dstack._internal.core.models.volumes import Volume
 from dstack._internal.utils.logging import get_logger
@@ -64,6 +65,7 @@ class {{ backend_name }}Compute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         # TODO: Implement if backend supports creating instances (VM-based).
         # Delete if backend can only run jobs (container-based).

dstack/_internal/core/backends/tensordock/compute.py

@@ -19,6 +19,7 @@ from dstack._internal.core.models.instances import (
     InstanceConfiguration,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
 
@@ -57,6 +58,7 @@ class TensorDockCompute(
         self,
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=MAX_INSTANCE_NAME_LEN

dstack/_internal/core/backends/vultr/compute.py

@@ -22,6 +22,7 @@ from dstack._internal.core.models.instances import (
     InstanceOffer,
     InstanceOfferWithAvailability,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
 
@@ -58,7 +59,10 @@ class VultrCompute(
         return offers
 
     def create_instance(
-        self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
     ) -> JobProvisioningData:
         instance_name = generate_unique_instance_name(
             instance_config, max_length=MAX_INSTANCE_NAME_LEN

dstack/_internal/core/models/instances.py

@@ -49,11 +49,13 @@ class Resources(CoreModel):
     spot: bool
     disk: Disk = Disk(size_mib=102400)  # the default value (100GB) for backward compatibility
     description: str = ""
+    cpu_arch: Optional[gpuhunt.CPUArchitecture] = None
 
     def pretty_format(self, include_spot: bool = False) -> str:
         resources = {}
         if self.cpus > 0:
             resources["cpus"] = self.cpus
+            resources["cpu_arch"] = self.cpu_arch
         if self.memory_mib > 0:
             resources["memory"] = f"{self.memory_mib / 1024:.0f}GB"
         if self.disk.size_mib > 0:
@@ -105,7 +107,6 @@ class InstanceConfiguration(CoreModel):
     user: str  # dstack user name
     ssh_keys: List[SSHKey]
     instance_id: Optional[str] = None
-    placement_group_name: Optional[str] = None
     reservation: Optional[str] = None
     volumes: Optional[List[Volume]] = None
     tags: Optional[Dict[str, str]] = None