dstack 0.19.2__tar.gz → 0.19.3__tar.gz

{dstack-0.19.2 → dstack-0.19.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.19.2
+Version: 0.19.3
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov

{dstack-0.19.2 → dstack-0.19.3}/setup.py

@@ -49,7 +49,7 @@ BASE_DEPS = [
     "python-multipart>=0.0.16",
     "filelock",
     "psutil",
-    "gpuhunt>=0.1.1,<0.2.0",
+    "gpuhunt>=0.1.2,<0.2.0",
     "argcomplete>=3.5.0",
 ]
 

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/cli/services/profile.py

@@ -65,6 +65,13 @@ def register_profile_args(parser: argparse.ArgumentParser):
     )
 
     fleets_group = parser.add_argument_group("Fleets")
+    fleets_group.add_argument(
+        "--fleet",
+        action="append",
+        metavar="NAME",
+        dest="fleets",
+        help="Consider only instances from the specified fleet(s) for reuse",
+    )
     fleets_group_exc = fleets_group.add_mutually_exclusive_group()
     fleets_group_exc.add_argument(
         "-R",
@@ -147,6 +154,8 @@ def apply_profile_args(
     if args.max_duration is not None:
         profile_settings.max_duration = args.max_duration
 
+    if args.fleets:
+        profile_settings.fleets = args.fleets
     if args.idle_duration is not None:
         profile_settings.idle_duration = args.idle_duration
     elif args.dont_destroy:

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/aws/configurator.py

@@ -34,6 +34,7 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+# where dstack OS images are published
 REGIONS = [
     ("US East, N. Virginia", "us-east-1"),
     ("US East, Ohio", "us-east-2"),

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/base/compute.py

@@ -528,7 +528,7 @@ def get_run_shim_script(is_privileged: bool, pjrt_device: Optional[str]) -> List
     pjrt_device_env = f"--pjrt-device={pjrt_device}" if pjrt_device else ""
 
     return [
-        f"nohup dstack-shim {privileged_flag} {pjrt_device_env} &",
+        f"nohup {DSTACK_SHIM_BINARY_PATH} {privileged_flag} {pjrt_device_env} &",
     ]
 
 

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/cudo/compute.py

@@ -41,6 +41,7 @@ class CudoCompute(
     ) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.CUDO,
+            locations=self.config.regions,
             requirements=requirements,
         )
         offers = [
@@ -48,6 +49,7 @@ class CudoCompute(
                 **offer.dict(), availability=InstanceAvailability.AVAILABLE
             )
             for offer in offers
+            # in-hyderabad-1 is known to have provisioning issues
             if offer.region not in ["in-hyderabad-1"]
         ]
         return offers

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/cudo/configurator.py

@@ -17,17 +17,6 @@ from dstack._internal.core.backends.cudo.models import (
 )
 from dstack._internal.core.models.backends.base import BackendType
 
-REGIONS = [
-    "no-luster-1",
-    "se-smedjebacken-1",
-    "gb-london-1",
-    "se-stockholm-1",
-    "us-newyork-1",
-    "us-santaclara-1",
-]
-
-DEFAULT_REGION = "no-luster-1"
-
 
 class CudoConfigurator(Configurator):
     TYPE = BackendType.CUDO
@@ -39,8 +28,6 @@ class CudoConfigurator(Configurator):
     def create_backend(
         self, project_name: str, config: CudoBackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config=CudoStoredConfig(
                 **CudoBackendConfig.__response__.parse_obj(config).dict()

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/datacrunch/compute.py

@@ -1,5 +1,9 @@
 from typing import Dict, List, Optional
 
+from datacrunch import DataCrunchClient
+from datacrunch.exceptions import APIException
+from datacrunch.instances.instances import Instance
+
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
@@ -7,8 +11,8 @@ from dstack._internal.core.backends.base.compute import (
     get_shim_commands,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
-from dstack._internal.core.backends.datacrunch.api_client import DataCrunchAPIClient
 from dstack._internal.core.backends.datacrunch.models import DataCrunchConfig
+from dstack._internal.core.errors import NoCapacityError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
@@ -19,14 +23,12 @@ from dstack._internal.core.models.instances import (
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
+from dstack._internal.utils.ssh import get_public_key_fingerprint
 
 logger = get_logger("datacrunch.compute")
 
 MAX_INSTANCE_NAME_LEN = 60
 
-# Ubuntu 22.04 + CUDA 12.0 + Docker
-# from API https://datacrunch.stoplight.io/docs/datacrunch-public/c46ab45dbc508-get-all-image-types
-IMAGE_ID = "2088da25-bb0d-41cc-a191-dccae45d96fd"
 IMAGE_SIZE = Memory.parse("50GB")
 
 CONFIGURABLE_DISK_SIZE = Range[Memory](min=IMAGE_SIZE, max=None)
@@ -39,7 +41,10 @@ class DataCrunchCompute(
     def __init__(self, config: DataCrunchConfig):
         super().__init__()
         self.config = config
-        self.api_client = DataCrunchAPIClient(config.creds.client_id, config.creds.client_secret)
+        self.client = DataCrunchClient(
+            client_id=self.config.creds.client_id,
+            client_secret=self.config.creds.client_secret,
+        )
 
     def get_offers(
         self, requirements: Optional[Requirements] = None
@@ -56,14 +61,12 @@ class DataCrunchCompute(
     def _get_offers_with_availability(
         self, offers: List[InstanceOffer]
     ) -> List[InstanceOfferWithAvailability]:
-        raw_availabilities: List[Dict] = self.api_client.client.instances.get_availabilities()
+        raw_availabilities: List[Dict] = self.client.instances.get_availabilities()
 
         region_availabilities = {}
         for location in raw_availabilities:
             location_code = location["location_code"]
             availabilities = location["availabilities"]
-            if location_code not in self.config.regions:
-                continue
             for name in availabilities:
                 key = (name, location_code)
                 region_availabilities[key] = InstanceAvailability.AVAILABLE
@@ -91,50 +94,50 @@ class DataCrunchCompute(
         for ssh_public_key in public_keys:
             ssh_ids.append(
                 # datacrunch allows you to use the same name
-                self.api_client.get_or_create_ssh_key(
+                _get_or_create_ssh_key(
+                    client=self.client,
                     name=f"dstack-{instance_config.instance_name}.key",
                     public_key=ssh_public_key,
                 )
             )
 
         commands = get_shim_commands(authorized_keys=public_keys)
-
         startup_script = " ".join([" && ".join(commands)])
         script_name = f"dstack-{instance_config.instance_name}.sh"
-
-        logger.debug("startup script:", startup_script)
-
-        startup_script_ids = self.api_client.get_or_create_startup_scrpit(
-            name=script_name, script=startup_script
+        startup_script_ids = _get_or_create_startup_scrpit(
+            client=self.client,
+            name=script_name,
+            script=startup_script,
         )
 
         disk_size = round(instance_offer.instance.resources.disk.size_mib / 1024)
-
-        instance = self.api_client.deploy_instance(
-            instance_type=instance_offer.instance.name,
-            ssh_key_ids=ssh_ids,
-            startup_script_id=startup_script_ids,
-            hostname=instance_name,
-            description=instance_name,
-            image=IMAGE_ID,
-            disk_size=disk_size,
-            location=instance_offer.region,
-        )
+        image_id = _get_vm_image_id(instance_offer)
 
         logger.debug(
-            "deploy_instance",
+            "Deploying datacrunch instance",
             {
                 "instance_type": instance_offer.instance.name,
                 "ssh_key_ids": ssh_ids,
                 "startup_script_id": startup_script_ids,
                 "hostname": instance_name,
                 "description": instance_name,
-                "image": IMAGE_ID,
+                "image": image_id,
                 "disk_size": disk_size,
                 "location": instance_offer.region,
             },
         )
-
+        instance = _deploy_instance(
+            client=self.client,
+            instance_type=instance_offer.instance.name,
+            ssh_key_ids=ssh_ids,
+            startup_script_id=startup_script_ids,
+            hostname=instance_name,
+            description=instance_name,
+            image=image_id,
+            disk_size=disk_size,
+            is_spot=instance_offer.instance.resources.spot,
+            location=instance_offer.region,
+        )
         return JobProvisioningData(
             backend=instance_offer.backend,
             instance_type=instance_offer.instance,
@@ -152,8 +155,14 @@ class DataCrunchCompute(
 
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
-    ) -> None:
-        self.api_client.delete_instance(instance_id)
+    ):
+        try:
+            self.client.instances.action(id_list=[instance_id], action="delete")
+        except APIException as e:
+            if e.message == "Invalid instance id":
+                logger.debug("Skipping instance %s termination. Instance not found.", instance_id)
+                return
+            raise
 
     def update_provisioning_data(
         self,
@@ -161,6 +170,83 @@ class DataCrunchCompute(
         project_ssh_public_key: str,
         project_ssh_private_key: str,
     ):
-        instance = self.api_client.get_instance_by_id(provisioning_data.instance_id)
+        instance = _get_instance_by_id(self.client, provisioning_data.instance_id)
         if instance is not None and instance.status == "running":
             provisioning_data.hostname = instance.ip
+
+
+def _get_vm_image_id(instance_offer: InstanceOfferWithAvailability) -> str:
+    # https://api.datacrunch.io/v1/images
+    if (
+        len(instance_offer.instance.resources.gpus) > 0
+        and instance_offer.instance.resources.gpus[0].name == "V100"
+    ):
+        # Ubuntu 22.04 + CUDA 12.0 + Docker
+        return "2088da25-bb0d-41cc-a191-dccae45d96fd"
+    # Ubuntu 24.04 + CUDA 12.8 Open + Docker
+    return "77777777-4f48-4249-82b3-f199fb9b701b"
+
+
+def _get_or_create_ssh_key(client: DataCrunchClient, name: str, public_key: str) -> str:
+    fingerprint = get_public_key_fingerprint(public_key)
+    keys = client.ssh_keys.get()
+    found_keys = [key for key in keys if fingerprint == get_public_key_fingerprint(key.public_key)]
+    if found_keys:
+        key = found_keys[0]
+        return key.id
+    key = client.ssh_keys.create(name, public_key)
+    return key.id
+
+
+def _get_or_create_startup_scrpit(client: DataCrunchClient, name: str, script: str) -> str:
+    scripts = client.startup_scripts.get()
+    found_scripts = [startup_script for startup_script in scripts if script == startup_script]
+    if found_scripts:
+        startup_script = found_scripts[0]
+        return startup_script.id
+
+    startup_script = client.startup_scripts.create(name, script)
+    return startup_script.id
+
+
+def _get_instance_by_id(
+    client: DataCrunchClient,
+    instance_id: str,
+) -> Optional[Instance]:
+    try:
+        return client.instances.get_by_id(instance_id)
+    except APIException as e:
+        if e.message == "Invalid instance id":
+            return None
+        raise
+
+
+def _deploy_instance(
+    client: DataCrunchClient,
+    instance_type: str,
+    image: str,
+    ssh_key_ids: List[str],
+    hostname: str,
+    description: str,
+    startup_script_id: str,
+    disk_size: int,
+    is_spot: bool,
+    location: str,
+) -> Instance:
+    try:
+        instance = client.instances.create(
+            instance_type=instance_type,
+            image=image,
+            ssh_key_ids=ssh_key_ids,
+            hostname=hostname,
+            description=description,
+            startup_script_id=startup_script_id,
+            is_spot=is_spot,
+            location=location,
+            os_volume={"name": "OS volume", "size": disk_size},
+        )
+    except APIException as e:
+        # FIXME: Catch only no capacity errors
+        raise NoCapacityError(f"DataCrunch API error: {e.message}")
+
+    return instance

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/datacrunch/configurator.py

@@ -1,8 +1,12 @@
 import json
 
+from datacrunch import DataCrunchClient
+from datacrunch.exceptions import APIException
+
 from dstack._internal.core.backends.base.configurator import (
     BackendRecord,
     Configurator,
+    raise_invalid_credentials_error,
 )
 from dstack._internal.core.backends.datacrunch.backend import DataCrunchBackend
 from dstack._internal.core.backends.datacrunch.models import (
@@ -17,13 +21,6 @@ from dstack._internal.core.models.backends.base import (
     BackendType,
 )
 
-REGIONS = [
-    "FIN-01",
-    "ICE-01",
-]
-
-DEFAULT_REGION = "FIN-01"
-
 
 class DataCrunchConfigurator(Configurator):
     TYPE = BackendType.DATACRUNCH
@@ -32,14 +29,11 @@ class DataCrunchConfigurator(Configurator):
     def validate_config(
         self, config: DataCrunchBackendConfigWithCreds, default_creds_enabled: bool
     ):
-        # FIXME: validate datacrunch creds
-        return
+        self._validate_creds(config.creds)
 
     def create_backend(
         self, project_name: str, config: DataCrunchBackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config=DataCrunchStoredConfig(
                 **DataCrunchBackendConfig.__response__.parse_obj(config).dict()
@@ -64,3 +58,14 @@ class DataCrunchConfigurator(Configurator):
             **json.loads(record.config),
             creds=DataCrunchCreds.parse_raw(record.auth),
         )
+
+    def _validate_creds(self, creds: DataCrunchCreds):
+        try:
+            DataCrunchClient(
+                client_id=creds.client_id,
+                client_secret=creds.client_secret,
+            )
+        except APIException as e:
+            if e.code == "unauthorized_request":
+                raise_invalid_credentials_error(fields=[["creds", "api_key"]])
+            raise

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/gcp/compute.py

@@ -1,7 +1,7 @@
 import concurrent.futures
 import json
 from collections import defaultdict
-from typing import Callable, Dict, List, Literal, Optional
+from typing import Callable, Dict, List, Literal, Optional, Tuple
 
 import google.api_core.exceptions
 import google.cloud.compute_v1 as compute_v1
@@ -10,11 +10,13 @@ from gpuhunt import KNOWN_TPUS
 
 import dstack._internal.core.backends.gcp.auth as auth
 import dstack._internal.core.backends.gcp.resources as gcp_resources
+from dstack import version
 from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     ComputeWithVolumeSupport,
     generate_unique_gateway_instance_name,
     generate_unique_instance_name,
@@ -25,11 +27,13 @@ from dstack._internal.core.backends.base.compute import (
     merge_tags,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.gcp.features import tcpx as tcpx_features
 from dstack._internal.core.backends.gcp.models import GCPConfig
 from dstack._internal.core.errors import (
     ComputeError,
     ComputeResourceNotFoundError,
     NoCapacityError,
+    PlacementGroupInUseError,
     ProvisioningError,
 )
 from dstack._internal.core.models.backends.base import BackendType
@@ -46,6 +50,7 @@ from dstack._internal.core.models.instances import (
     InstanceType,
     Resources,
 )
+from dstack._internal.core.models.placement import PlacementGroup, PlacementGroupProvisioningData
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.core.models.volumes import (
@@ -74,6 +79,7 @@ class GCPVolumeDiskBackendData(CoreModel):
 class GCPCompute(
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
     ComputeWithGatewaySupport,
     ComputeWithVolumeSupport,
     Compute,
@@ -89,6 +95,9 @@ class GCPCompute(
         self.routers_client = compute_v1.RoutersClient(credentials=self.credentials)
         self.tpu_client = tpu_v2.TpuClient(credentials=self.credentials)
         self.disk_client = compute_v1.DisksClient(credentials=self.credentials)
+        self.resource_policies_client = compute_v1.ResourcePoliciesClient(
+            credentials=self.credentials
+        )
 
     def get_offers(
         self, requirements: Optional[Requirements] = None
@@ -183,6 +192,19 @@ class GCPCompute(
             config=self.config,
             region=instance_offer.region,
         )
+        extra_subnets = _get_extra_subnets(
+            subnetworks_client=self.subnetworks_client,
+            config=self.config,
+            region=instance_offer.region,
+            instance_type_name=instance_offer.instance.name,
+        )
+        placement_policy = None
+        if instance_config.placement_group_name is not None:
+            placement_policy = gcp_resources.get_placement_policy_resource_name(
+                project_id=self.config.project_id,
+                region=instance_offer.region,
+                placement_policy=instance_config.placement_group_name,
+            )
         labels = {
             "owner": "dstack",
             "dstack_project": instance_config.project_name.lower(),
@@ -259,8 +281,9 @@ class GCPCompute(
             request.project = self.config.project_id
             request.instance_resource = gcp_resources.create_instance_struct(
                 disk_size=disk_size,
-                image_id=gcp_resources.get_image_id(
-                    len(instance_offer.instance.resources.gpus) > 0,
+                image_id=_get_image_id(
+                    instance_type_name=instance_offer.instance.name,
+                    cuda=len(instance_offer.instance.resources.gpus) > 0,
                 ),
                 machine_type=instance_offer.instance.name,
                 accelerators=gcp_resources.get_accelerators(
@@ -269,7 +292,12 @@ class GCPCompute(
                     gpus=instance_offer.instance.resources.gpus,
                 ),
                 spot=instance_offer.instance.resources.spot,
-                user_data=get_user_data(authorized_keys),
+                user_data=get_user_data(
+                    authorized_keys,
+                    backend_specific_commands=_get_backend_specific_commands(
+                        instance_offer.instance.name
+                    ),
+                ),
                 authorized_keys=authorized_keys,
                 labels=labels,
                 tags=[gcp_resources.DSTACK_INSTANCE_TAG],
@@ -278,7 +306,9 @@ class GCPCompute(
                 service_account=self.config.vm_service_account,
                 network=self.config.vpc_resource_name,
                 subnetwork=subnetwork,
+                extra_subnetworks=extra_subnets,
                 allocate_public_ip=allocate_public_ip,
+                placement_policy=placement_policy,
             )
             try:
                 # GCP needs some time to return an error in case of no capacity (< 30s).
@@ -371,6 +401,43 @@ class GCPCompute(
             f"Failed to get instance IP address. Instance status: {instance.status}"
         )
 
+    def create_placement_group(
+        self,
+        placement_group: PlacementGroup,
+    ) -> PlacementGroupProvisioningData:
+        policy = compute_v1.ResourcePolicy(
+            name=placement_group.name,
+            region=placement_group.configuration.region,
+            group_placement_policy=compute_v1.ResourcePolicyGroupPlacementPolicy(
+                availability_domain_count=1,
+                collocation="COLLOCATED",
+            ),
+        )
+        self.resource_policies_client.insert(
+            project=self.config.project_id,
+            region=placement_group.configuration.region,
+            resource_policy_resource=policy,
+        )
+        return PlacementGroupProvisioningData(backend=BackendType.GCP)
+
+    def delete_placement_group(
+        self,
+        placement_group: PlacementGroup,
+    ):
+        try:
+            operation = self.resource_policies_client.delete(
+                project=self.config.project_id,
+                region=placement_group.configuration.region,
+                resource_policy=placement_group.name,
+            )
+            operation.result()  # Wait for operation to complete
+        except google.api_core.exceptions.NotFound:
+            logger.debug("Placement group %s not found", placement_group.name)
+        except google.api_core.exceptions.BadRequest as e:
+            if "is already being used by" in e.message:
+                raise PlacementGroupInUseError()
+            raise
+
     def create_gateway(
         self,
         configuration: GatewayComputeConfiguration,
@@ -412,7 +479,7 @@ class GCPCompute(
         request.project = self.config.project_id
         request.instance_resource = gcp_resources.create_instance_struct(
             disk_size=10,
-            image_id=gcp_resources.get_gateway_image_id(),
+            image_id=_get_gateway_image_id(),
             machine_type="e2-small",
             accelerators=[],
             spot=False,
@@ -681,21 +748,6 @@ class GCPCompute(
         )
 
 
-def _get_vpc_subnet(
-    subnetworks_client: compute_v1.SubnetworksClient,
-    config: GCPConfig,
-    region: str,
-) -> Optional[str]:
-    if config.vpc_name is None:
-        return None
-    return gcp_resources.get_vpc_subnet_or_error(
-        subnetworks_client=subnetworks_client,
-        vpc_project_id=config.vpc_project_id or config.project_id,
-        vpc_name=config.vpc_name,
-        region=region,
-    )
-
-
 def _supported_instances_and_zones(
     regions: List[str],
 ) -> Optional[Callable[[InstanceOffer], bool]]:
@@ -754,6 +806,74 @@ def _unique_instance_name(instance: InstanceType) -> str:
     return f"{name}-{gpu.name}-{gpu.memory_mib}"
 
 
+def _get_vpc_subnet(
+    subnetworks_client: compute_v1.SubnetworksClient,
+    config: GCPConfig,
+    region: str,
+) -> Optional[str]:
+    if config.vpc_name is None:
+        return None
+    return gcp_resources.get_vpc_subnet_or_error(
+        subnetworks_client=subnetworks_client,
+        vpc_project_id=config.vpc_project_id or config.project_id,
+        vpc_name=config.vpc_name,
+        region=region,
+    )
+
+
+def _get_extra_subnets(
+    subnetworks_client: compute_v1.SubnetworksClient,
+    config: GCPConfig,
+    region: str,
+    instance_type_name: str,
+) -> List[Tuple[str, str]]:
+    if config.extra_vpcs is None:
+        return []
+    if instance_type_name != "a3-megagpu-8g":
+        return []
+    extra_subnets = []
+    for vpc_name in config.extra_vpcs:
+        subnet = gcp_resources.get_vpc_subnet_or_error(
+            subnetworks_client=subnetworks_client,
+            vpc_project_id=config.vpc_project_id or config.project_id,
+            vpc_name=vpc_name,
+            region=region,
+        )
+        vpc_resource_name = gcp_resources.vpc_name_to_vpc_resource_name(
+            project_id=config.vpc_project_id or config.project_id,
+            vpc_name=vpc_name,
+        )
+        extra_subnets.append((vpc_resource_name, subnet))
+    return extra_subnets[:8]
+
+
+def _get_image_id(instance_type_name: str, cuda: bool) -> str:
+    if instance_type_name == "a3-megagpu-8g":
+        image_name = "dstack-a3mega-5"
+    elif cuda:
+        image_name = f"dstack-cuda-{version.base_image}"
+    else:
+        image_name = f"dstack-{version.base_image}"
+    image_name = image_name.replace(".", "-")
+    return f"projects/dstack/global/images/{image_name}"
+
+
+def _get_gateway_image_id() -> str:
+    return "projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20230714"
+
+
+def _get_backend_specific_commands(instance_type_name: str) -> List[str]:
+    if instance_type_name == "a3-megagpu-8g":
+        return tcpx_features.get_backend_specific_commands_tcpxo()
+    return []
+
+
+def _get_volume_price(size: int) -> float:
+    # https://cloud.google.com/compute/disks-image-pricing#persistentdisk
+    # The price is different in different regions. Take max across supported regions.
+    return size * 0.12
+
+
 def _get_tpu_startup_script(authorized_keys: List[str]) -> str:
     commands = get_shim_commands(
         authorized_keys=authorized_keys, is_privileged=True, pjrt_device="TPU"
@@ -805,12 +925,6 @@ def _is_single_host_tpu(instance_name: str) -> bool:
         return False
 
 
-def _get_volume_price(size: int) -> float:
-    # https://cloud.google.com/compute/disks-image-pricing#persistentdisk
-    # The price is different in different regions. Take max across supported regions.
-    return size * 0.12
-
-
 def _get_tpu_data_disks(
     project_id: str, volumes: Optional[List[Volume]]
 ) -> List[tpu_v2.AttachedDisk]:

{dstack-0.19.2 → dstack-0.19.3}/src/dstack/_internal/core/backends/gcp/configurator.py

@@ -199,3 +199,5 @@ class GCPConfigurator(Configurator):
             )
         except BackendError as e:
             raise ServerClientError(e.args[0])
+        # Not checking config.extra_vpc so that users are not required to configure subnets for all regions
+        # but only for regions they intend to use. Validation will be done on provisioning.