dstack 0.19.4rc3__py3-none-any.whl → 0.19.6rc1__py3-none-any.whl

dstack/_internal/core/backends/azure/compute.py

@@ -136,13 +136,18 @@ class AzureCompute(
             location=location,
         )
 
-        tags = {
+        base_tags = {
             "owner": "dstack",
             "dstack_project": instance_config.project_name,
             "dstack_name": instance_config.instance_name,
             "dstack_user": instance_config.user,
         }
-        tags = merge_tags(tags=tags, backend_tags=self.config.tags)
+        tags = merge_tags(
+            base_tags=base_tags,
+            backend_tags=self.config.tags,
+            resource_tags=instance_config.tags,
+        )
+        tags = azure_resources.filter_invalid_tags(tags)
 
         # TODO: Support custom availability_zones.
         # Currently, VMs are regional, which means they don't have zone info.
@@ -228,14 +233,19 @@ class AzureCompute(
             location=configuration.region,
         )
 
-        tags = {
+        base_tags = {
             "owner": "dstack",
             "dstack_project": configuration.project_name,
             "dstack_name": configuration.instance_name,
         }
         if settings.DSTACK_VERSION is not None:
-            tags["dstack_version"] = settings.DSTACK_VERSION
-        tags = merge_tags(tags=tags, backend_tags=self.config.tags)
+            base_tags["dstack_version"] = settings.DSTACK_VERSION
+        tags = merge_tags(
+            base_tags=base_tags,
+            backend_tags=self.config.tags,
+            resource_tags=configuration.tags,
+        )
+        tags = azure_resources.filter_invalid_tags(tags)
 
         vm = _launch_instance(
             compute_client=self._compute_client,

dstack/_internal/core/backends/azure/configurator.py

@@ -46,7 +46,6 @@ from dstack._internal.core.errors import (
 from dstack._internal.core.models.backends.base import (
     BackendType,
 )
-from dstack._internal.core.models.common import is_core_model_instance
 
 LOCATIONS = [
     ("(US) Central US", "centralus"),
@@ -76,14 +75,14 @@ class AzureConfigurator(Configurator):
     BACKEND_CLASS = AzureBackend
 
     def validate_config(self, config: AzureBackendConfigWithCreds, default_creds_enabled: bool):
-        if is_core_model_instance(config.creds, AzureDefaultCreds) and not default_creds_enabled:
+        if isinstance(config.creds, AzureDefaultCreds) and not default_creds_enabled:
             raise_invalid_credentials_error(fields=[["creds"]])
-        if is_core_model_instance(config.creds, AzureClientCreds):
+        if isinstance(config.creds, AzureClientCreds):
             self._set_client_creds_tenant_id(config.creds, config.tenant_id)
         try:
             credential, _ = auth.authenticate(config.creds)
         except BackendAuthError:
-            if is_core_model_instance(config.creds, AzureClientCreds):
+            if isinstance(config.creds, AzureClientCreds):
                 raise_invalid_credentials_error(
                     fields=[
                         ["creds", "tenant_id"],
@@ -105,7 +104,7 @@ class AzureConfigurator(Configurator):
     ) -> BackendRecord:
         if config.regions is None:
             config.regions = DEFAULT_LOCATIONS
-        if is_core_model_instance(config.creds, AzureClientCreds):
+        if isinstance(config.creds, AzureClientCreds):
             self._set_client_creds_tenant_id(config.creds, config.tenant_id)
         credential, _ = auth.authenticate(config.creds)
         if config.resource_group is None:

dstack/_internal/core/backends/azure/resources.py

@@ -5,6 +5,10 @@ from azure.mgmt import network as network_mgmt
 from azure.mgmt.network.models import Subnet
 
 from dstack._internal.core.errors import BackendError
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
 
 MAX_RESOURCE_NAME_LEN = 64
 
@@ -77,6 +81,16 @@ def _is_eligible_private_subnet(
     return False
 
 
+def filter_invalid_tags(tags: Dict[str, str]) -> Dict[str, str]:
+    filtered_tags = {}
+    for k, v in tags.items():
+        if not _is_valid_tag(k, v):
+            logger.warning("Skipping invalid tag '%s: %s'", k, v)
+            continue
+        filtered_tags[k] = v
+    return filtered_tags
+
+
 def validate_tags(tags: Dict[str, str]):
     for k, v in tags.items():
         if not _is_valid_tag(k, v):

dstack/_internal/core/backends/base/compute.py

@@ -5,6 +5,7 @@ import string
 import threading
 from abc import ABC, abstractmethod
 from functools import lru_cache
+from pathlib import Path
 from typing import Dict, List, Optional
 
 import git
@@ -36,14 +37,12 @@ from dstack._internal.core.models.volumes import (
 )
 from dstack._internal.core.services import is_valid_dstack_resource_name
 from dstack._internal.utils.logging import get_logger
+from dstack._internal.utils.path import PathLike
 
 logger = get_logger(__name__)
 
-DSTACK_WORKING_DIR = "/root/.dstack"
 DSTACK_SHIM_BINARY_NAME = "dstack-shim"
-DSTACK_SHIM_BINARY_PATH = f"/usr/local/bin/{DSTACK_SHIM_BINARY_NAME}"
 DSTACK_RUNNER_BINARY_NAME = "dstack-runner"
-DSTACK_RUNNER_BINARY_PATH = f"/usr/local/bin/{DSTACK_RUNNER_BINARY_NAME}"
 
 
 class Compute(ABC):
@@ -173,6 +172,7 @@ class ComputeWithCreateInstanceSupport(ABC):
             ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
             volumes=volumes,
             reservation=run.run_spec.configuration.reservation,
+            tags=run.run_spec.merged_profile.tags,
         )
         instance_offer = instance_offer.copy()
         self._restrict_instance_offer_az_to_volumes_az(instance_offer, volumes)
@@ -335,6 +335,24 @@ class ComputeWithVolumeSupport(ABC):
         return True
 
 
+def get_dstack_working_dir(base_path: Optional[PathLike] = None) -> str:
+    if base_path is None:
+        base_path = "/root"
+    return str(Path(base_path, ".dstack"))
+
+
+def get_dstack_shim_binary_path(bin_path: Optional[PathLike] = None) -> str:
+    if bin_path is None:
+        bin_path = "/usr/local/bin"
+    return str(Path(bin_path, DSTACK_SHIM_BINARY_NAME))
+
+
+def get_dstack_runner_binary_path(bin_path: Optional[PathLike] = None) -> str:
+    if bin_path is None:
+        bin_path = "/usr/local/bin"
+    return str(Path(bin_path, DSTACK_RUNNER_BINARY_NAME))
+
+
 def get_job_instance_name(run: Run, job: Job) -> str:
     return job.job_spec.job_name
 
@@ -441,9 +459,18 @@ def get_cloud_config(**config) -> str:
 
 
 def get_user_data(
-    authorized_keys: List[str], backend_specific_commands: Optional[List[str]] = None
+    authorized_keys: List[str],
+    backend_specific_commands: Optional[List[str]] = None,
+    base_path: Optional[PathLike] = None,
+    bin_path: Optional[PathLike] = None,
+    backend_shim_env: Optional[Dict[str, str]] = None,
 ) -> str:
-    shim_commands = get_shim_commands(authorized_keys)
+    shim_commands = get_shim_commands(
+        authorized_keys=authorized_keys,
+        base_path=base_path,
+        bin_path=bin_path,
+        backend_shim_env=backend_shim_env,
+    )
     commands = (backend_specific_commands or []) + shim_commands
     return get_cloud_config(
         runcmd=[["sh", "-c", " && ".join(commands)]],
@@ -451,29 +478,55 @@ def get_user_data(
     )
 
 
-def get_shim_env(authorized_keys: List[str]) -> Dict[str, str]:
+def get_shim_env(
+    authorized_keys: List[str],
+    base_path: Optional[PathLike] = None,
+    bin_path: Optional[PathLike] = None,
+    backend_shim_env: Optional[Dict[str, str]] = None,
+) -> Dict[str, str]:
     log_level = "6"  # Trace
     envs = {
-        "DSTACK_SHIM_HOME": DSTACK_WORKING_DIR,
+        "DSTACK_SHIM_HOME": get_dstack_working_dir(base_path),
         "DSTACK_SHIM_HTTP_PORT": str(DSTACK_SHIM_HTTP_PORT),
         "DSTACK_SHIM_LOG_LEVEL": log_level,
         "DSTACK_RUNNER_DOWNLOAD_URL": get_dstack_runner_download_url(),
-        "DSTACK_RUNNER_BINARY_PATH": DSTACK_RUNNER_BINARY_PATH,
+        "DSTACK_RUNNER_BINARY_PATH": get_dstack_runner_binary_path(bin_path),
         "DSTACK_RUNNER_HTTP_PORT": str(DSTACK_RUNNER_HTTP_PORT),
         "DSTACK_RUNNER_SSH_PORT": str(DSTACK_RUNNER_SSH_PORT),
         "DSTACK_RUNNER_LOG_LEVEL": log_level,
         "DSTACK_PUBLIC_SSH_KEY": "\n".join(authorized_keys),
     }
+    if backend_shim_env is not None:
+        envs |= backend_shim_env
     return envs
 
 
 def get_shim_commands(
-    authorized_keys: List[str], *, is_privileged: bool = False, pjrt_device: Optional[str] = None
+    authorized_keys: List[str],
+    *,
+    is_privileged: bool = False,
+    pjrt_device: Optional[str] = None,
+    base_path: Optional[PathLike] = None,
+    bin_path: Optional[PathLike] = None,
+    backend_shim_env: Optional[Dict[str, str]] = None,
 ) -> List[str]:
-    commands = get_shim_pre_start_commands()
-    for k, v in get_shim_env(authorized_keys).items():
+    commands = get_shim_pre_start_commands(
+        base_path=base_path,
+        bin_path=bin_path,
+    )
+    shim_env = get_shim_env(
+        authorized_keys=authorized_keys,
+        base_path=base_path,
+        bin_path=bin_path,
+        backend_shim_env=backend_shim_env,
+    )
+    for k, v in shim_env.items():
         commands += [f'export "{k}={v}"']
-    commands += get_run_shim_script(is_privileged, pjrt_device)
+    commands += get_run_shim_script(
+        is_privileged=is_privileged,
+        pjrt_device=pjrt_device,
+        bin_path=bin_path,
+    )
     return commands
 
 
@@ -510,25 +563,33 @@ def get_dstack_shim_download_url() -> str:
     return f"https://{bucket}.s3.eu-west-1.amazonaws.com/{build}/binaries/dstack-shim-linux-amd64"
 
 
-def get_shim_pre_start_commands() -> List[str]:
+def get_shim_pre_start_commands(
+    base_path: Optional[PathLike] = None,
+    bin_path: Optional[PathLike] = None,
+) -> List[str]:
     url = get_dstack_shim_download_url()
-
+    dstack_shim_binary_path = get_dstack_shim_binary_path(bin_path)
+    dstack_working_dir = get_dstack_working_dir(base_path)
     return [
         f"dlpath=$(sudo mktemp -t {DSTACK_SHIM_BINARY_NAME}.XXXXXXXXXX)",
         # -sS -- disable progress meter and warnings, but still show errors (unlike bare -s)
         f'sudo curl -sS --compressed --connect-timeout 60 --max-time 240 --retry 1 --output "$dlpath" "{url}"',
-        f'sudo mv "$dlpath" {DSTACK_SHIM_BINARY_PATH}',
-        f"sudo chmod +x {DSTACK_SHIM_BINARY_PATH}",
-        f"sudo mkdir {DSTACK_WORKING_DIR} -p",
+        f'sudo mv "$dlpath" {dstack_shim_binary_path}',
+        f"sudo chmod +x {dstack_shim_binary_path}",
+        f"sudo mkdir {dstack_working_dir} -p",
     ]
 
 
-def get_run_shim_script(is_privileged: bool, pjrt_device: Optional[str]) -> List[str]:
+def get_run_shim_script(
+    is_privileged: bool,
+    pjrt_device: Optional[str],
+    bin_path: Optional[PathLike] = None,
+) -> List[str]:
+    dstack_shim_binary_path = get_dstack_shim_binary_path(bin_path)
     privileged_flag = "--privileged" if is_privileged else ""
     pjrt_device_env = f"--pjrt-device={pjrt_device}" if pjrt_device else ""
-
     return [
-        f"nohup {DSTACK_SHIM_BINARY_PATH} {privileged_flag} {pjrt_device_env} &",
+        f"nohup {dstack_shim_binary_path} {privileged_flag} {pjrt_device_env} &",
     ]
 
 
@@ -555,8 +616,10 @@ def get_gateway_user_data(authorized_key: str) -> str:
 
 
 def get_docker_commands(
-    authorized_keys: List[str], fix_path_in_dot_profile: bool = True
-) -> List[str]:
+    authorized_keys: list[str],
+    bin_path: Optional[PathLike] = None,
+) -> list[str]:
+    dstack_runner_binary_path = get_dstack_runner_binary_path(bin_path)
     authorized_keys_content = "\n".join(authorized_keys).strip()
     commands = [
         # save and unset ld.so variables
@@ -580,9 +643,6 @@ def get_docker_commands(
         "chmod 700 ~/.ssh",
         f"echo '{authorized_keys_content}' > ~/.ssh/authorized_keys",
         "chmod 600 ~/.ssh/authorized_keys",
-        r"""if [ -f ~/.profile ]; then sed -ie '1s@^@export PATH="'"$PATH"':$PATH"\n\n@' ~/.profile; fi"""
-        if fix_path_in_dot_profile
-        else ":",
         # regenerate host keys
         "rm -rf /etc/ssh/ssh_host_*",
         "ssh-keygen -A > /dev/null",
@@ -600,7 +660,6 @@ def get_docker_commands(
             " -o PidFile=none"
             " -o PasswordAuthentication=no"
             " -o AllowTcpForwarding=yes"
-            " -o PermitUserEnvironment=yes"
             " -o ClientAliveInterval=30"
             " -o ClientAliveCountMax=4"
         ),
@@ -611,10 +670,10 @@ def get_docker_commands(
 
     url = get_dstack_runner_download_url()
     commands += [
-        f"curl --connect-timeout 60 --max-time 240 --retry 1 --output {DSTACK_RUNNER_BINARY_PATH} {url}",
-        f"chmod +x {DSTACK_RUNNER_BINARY_PATH}",
+        f"curl --connect-timeout 60 --max-time 240 --retry 1 --output {dstack_runner_binary_path} {url}",
+        f"chmod +x {dstack_runner_binary_path}",
         (
-            f"{DSTACK_RUNNER_BINARY_PATH}"
+            f"{dstack_runner_binary_path}"
             " --log-level 6"
             " start"
             f" --http-port {DSTACK_RUNNER_HTTP_PORT}"
@@ -692,9 +751,18 @@ def get_dstack_gateway_commands() -> List[str]:
     ]
 
 
-def merge_tags(tags: Dict[str, str], backend_tags: Optional[Dict[str, str]]) -> Dict[str, str]:
-    res = tags.copy()
+def merge_tags(
+    base_tags: Dict[str, str],
+    backend_tags: Optional[Dict[str, str]] = None,
+    resource_tags: Optional[Dict[str, str]] = None,
+) -> Dict[str, str]:
+    res = base_tags.copy()
+    # backend_tags have priority over resource_tags
+    # so that regular users do not override the tags set by admins
     if backend_tags is not None:
         for k, v in backend_tags.items():
             res.setdefault(k, v)
+    if resource_tags is not None:
+        for k, v in resource_tags.items():
+            res.setdefault(k, v)
     return res

dstack/_internal/core/backends/gcp/auth.py

@@ -13,7 +13,6 @@ from dstack._internal.core.backends.gcp.models import (
     GCPServiceAccountCreds,
 )
 from dstack._internal.core.errors import BackendAuthError
-from dstack._internal.core.models.common import is_core_model_instance
 
 
 def authenticate(creds: AnyGCPCreds, project_id: Optional[str] = None) -> Tuple[Credentials, str]:
@@ -30,7 +29,7 @@ def authenticate(creds: AnyGCPCreds, project_id: Optional[str] = None) -> Tuple[
 
 
 def get_credentials(creds: AnyGCPCreds) -> Tuple[Credentials, Optional[str]]:
-    if is_core_model_instance(creds, GCPServiceAccountCreds):
+    if isinstance(creds, GCPServiceAccountCreds):
         try:
             service_account_info = json.loads(creds.data)
             credentials = service_account.Credentials.from_service_account_info(

dstack/_internal/core/backends/gcp/compute.py

@@ -211,8 +211,12 @@ class GCPCompute(
             "dstack_name": instance_config.instance_name,
             "dstack_user": instance_config.user.lower(),
         }
-        labels = {k: v for k, v in labels.items() if gcp_resources.is_valid_label_value(v)}
-        labels = merge_tags(tags=labels, backend_tags=self.config.tags)
+        labels = merge_tags(
+            base_tags=labels,
+            backend_tags=self.config.tags,
+            resource_tags=instance_config.tags,
+        )
+        labels = gcp_resources.filter_invalid_labels(labels)
         is_tpu = (
             _is_tpu(instance_offer.instance.resources.gpus[0].name)
             if instance_offer.instance.resources.gpus
@@ -292,11 +296,9 @@ class GCPCompute(
                     gpus=instance_offer.instance.resources.gpus,
                 ),
                 spot=instance_offer.instance.resources.spot,
-                user_data=get_user_data(
-                    authorized_keys,
-                    backend_specific_commands=_get_backend_specific_commands(
-                        instance_offer.instance.name
-                    ),
+                user_data=_get_user_data(
+                    authorized_keys=authorized_keys,
+                    instance_type_name=instance_offer.instance.name,
                 ),
                 authorized_keys=authorized_keys,
                 labels=labels,
@@ -471,8 +473,12 @@ class GCPCompute(
             "dstack_project": configuration.project_name.lower(),
             "dstack_name": configuration.instance_name,
         }
-        labels = {k: v for k, v in labels.items() if gcp_resources.is_valid_label_value(v)}
-        labels = merge_tags(tags=labels, backend_tags=self.config.tags)
+        labels = merge_tags(
+            base_tags=labels,
+            backend_tags=self.config.tags,
+            resource_tags=configuration.tags,
+        )
+        labels = gcp_resources.filter_invalid_labels(labels)
 
         request = compute_v1.InsertInstanceRequest()
         request.zone = zone
@@ -573,8 +579,12 @@ class GCPCompute(
             "dstack_name": volume.name,
             "dstack_user": volume.user,
         }
-        labels = {k: v for k, v in labels.items() if gcp_resources.is_valid_label_value(v)}
-        labels = merge_tags(tags=labels, backend_tags=self.config.tags)
+        labels = merge_tags(
+            base_tags=labels,
+            backend_tags=self.config.tags,
+            resource_tags=volume.configuration.tags,
+        )
+        labels = gcp_resources.filter_invalid_labels(labels)
 
         disk = compute_v1.Disk()
         disk.name = disk_name
@@ -829,10 +839,14 @@ def _get_extra_subnets(
 ) -> List[Tuple[str, str]]:
     if config.extra_vpcs is None:
         return []
-    if instance_type_name != "a3-megagpu-8g":
+    if instance_type_name == "a3-megagpu-8g":
+        subnets_num = 8
+    elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+        subnets_num = 4
+    else:
         return []
     extra_subnets = []
-    for vpc_name in config.extra_vpcs:
+    for vpc_name in config.extra_vpcs[:subnets_num]:
         subnet = gcp_resources.get_vpc_subnet_or_error(
             subnetworks_client=subnetworks_client,
             vpc_project_id=config.vpc_project_id or config.project_id,
@@ -844,12 +858,14 @@ def _get_extra_subnets(
             vpc_name=vpc_name,
         )
         extra_subnets.append((vpc_resource_name, subnet))
-    return extra_subnets[:8]
+    return extra_subnets
 
 
 def _get_image_id(instance_type_name: str, cuda: bool) -> str:
     if instance_type_name == "a3-megagpu-8g":
         image_name = "dstack-a3mega-5"
+    elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+        return "projects/cos-cloud/global/images/cos-105-17412-535-78"
     elif cuda:
         image_name = f"dstack-cuda-{version.base_image}"
     else:
@@ -862,9 +878,37 @@ def _get_gateway_image_id() -> str:
     return "projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20230714"
 
 
+def _get_user_data(authorized_keys: List[str], instance_type_name: str) -> str:
+    base_path = None
+    bin_path = None
+    backend_shim_env = None
+    if instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+        # In the COS image the / file system is not writable.
+        # /home and /var are writable but not executable.
+        # Only /etc is both writable and executable, so use it for shim/runner binaries.
+        # See: https://cloud.google.com/container-optimized-os/docs/concepts/disks-and-filesystem
+        base_path = bin_path = "/etc"
+        backend_shim_env = {
+            # In COS nvidia binaries are not installed on PATH by default.
+            # Set so that shim can run nvidia-smi.
+            "PATH": "/var/lib/nvidia/bin:$PATH",
+        }
+    return get_user_data(
+        authorized_keys=authorized_keys,
+        backend_specific_commands=_get_backend_specific_commands(
+            instance_type_name=instance_type_name,
+        ),
+        base_path=base_path,
+        bin_path=bin_path,
+        backend_shim_env=backend_shim_env,
+    )
+
+
 def _get_backend_specific_commands(instance_type_name: str) -> List[str]:
     if instance_type_name == "a3-megagpu-8g":
         return tcpx_features.get_backend_specific_commands_tcpxo()
+    if instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+        return tcpx_features.get_backend_specific_commands_tcpx()
     return []
 
 

dstack/_internal/core/backends/gcp/configurator.py

@@ -24,7 +24,6 @@ from dstack._internal.core.errors import BackendAuthError, BackendError, ServerC
 from dstack._internal.core.models.backends.base import (
     BackendType,
 )
-from dstack._internal.core.models.common import is_core_model_instance
 
 LOCATIONS = [
     {
@@ -115,7 +114,7 @@ class GCPConfigurator(Configurator):
     BACKEND_CLASS = GCPBackend
 
     def validate_config(self, config: GCPBackendConfigWithCreds, default_creds_enabled: bool):
-        if is_core_model_instance(config.creds, GCPDefaultCreds) and not default_creds_enabled:
+        if isinstance(config.creds, GCPDefaultCreds) and not default_creds_enabled:
             raise_invalid_credentials_error(fields=[["creds"]])
         try:
             credentials, _ = auth.authenticate(creds=config.creds, project_id=config.project_id)
@@ -123,7 +122,7 @@ class GCPConfigurator(Configurator):
             details = None
             if len(e.args) > 0:
                 details = e.args[0]
-            if is_core_model_instance(config.creds, GCPServiceAccountCreds):
+            if isinstance(config.creds, GCPServiceAccountCreds):
                 raise_invalid_credentials_error(fields=[["creds", "data"]], details=details)
             else:
                 raise_invalid_credentials_error(fields=[["creds"]], details=details)

dstack/_internal/core/backends/gcp/features/tcpx.py

@@ -32,3 +32,34 @@ def get_backend_specific_commands_tcpxo() -> List[str]:
             "--num_hops=2 --num_nics=8 --uid= --alsologtostderr"
         ),
     ]
+
+
+def get_backend_specific_commands_tcpx() -> List[str]:
+    return [
+        "cos-extensions install gpu -- --version=latest",
+        "sudo mount --bind /var/lib/nvidia /var/lib/nvidia",
+        "sudo mount -o remount,exec /var/lib/nvidia",
+        (
+            "docker run "
+            "--detach "
+            "--pull=always "
+            "--name receive-datapath-manager "
+            "--privileged "
+            "--cap-add=NET_ADMIN --network=host "
+            "--volume /var/lib/nvidia/lib64:/usr/local/nvidia/lib64 "
+            "--device /dev/nvidia0:/dev/nvidia0 --device /dev/nvidia1:/dev/nvidia1 "
+            "--device /dev/nvidia2:/dev/nvidia2 --device /dev/nvidia3:/dev/nvidia3 "
+            "--device /dev/nvidia4:/dev/nvidia4 --device /dev/nvidia5:/dev/nvidia5 "
+            "--device /dev/nvidia6:/dev/nvidia6 --device /dev/nvidia7:/dev/nvidia7 "
+            "--device /dev/nvidia-uvm:/dev/nvidia-uvm --device /dev/nvidiactl:/dev/nvidiactl "
+            "--env LD_LIBRARY_PATH=/usr/local/nvidia/lib64 "
+            "--volume /run/tcpx:/run/tcpx "
+            "--entrypoint /tcpgpudmarxd/build/app/tcpgpudmarxd "
+            "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/tcpgpudmarxd "
+            '--gpu_nic_preset a3vm --gpu_shmem_type fd --uds_path "/run/tcpx" --setup_param "--verbose 128 2 0"'
+        ),
+        "sudo iptables -I INPUT -p tcp -m tcp -j ACCEPT",
+        "docker run --rm -v /var/lib:/var/lib us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx install --install-nccl",
+        "sudo mount --bind /var/lib/tcpx /var/lib/tcpx",
+        "sudo mount -o remount,exec /var/lib/tcpx",
+    ]

dstack/_internal/core/backends/gcp/resources.py

@@ -332,6 +332,16 @@ def get_accelerators(
     return [accelerator_config]
 
 
+def filter_invalid_labels(labels: Dict[str, str]) -> Dict[str, str]:
+    filtered_labels = {}
+    for k, v in labels.items():
+        if not _is_valid_label(k, v):
+            logger.warning("Skipping invalid label '%s: %s'", k, v)
+            continue
+        filtered_labels[k] = v
+    return filtered_labels
+
+
 def validate_labels(labels: Dict[str, str]):
     for k, v in labels.items():
         if not _is_valid_label(k, v):

dstack/_internal/core/backends/nebius/compute.py

@@ -86,7 +86,11 @@ class NebiusCompute(
 
     @cached_property
     def _region_to_project_id(self) -> dict[str, str]:
-        return resources.get_region_to_project_id_map(self._sdk)
+        return resources.get_region_to_project_id_map(
+            self._sdk,
+            configured_regions=self.config.regions,
+            configured_project_ids=self.config.projects,
+        )
 
     def _get_subnet_id(self, region: str) -> str:
         if region not in self._subnet_id_cache:
@@ -100,7 +104,7 @@ class NebiusCompute(
     ) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.NEBIUS,
-            locations=self.config.regions or list(self._region_to_project_id),
+            locations=list(self._region_to_project_id),
             requirements=requirements,
             extra_filter=_supported_instances,
             configurable_disk_size=CONFIGURABLE_DISK_SIZE,

dstack/_internal/core/backends/nebius/configurator.py

@@ -29,21 +29,15 @@ class NebiusConfigurator(Configurator):
         assert isinstance(config.creds, NebiusServiceAccountCreds)
         try:
             sdk = resources.make_sdk(config.creds)
-            available_regions = set(resources.get_region_to_project_id_map(sdk))
+            # check that it's possible to build the projects map with configured settings
+            resources.get_region_to_project_id_map(
+                sdk, configured_regions=config.regions, configured_project_ids=config.projects
+            )
         except (ValueError, RequestError) as e:
             raise_invalid_credentials_error(
                 fields=[["creds"]],
                 details=str(e),
             )
-        if invalid_regions := set(config.regions or []) - available_regions:
-            raise_invalid_credentials_error(
-                fields=[["regions"]],
-                details=(
-                    f"Configured regions {invalid_regions} do not exist in this Nebius tenancy."
-                    " Omit `regions` to use all regions or select some of the available regions:"
-                    f" {available_regions}"
-                ),
-            )
 
     def create_backend(
         self, project_name: str, config: NebiusBackendConfigWithCreds

dstack/_internal/core/backends/nebius/models.py

@@ -5,6 +5,8 @@ from pydantic import Field, root_validator
 from dstack._internal.core.backends.base.models import fill_data
 from dstack._internal.core.models.common import CoreModel
 
+DEFAULT_PROJECT_NAME_PREFIX = "default-project"
+
 
 class NebiusServiceAccountCreds(CoreModel):
     type: Annotated[Literal["service_account"], Field(description="The type of credentials")] = (
@@ -70,9 +72,20 @@ class NebiusBackendConfig(CoreModel):
         Literal["nebius"],
         Field(description="The type of backend"),
     ] = "nebius"
+    projects: Annotated[
+        Optional[list[str]],
+        Field(
+            description=(
+                "The list of allowed Nebius project IDs."
+                " Omit to use the default project in each region."
+                " The project is considered default if it is the only project in the region"
+                f" or if its name starts with `{DEFAULT_PROJECT_NAME_PREFIX}`"
+            )
+        ),
+    ] = None
     regions: Annotated[
         Optional[list[str]],
-        Field(description="The list of Nebius regions. Omit to use all regions"),
+        Field(description="The list of allowed Nebius regions. Omit to allow all regions"),
     ] = None