dstack 0.19.29__py3-none-any.whl → 0.19.30__py3-none-any.whl

dstack/_internal/cli/main.py

@@ -22,7 +22,7 @@ from dstack._internal.cli.commands.server import ServerCommand
 from dstack._internal.cli.commands.stats import StatsCommand
 from dstack._internal.cli.commands.stop import StopCommand
 from dstack._internal.cli.commands.volume import VolumeCommand
-from dstack._internal.cli.utils.common import _colors, console
+from dstack._internal.cli.utils.common import _colors, configure_logging, console
 from dstack._internal.cli.utils.updates import check_for_updates
 from dstack._internal.core.errors import ClientError, CLIError, ConfigurationError, SSHError
 from dstack._internal.core.services.ssh.client import get_ssh_client_info
@@ -39,6 +39,8 @@ def main():
     RichHelpFormatter.styles["argparse.groups"] = "bold grey74"
     RichHelpFormatter.styles["argparse.text"] = "grey74"
 
+    configure_logging()
+
     parser = argparse.ArgumentParser(
         description=(
             "Not sure where to start?"

dstack/_internal/cli/services/configurators/fleet.py

@@ -159,12 +159,19 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[Fle
         console.print(
             get_fleets_table(
                 [fleet],
-                verbose=_failed_provisioning(fleet),
+                verbose=_fleet_has_failed_instances(fleet),
                 format_date=local_time,
             )
         )
-        if _failed_provisioning(fleet):
-            console.print("\n[error]Some instances failed. Check the table above for errors.[/]")
+        if _fleet_has_failed_instances(fleet):
+            if _fleet_retrying(fleet):
+                console.print(
+                    "\n[error]Some instances failed. Provisioning will be retried in the background.[/]"
+                )
+            else:
+                console.print(
+                    "\n[error]Some instances failed. Check the table above for errors.[/]"
+                )
             exit(1)
 
     def _apply_plan_on_old_server(self, plan: FleetPlan, command_args: argparse.Namespace):
@@ -253,11 +260,11 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[Fle
         console.print(
             get_fleets_table(
                 [fleet],
-                verbose=_failed_provisioning(fleet),
+                verbose=_fleet_has_failed_instances(fleet),
                 format_date=local_time,
             )
         )
-        if _failed_provisioning(fleet):
+        if _fleet_has_failed_instances(fleet):
             console.print("\n[error]Some instances failed. Check the table above for errors.[/]")
             exit(1)
 
@@ -462,13 +469,20 @@ def _finished_provisioning(fleet: Fleet) -> bool:
     return True
 
 
-def _failed_provisioning(fleet: Fleet) -> bool:
+def _fleet_has_failed_instances(fleet: Fleet) -> bool:
     for instance in fleet.instances:
         if instance.status == InstanceStatus.TERMINATED:
             return True
     return False
 
 
+def _fleet_retrying(fleet: Fleet) -> bool:
+    if fleet.spec.configuration.nodes is None:
+        return False
+    active_instances = [i for i in fleet.instances if i.status.is_active()]
+    return len(active_instances) < fleet.spec.configuration.nodes.min
+
+
 def _apply_plan(api: Client, plan: FleetPlan) -> Fleet:
     try:
         return api.client.fleets.apply_plan(

dstack/_internal/cli/utils/gpu.py

@@ -9,7 +9,7 @@ from dstack._internal.core.models.runs import Requirements, RunSpec, get_policy_
 from dstack._internal.server.schemas.gpus import GpuGroup
 
 
-def print_gpu_json(gpu_response, run_spec, group_by_cli, api_project):
+def print_gpu_json(gpus, run_spec, group_by_cli, api_project):
     """Print GPU information in JSON format."""
     req = Requirements(
         resources=run_spec.configuration.resources,
@@ -36,7 +36,7 @@ def print_gpu_json(gpu_response, run_spec, group_by_cli, api_project):
         "gpus": [],
     }
 
-    for gpu_group in gpu_response.gpus:
+    for gpu_group in gpus:
         gpu_data = {
             "name": gpu_group.name,
             "memory_mib": gpu_group.memory_mib,

dstack/_internal/core/backends/aws/compute.py

@@ -291,7 +291,11 @@ class AWSCompute(
                 image_id, username = self._get_image_id_and_username(
                     ec2_client=ec2_client,
                     region=instance_offer.region,
-                    cuda=len(instance_offer.instance.resources.gpus) > 0,
+                    gpu_name=(
+                        instance_offer.instance.resources.gpus[0].name
+                        if len(instance_offer.instance.resources.gpus) > 0
+                        else None
+                    ),
                     instance_type=instance_offer.instance.name,
                     image_config=self.config.os_images,
                 )
@@ -897,11 +901,13 @@ class AWSCompute(
         self,
         ec2_client: botocore.client.BaseClient,
         region: str,
-        cuda: bool,
+        gpu_name: Optional[str],
         instance_type: str,
         image_config: Optional[AWSOSImageConfig] = None,
     ) -> tuple:
-        return hashkey(region, cuda, instance_type, image_config.json() if image_config else None)
+        return hashkey(
+            region, gpu_name, instance_type, image_config.json() if image_config else None
+        )
 
     @cachedmethod(
         cache=lambda self: self._get_image_id_and_username_cache,
@@ -912,13 +918,13 @@ class AWSCompute(
         self,
         ec2_client: botocore.client.BaseClient,
         region: str,
-        cuda: bool,
+        gpu_name: Optional[str],
         instance_type: str,
         image_config: Optional[AWSOSImageConfig] = None,
     ) -> tuple[str, str]:
         return aws_resources.get_image_id_and_username(
             ec2_client=ec2_client,
-            cuda=cuda,
+            gpu_name=gpu_name,
             instance_type=instance_type,
             image_config=image_config,
         )

dstack/_internal/core/backends/aws/resources.py

@@ -6,6 +6,8 @@ import botocore.exceptions
 
 import dstack.version as version
 from dstack._internal.core.backends.aws.models import AWSOSImageConfig
+from dstack._internal.core.backends.base.compute import requires_nvidia_proprietary_kernel_modules
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import BackendError, ComputeError, ComputeResourceNotFoundError
 from dstack._internal.utils.logging import get_logger
 
@@ -17,14 +19,14 @@ DLAMI_OWNER_ACCOUNT_ID = "898082745236"
 
 def get_image_id_and_username(
     ec2_client: botocore.client.BaseClient,
-    cuda: bool,
+    gpu_name: Optional[str],
     instance_type: str,
     image_config: Optional[AWSOSImageConfig] = None,
 ) -> tuple[str, str]:
     if image_config is not None:
-        image = image_config.nvidia if cuda else image_config.cpu
+        image = image_config.nvidia if gpu_name else image_config.cpu
         if image is None:
-            logger.warning("%s image not configured", "nvidia" if cuda else "cpu")
+            logger.warning("%s image not configured", "nvidia" if gpu_name else "cpu")
             raise ComputeResourceNotFoundError()
         image_name = image.name
         image_owner = image.owner
@@ -35,9 +37,12 @@ def get_image_id_and_username(
         image_owner = DLAMI_OWNER_ACCOUNT_ID
         username = "ubuntu"
     else:
-        image_name = (
-            f"dstack-{version.base_image}" if not cuda else f"dstack-cuda-{version.base_image}"
-        )
+        if gpu_name is None:
+            image_name = f"dstack-{version.base_image}"
+        elif not requires_nvidia_proprietary_kernel_modules(gpu_name):
+            image_name = f"dstack-cuda-{version.base_image}"
+        else:
+            image_name = f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
         image_owner = DSTACK_ACCOUNT_ID
         username = "ubuntu"
     response = ec2_client.describe_images(

dstack/_internal/core/backends/azure/compute.py

@@ -48,8 +48,10 @@ from dstack._internal.core.backends.base.compute import (
     get_gateway_user_data,
     get_user_data,
     merge_tags,
+    requires_nvidia_proprietary_kernel_modules,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers, get_offers_disk_modifier
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import ComputeError, NoCapacityError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.gateways import (
@@ -372,6 +374,7 @@ def _parse_config_vpc_id(vpc_id: str) -> Tuple[str, str]:
 class VMImageVariant(enum.Enum):
     GRID = enum.auto()
     CUDA = enum.auto()
+    CUDA_WITH_PROPRIETARY_KERNEL_MODULES = enum.auto()
     STANDARD = enum.auto()
 
     @classmethod
@@ -379,18 +382,24 @@ class VMImageVariant(enum.Enum):
         if "_A10_v5" in instance.name:
             return cls.GRID
         elif len(instance.resources.gpus) > 0:
-            return cls.CUDA
+            if not requires_nvidia_proprietary_kernel_modules(instance.resources.gpus[0].name):
+                return cls.CUDA
+            else:
+                return cls.CUDA_WITH_PROPRIETARY_KERNEL_MODULES
         else:
             return cls.STANDARD
 
     def get_image_name(self) -> str:
-        name = "dstack-"
         if self is self.GRID:
-            name += "grid-"
+            return f"dstack-grid-{version.base_image}"
         elif self is self.CUDA:
-            name += "cuda-"
-        name += version.base_image
-        return name
+            return f"dstack-cuda-{version.base_image}"
+        elif self is self.CUDA_WITH_PROPRIETARY_KERNEL_MODULES:
+            return f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
+        elif self is self.STANDARD:
+            return f"dstack-{version.base_image}"
+        else:
+            raise ValueError(f"Unexpected image variant {self!r}")
 
 
 _SUPPORTED_VM_SERIES_PATTERNS = [

dstack/_internal/core/backends/base/compute.py

@@ -48,6 +48,22 @@ logger = get_logger(__name__)
 DSTACK_SHIM_BINARY_NAME = "dstack-shim"
 DSTACK_RUNNER_BINARY_NAME = "dstack-runner"
 DEFAULT_PRIVATE_SUBNETS = ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")
+NVIDIA_GPUS_REQUIRING_PROPRIETARY_KERNEL_MODULES = frozenset(
+    # All NVIDIA architectures prior to Turing do not support Open Kernel Modules and require
+    # proprietary modules. This list is incomplete, update when necessary.
+    [
+        "v100",
+        "p100",
+        "p40",
+        "p4",
+        "m60",
+        "m40",
+        "m4",
+        "k80",
+        "k40",
+        "k20",
+    ]
+)
 
 GoArchType = Literal["amd64", "arm64"]
 
@@ -969,3 +985,12 @@ def merge_tags(
         for k, v in resource_tags.items():
             res.setdefault(k, v)
     return res
+
+
+def requires_nvidia_proprietary_kernel_modules(gpu_name: str) -> bool:
+    """
+    Returns:
+        Whether this NVIDIA GPU requires NVIDIA proprietary kernel modules
+        instead of open kernel modules.
+    """
+    return gpu_name.lower() in NVIDIA_GPUS_REQUIRING_PROPRIETARY_KERNEL_MODULES

dstack/_internal/core/backends/gcp/compute.py

@@ -31,6 +31,7 @@ from dstack._internal.core.backends.base.compute import (
     get_shim_commands,
     get_user_data,
     merge_tags,
+    requires_nvidia_proprietary_kernel_modules,
 )
 from dstack._internal.core.backends.base.offers import (
     get_catalog_offers,
@@ -38,6 +39,7 @@ from dstack._internal.core.backends.base.offers import (
 )
 from dstack._internal.core.backends.gcp.features import tcpx as tcpx_features
 from dstack._internal.core.backends.gcp.models import GCPConfig
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import (
     ComputeError,
     ComputeResourceNotFoundError,
@@ -293,7 +295,11 @@ class GCPCompute(
 
         image = _get_image(
             instance_type_name=instance_offer.instance.name,
-            cuda=len(instance_offer.instance.resources.gpus) > 0,
+            gpu_name=(
+                instance_offer.instance.resources.gpus[0].name
+                if len(instance_offer.instance.resources.gpus) > 0
+                else None
+            ),
         )
 
         for zone in zones:
@@ -904,7 +910,7 @@ class GCPImage:
     is_ufw_installed: bool
 
 
-def _get_image(instance_type_name: str, cuda: bool) -> GCPImage:
+def _get_image(instance_type_name: str, gpu_name: Optional[str]) -> GCPImage:
     if instance_type_name == "a3-megagpu-8g":
         image_name = "dstack-a3mega-5"
         is_ufw_installed = False
@@ -913,8 +919,11 @@ def _get_image(instance_type_name: str, cuda: bool) -> GCPImage:
             id="projects/cos-cloud/global/images/cos-105-17412-535-78",
             is_ufw_installed=False,
         )
-    elif cuda:
-        image_name = f"dstack-cuda-{version.base_image}"
+    elif gpu_name is not None:
+        if not requires_nvidia_proprietary_kernel_modules(gpu_name):
+            image_name = f"dstack-cuda-{version.base_image}"
+        else:
+            image_name = f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
         is_ufw_installed = True
     else:
         image_name = f"dstack-{version.base_image}"

dstack/_internal/core/backends/oci/compute.py

@@ -118,7 +118,11 @@ class OCICompute(
         availability_domain = instance_offer.availability_zones[0]
 
         listing, package = resources.get_marketplace_listing_and_package(
-            cuda=len(instance_offer.instance.resources.gpus) > 0,
+            gpu_name=(
+                instance_offer.instance.resources.gpus[0].name
+                if len(instance_offer.instance.resources.gpus) > 0
+                else None
+            ),
             client=region.marketplace_client,
         )
         resources.accept_marketplace_listing_agreements(

dstack/_internal/core/backends/oci/resources.py

@@ -23,7 +23,9 @@ import oci
 from oci.object_storage.models import CreatePreauthenticatedRequestDetails
 
 from dstack import version
+from dstack._internal.core.backends.base.compute import requires_nvidia_proprietary_kernel_modules
 from dstack._internal.core.backends.oci.region import OCIRegionClient
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import BackendError
 from dstack._internal.core.models.instances import InstanceOffer
 from dstack._internal.utils.common import batched
@@ -352,11 +354,14 @@ def terminate_instance_if_exists(client: oci.core.ComputeClient, instance_id: st
 
 
 def get_marketplace_listing_and_package(
-    cuda: bool, client: oci.marketplace.MarketplaceClient
+    gpu_name: Optional[str], client: oci.marketplace.MarketplaceClient
 ) -> Tuple[oci.marketplace.models.Listing, oci.marketplace.models.ImageListingPackage]:
     listing_name = f"dstack-{version.base_image}"
-    if cuda:
-        listing_name = f"dstack-cuda-{version.base_image}"
+    if gpu_name is not None:
+        if not requires_nvidia_proprietary_kernel_modules(gpu_name):
+            listing_name = f"dstack-cuda-{version.base_image}"
+        else:
+            listing_name = f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
 
     listing_summaries = list_marketplace_listings(listing_name, client)
     if len(listing_summaries) != 1:

dstack/_internal/core/consts.py

@@ -4,3 +4,5 @@ DSTACK_SHIM_HTTP_PORT = 10998
 DSTACK_RUNNER_HTTP_PORT = 10999
 # ssh server (runs alongside the runner inside a container) listen port
 DSTACK_RUNNER_SSH_PORT = 10022
+# legacy AWS, Azure, GCP, and OCI image for older GPUs
+DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES = "0.10"

dstack/_internal/core/services/repos.py

@@ -36,24 +36,59 @@ def get_repo_creds_and_default_branch(
 
     # no auth
     with suppress(InvalidRepoCredentialsError):
-        return _get_repo_creds_and_default_branch_https(url)
+        creds, default_branch = _get_repo_creds_and_default_branch_https(url)
+        logger.debug(
+            "Git repo %s is public. Using no auth. Default branch: %s", repo_url, default_branch
+        )
+        return creds, default_branch
 
     # ssh key provided by the user or pulled from the server
     if identity_file is not None or private_key is not None:
         if identity_file is not None:
             private_key = _read_private_key(identity_file)
-            return _get_repo_creds_and_default_branch_ssh(url, identity_file, private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, identity_file, private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using identity file: %s. Default branch: %s",
+                repo_url,
+                identity_file,
+                default_branch,
+            )
+            return creds, default_branch
         elif private_key is not None:
             with NamedTemporaryFile("w+", 0o600) as f:
                 f.write(private_key)
                 f.flush()
-                return _get_repo_creds_and_default_branch_ssh(url, f.name, private_key)
+                creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                    url, f.name, private_key
+                )
+                masked_key = "***" + private_key[-10:] if len(private_key) > 10 else "***MASKED***"
+                logger.debug(
+                    "Git repo %s is private. Using private key: %s. Default branch: %s",
+                    repo_url,
+                    masked_key,
+                    default_branch,
+                )
+                return creds, default_branch
         else:
             assert False, "should not reach here"
 
     # oauth token provided by the user or pulled from the server
     if oauth_token is not None:
-        return _get_repo_creds_and_default_branch_https(url, oauth_token)
+        creds, default_branch = _get_repo_creds_and_default_branch_https(url, oauth_token)
+        masked_token = (
+            len(oauth_token[:-4]) * "*" + oauth_token[-4:]
+            if len(oauth_token) > 4
+            else "***MASKED***"
+        )
+        logger.debug(
+            "Git repo %s is private. Using provided OAuth token: %s. Default branch: %s",
+            repo_url,
+            masked_token,
+            default_branch,
+        )
+        return creds, default_branch
 
     # key from ssh config
     identities = get_host_config(url.original_host).get("identityfile")
@@ -61,7 +96,16 @@ def get_repo_creds_and_default_branch(
         _identity_file = identities[0]
         with suppress(InvalidRepoCredentialsError):
             _private_key = _read_private_key(_identity_file)
-            return _get_repo_creds_and_default_branch_ssh(url, _identity_file, _private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, _identity_file, _private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using SSH config identity file: %s. Default branch: %s",
+                repo_url,
+                _identity_file,
+                default_branch,
+            )
+            return creds, default_branch
 
     # token from gh config
     if os.path.exists(gh_config_path):
@@ -70,13 +114,35 @@ def get_repo_creds_and_default_branch(
         _oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
         if _oauth_token is not None:
             with suppress(InvalidRepoCredentialsError):
-                return _get_repo_creds_and_default_branch_https(url, _oauth_token)
+                creds, default_branch = _get_repo_creds_and_default_branch_https(url, _oauth_token)
+                masked_token = (
+                    len(_oauth_token[:-4]) * "*" + _oauth_token[-4:]
+                    if len(_oauth_token) > 4
+                    else "***MASKED***"
+                )
+                logger.debug(
+                    "Git repo %s is private. Using GitHub config token: %s from %s. Default branch: %s",
+                    repo_url,
+                    masked_token,
+                    gh_config_path,
+                    default_branch,
+                )
+                return creds, default_branch
 
     # default user key
     if os.path.exists(default_ssh_key):
         with suppress(InvalidRepoCredentialsError):
             _private_key = _read_private_key(default_ssh_key)
-            return _get_repo_creds_and_default_branch_ssh(url, default_ssh_key, _private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, default_ssh_key, _private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using default identity file: %s. Default branch: %s",
+                repo_url,
+                default_ssh_key,
+                default_branch,
+            )
+            return creds, default_branch
 
     raise InvalidRepoCredentialsError(
         "No valid default Git credentials found. Pass valid `--token` or `--git-identity`."
@@ -87,8 +153,9 @@ def _get_repo_creds_and_default_branch_ssh(
     url: GitRepoURL, identity_file: PathLike, private_key: str
 ) -> tuple[RemoteRepoCreds, Optional[str]]:
     _url = url.as_ssh()
+    env = _make_git_env_for_creds_check(identity_file=identity_file)
     try:
-        default_branch = _get_repo_default_branch(_url, make_git_env(identity_file=identity_file))
+        default_branch = _get_repo_default_branch(_url, env)
     except GitCommandError as e:
         message = f"Cannot access `{_url}` using the `{identity_file}` private SSH key"
         raise InvalidRepoCredentialsError(message) from e
@@ -104,8 +171,9 @@ def _get_repo_creds_and_default_branch_https(
     url: GitRepoURL, oauth_token: Optional[str] = None
 ) -> tuple[RemoteRepoCreds, Optional[str]]:
     _url = url.as_https()
+    env = _make_git_env_for_creds_check()
     try:
-        default_branch = _get_repo_default_branch(url.as_https(oauth_token), make_git_env())
+        default_branch = _get_repo_default_branch(url.as_https(oauth_token), env)
     except GitCommandError as e:
         message = f"Cannot access `{_url}`"
         if oauth_token is not None:
@@ -120,9 +188,32 @@ def _get_repo_creds_and_default_branch_https(
     return creds, default_branch
 
 
+def _make_git_env_for_creds_check(identity_file: Optional[PathLike] = None) -> dict[str, str]:
+    # Our goal is to check if _provided_ creds (if any) are correct, so we need to be sure that
+    # only the provided creds are used, without falling back to any additional mechanisms.
+    # To do this, we:
+    # 1. Disable all configs to ignore any stored creds
+    # 2. Disable askpass to avoid asking for creds interactively or fetching stored creds from
+    # a non-interactive askpass helper (for example, VS Code sets GIT_ASKPASS to its own helper,
+    # which silently provides creds to Git).
+    return make_git_env(disable_config=True, disable_askpass=True, identity_file=identity_file)
+
+
 def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]:
+    # Git shipped by Apple with XCode is patched to support an additional config scope
+    # above "system" called "xcode". There is no option in `git config list` to show this config,
+    # but you can list the merged config (`git config list` without options) and then exclude
+    # all settings listed in `git config list --{system,global,local,worktree}`.
+    # As of time of writing, there are only two settings in the "xcode" config, one of which breaks
+    # our "is repo public?" check, namely "credential.helper=osxkeychain".
+    # As there is no way to disable "xcode" config (no env variable, no CLI option, etc.),
+    # the only way to disable credential helper is to override this specific setting with an empty
+    # string via command line argument: `git -c credential.helper= COMMAND [ARGS ...]`.
+    # See: https://github.com/git/git/commit/3d4355712b9fe77a96ad4ad877d92dc7ff6e0874
+    # See: https://gist.github.com/ChrisTollefson/ab9c0a5d1dd4dd615217345c6936a307
+    _git = git.cmd.Git()(c="credential.helper=")
     # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD"
-    output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env)
+    output: str = _git.ls_remote("--symref", url, "HEAD", env=env)
     for line in output.splitlines():
         # line format: `<oid> TAB <ref> LF`
         oid, _, ref = line.partition("\t")

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -1139,7 +1139,7 @@ def _patch_base_image_for_aws_efa(
     efa_enabled_patterns = [
         # TODO: p6-b200 isn't supported yet in gpuhunt
         r"^p6-b200\.(48xlarge)$",
-        r"^p5\.(48xlarge)$",
+        r"^p5\.(4xlarge|48xlarge)$",
         r"^p5e\.(48xlarge)$",
         r"^p5en\.(48xlarge)$",
         r"^p4d\.(24xlarge)$",

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -3,7 +3,7 @@ import itertools
 import math
 import uuid
 from datetime import datetime, timedelta
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 from sqlalchemy import and_, func, not_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -25,6 +25,7 @@ from dstack._internal.core.models.instances import InstanceOfferWithAvailability
 from dstack._internal.core.models.profiles import (
     DEFAULT_RUN_TERMINATION_IDLE_TIME,
     CreationPolicy,
+    Profile,
     TerminationPolicy,
 )
 from dstack._internal.core.models.resources import Memory
@@ -34,6 +35,7 @@ from dstack._internal.core.models.runs import (
     JobRuntimeData,
     JobStatus,
     JobTerminationReason,
+    Requirements,
     Run,
     RunSpec,
 )
@@ -186,7 +188,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
     project = run_model.project
     run = run_model_to_run(run_model)
     run_spec = run.run_spec
-    profile = run_spec.merged_profile
+    run_profile = run_spec.merged_profile
     job = find_job(run.jobs, job_model.replica_num, job_model.job_num)
     multinode = job.job_spec.jobs_per_replica > 1
 
@@ -333,7 +335,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         job_model.status = JobStatus.PROVISIONING
     else:
         # Assigned no instance, create a new one
-        if profile.creation_policy == CreationPolicy.REUSE:
+        if run_profile.creation_policy == CreationPolicy.REUSE:
             logger.debug("%s: reuse instance failed", fmt(job_model))
             job_model.status = JobStatus.TERMINATING
             job_model.termination_reason = JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
@@ -362,7 +364,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             return
 
         logger.info("%s: now is provisioning a new instance", fmt(job_model))
-        job_provisioning_data, offer = run_job_result
+        job_provisioning_data, offer, effective_profile, _ = run_job_result
         job_model.job_provisioning_data = job_provisioning_data.json()
         job_model.status = JobStatus.PROVISIONING
         if fleet_model is None:
@@ -382,12 +384,11 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         instance = _create_instance_model_for_job(
             project=project,
             fleet_model=fleet_model,
-            run_spec=run_spec,
             job_model=job_model,
-            job=job,
             job_provisioning_data=job_provisioning_data,
             offer=offer,
             instance_num=instance_num,
+            profile=effective_profile,
         )
         job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
         # Both this task and process_fleets can add instances to fleets.
@@ -546,23 +547,22 @@ async def _find_optimal_fleet_with_offers(
             fleet_cheapest_pool_offer = fleet_instances_with_pool_offers[0][1].price
 
         candidate_fleet = fleet_model_to_fleet(candidate_fleet_model)
-        profile = combine_fleet_and_run_profiles(
-            candidate_fleet.spec.merged_profile, run_spec.merged_profile
-        )
-        fleet_requirements = get_fleet_requirements(candidate_fleet.spec)
-        requirements = combine_fleet_and_run_requirements(
-            fleet_requirements, job.job_spec.requirements
-        )
-        multinode = (
-            candidate_fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
-            or job.job_spec.jobs_per_replica > 1
-        )
+        profile = None
+        requirements = None
+        try:
+            profile, requirements = _get_run_profile_and_requirements_in_fleet(
+                job=job,
+                run_spec=run_spec,
+                fleet=candidate_fleet,
+            )
+        except ValueError:
+            pass
         fleet_backend_offers = []
-        if (
-            _check_can_create_new_instance_in_fleet(candidate_fleet)
-            and profile is not None
-            and requirements is not None
-        ):
+        if profile is not None and requirements is not None:
+            multinode = (
+                candidate_fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
+                or job.job_spec.jobs_per_replica > 1
+            )
             fleet_backend_offers = await get_offers_by_requirements(
                 project=project,
                 profile=profile,
@@ -704,7 +704,7 @@ async def _run_job_on_new_instance(
     master_job_provisioning_data: Optional[JobProvisioningData] = None,
     volumes: Optional[List[List[Volume]]] = None,
     fleet_model: Optional[FleetModel] = None,
-) -> Optional[Tuple[JobProvisioningData, InstanceOfferWithAvailability]]:
+) -> Optional[tuple[JobProvisioningData, InstanceOfferWithAvailability, Profile, Requirements]]:
     if volumes is None:
         volumes = []
     profile = run.run_spec.merged_profile
@@ -712,21 +712,14 @@ async def _run_job_on_new_instance(
     fleet = None
     if fleet_model is not None:
         fleet = fleet_model_to_fleet(fleet_model)
-        if not _check_can_create_new_instance_in_fleet(fleet):
-            logger.debug(
-                "%s: cannot fit new instance into fleet %s", fmt(job_model), fleet_model.name
-            )
-            return None
-        profile = combine_fleet_and_run_profiles(fleet.spec.merged_profile, profile)
-        if profile is None:
-            logger.debug("%s: cannot combine fleet %s profile", fmt(job_model), fleet_model.name)
-            return None
-        fleet_requirements = get_fleet_requirements(fleet.spec)
-        requirements = combine_fleet_and_run_requirements(fleet_requirements, requirements)
-        if requirements is None:
-            logger.debug(
-                "%s: cannot combine fleet %s requirements", fmt(job_model), fleet_model.name
+        try:
+            profile, requirements = _get_run_profile_and_requirements_in_fleet(
+                job=job,
+                run_spec=run.run_spec,
+                fleet=fleet,
             )
+        except ValueError as e:
+            logger.debug("%s: %s", fmt(job_model), e.args[0])
             return None
         # TODO: Respect fleet provisioning properties such as tags
 
@@ -766,7 +759,7 @@ async def _run_job_on_new_instance(
                 project_ssh_private_key,
                 offer_volumes,
             )
-            return job_provisioning_data, offer
+            return job_provisioning_data, offer, profile, requirements
         except BackendError as e:
             logger.warning(
                 "%s: %s launch in %s/%s failed: %s",
@@ -789,6 +782,25 @@ async def _run_job_on_new_instance(
     return None
 
 
+def _get_run_profile_and_requirements_in_fleet(
+    job: Job,
+    run_spec: RunSpec,
+    fleet: Fleet,
+) -> tuple[Profile, Requirements]:
+    if not _check_can_create_new_instance_in_fleet(fleet):
+        raise ValueError("Cannot fit new instance into fleet")
+    profile = combine_fleet_and_run_profiles(fleet.spec.merged_profile, run_spec.merged_profile)
+    if profile is None:
+        raise ValueError("Cannot combine fleet profile")
+    fleet_requirements = get_fleet_requirements(fleet.spec)
+    requirements = combine_fleet_and_run_requirements(
+        fleet_requirements, job.job_spec.requirements
+    )
+    if requirements is None:
+        raise ValueError("Cannot combine fleet requirements")
+    return profile, requirements
+
+
 def _check_can_create_new_instance_in_fleet(fleet: Fleet) -> bool:
     if fleet.spec.configuration.ssh_config is not None:
         return False
@@ -857,14 +869,12 @@ async def _get_next_instance_num(session: AsyncSession, fleet_model: FleetModel)
 def _create_instance_model_for_job(
     project: ProjectModel,
     fleet_model: FleetModel,
-    run_spec: RunSpec,
     job_model: JobModel,
-    job: Job,
     job_provisioning_data: JobProvisioningData,
     offer: InstanceOfferWithAvailability,
     instance_num: int,
+    profile: Profile,
 ) -> InstanceModel:
-    profile = run_spec.merged_profile
     if not job_provisioning_data.dockerized:
         # terminate vastai/k8s instances immediately
         termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE

dstack/_internal/utils/ssh.py

@@ -50,8 +50,28 @@ def make_ssh_command_for_git(identity_file: PathLike) -> str:
     )
 
 
-def make_git_env(*, identity_file: Optional[PathLike] = None) -> dict[str, str]:
-    env: dict[str, str] = {"GIT_TERMINAL_PROMPT": "0"}
+def make_git_env(
+    *,
+    disable_prompt: bool = True,
+    disable_askpass: bool = False,
+    disable_config: bool = False,
+    identity_file: Optional[PathLike] = None,
+) -> dict[str, str]:
+    env: dict[str, str] = {}
+    if disable_prompt:
+        # Fail with error instead of prompting on the terminal (e.g., when asking for
+        # HTTP authentication)
+        env["GIT_TERMINAL_PROMPT"] = "0"
+    if disable_askpass:
+        env["GIT_ASKPASS"] = ""
+        env["SSH_ASKPASS"] = ""
+    if disable_config:
+        # Disable system-wide config (usually /etc/gitconfig)
+        env["GIT_CONFIG_SYSTEM"] = os.devnull
+        # Disable user (aka "global") config ($XDG_CONFIG_HOME/git/config or ~/.git/config)
+        env["GIT_CONFIG_GLOBAL"] = os.devnull
+        # Disable repo (aka "local") config (./.git/config)
+        env["GIT_DIR"] = os.devnull
     if identity_file is not None:
         env["GIT_SSH_COMMAND"] = make_ssh_command_for_git(identity_file)
     return env

dstack/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.29"
+__version__ = "0.19.30"
 __is_release__ = True
-base_image = "0.10"
+base_image = "0.11rc2"
 base_image_ubuntu_version = "22.04"

{dstack-0.19.29.dist-info → dstack-0.19.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.29
+Version: 0.19.30
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -331,24 +331,26 @@ Description-Content-Type: text/markdown
 
 </div>
 
-`dstack` is an open-source container orchestrator that simplifies workload orchestration and drives GPU utilization for ML teams. It works with any GPU cloud, on-prem cluster, or accelerated hardware.
+`dstack` provides a unified control plane for running development, training, and inference on GPUs — across cloud VMs, Kubernetes, or on-prem clusters. It helps your team avoid vendor lock-in and reduce GPU costs.
 
 #### Accelerators
 
 `dstack` supports `NVIDIA`, `AMD`, `Google TPU`, `Intel Gaudi`, and `Tenstorrent` accelerators out of the box.
 
 ## Latest news ✨
+- [2025/09] [dstack 0.19.27: Offers UI, Digital Ocean and AMD Developer Cloud](https://github.com/dstackai/dstack/releases/tag/0.19.27)
+- [2025/08] [dstack 0.19.26: Repos – explicit repo configuration via YAML](https://github.com/dstackai/dstack/releases/tag/0.19.26)
+- [2025/08] [dstack 0.19.25: `dstack offer` CLI command](https://github.com/dstackai/dstack/releases/tag/0.19.25)
+- [2025/08] [dstack 0.19.22: Service probes, GPU health-checks, Tenstorrent Galaxy, Secrets UI](https://github.com/dstackai/dstack/releases/tag/0.19.22)
+- [2025/07] [dstack 0.19.21: Scheduled tasks](https://github.com/dstackai/dstack/releases/tag/0.19.21)
 - [2025/07] [dstack 0.19.17: Secrets, Files, Rolling deployment](https://github.com/dstackai/dstack/releases/tag/0.19.17)
 - [2025/06] [dstack 0.19.16: Docker in Docker, CloudRift](https://github.com/dstackai/dstack/releases/tag/0.19.16)
 - [2025/06] [dstack 0.19.13: InfiniBand support in default images](https://github.com/dstackai/dstack/releases/tag/0.19.13)
 - [2025/06] [dstack 0.19.12: Simplified use of MPI](https://github.com/dstackai/dstack/releases/tag/0.19.12)
-- [2025/05] [dstack 0.19.10: Priorities](https://github.com/dstackai/dstack/releases/tag/0.19.10)
-- [2025/05] [dstack 0.19.8: Nebius clusters, GH200 on Lambda](https://github.com/dstackai/dstack/releases/tag/0.19.8)
-- [2025/04] [dstack 0.19.6: Tenstorrent, Plugins](https://github.com/dstackai/dstack/releases/tag/0.19.6)
 
 ## How does it work?
 
-<img src="https://dstack.ai/static-assets/static-assets/images/dstack-architecture-diagram-v10.svg" width="750" />
+<img src="https://dstack.ai/static-assets/static-assets/images/dstack-architecture-diagram-v11.svg" width="750" />
 
 ### Installation
 

{dstack-0.19.29.dist-info → dstack-0.19.30.dist-info}/RECORD

@@ -1,10 +1,10 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=YFaOhGPwo6LX5fJekfVb7R4HaBPGERtQmy7ytf9Rmq0,102
+dstack/version.py,sha256=2A1fxFQ9n9Kcprebn6q6fi-3iIYieg8EN2ej5aCbbYs,105
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=FYtd7tRk17Oc62Kl_3O8NuT5JHb8TKhLThl1TsfjjVs,1390
 dstack/_internal/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/cli/main.py,sha256=AZ4TB025U5wpQq3qH5sk9KkOYZCpxMIKqfaLH_Mv0DU,3876
+dstack/_internal/cli/main.py,sha256=0QScKrDlNf0rHXVhSiprAKOMJxJbmlc6QKTRCZDpb0U,3920
 dstack/_internal/cli/commands/__init__.py,sha256=aZdSeBeL38BYl_kjkdTHNbsQZiFbpDAHxas01be9CG8,2269
 dstack/_internal/cli/commands/apply.py,sha256=py0ysrUSo8EbwzpHRS3d_vbO3ALxpDFQmscieSoKkjE,3725
 dstack/_internal/cli/commands/attach.py,sha256=75hojBqYRXj_65eQNvjypuyDy1G1HmajzOHSLe3pOIk,5148
@@ -31,7 +31,7 @@ dstack/_internal/cli/services/profile.py,sha256=TiNMP_0Xbx9cD22zc8dTHHNIsZeIpavW
 dstack/_internal/cli/services/repos.py,sha256=wvb3nHI9yJ0fb-dWf2yihl6zdG1ZRVpKiC0CUz9qL7Y,2100
 dstack/_internal/cli/services/configurators/__init__.py,sha256=AwbKV85_XH0V8IS8v2Zpgl8vnvVCLMLbMuPyvsZv3mQ,2971
 dstack/_internal/cli/services/configurators/base.py,sha256=9mabsRsKHY7xFl8yDRM__FOEQvtEYOn_4KeXKzO6-vA,3244
-dstack/_internal/cli/services/configurators/fleet.py,sha256=pPfQmXBdqXHy8QAxGm9yY3Y3vKis-vaKirLrCO84zQA,18827
+dstack/_internal/cli/services/configurators/fleet.py,sha256=xchLFqHkm0zVs3tEVMlYXd4lGjP_OWzdp3nzqNg7b8k,19374
 dstack/_internal/cli/services/configurators/gateway.py,sha256=B2aEoVhhRZGk25OekR7BT4eAvAgZGAyVOw7DlSuo2jg,8755
 dstack/_internal/cli/services/configurators/run.py,sha256=lD14geWv8WtImrVDa5w9bXmA8JK8xIcTUawBTbbu6ZE,39543
 dstack/_internal/cli/services/configurators/volume.py,sha256=ZeESHttKFERBFysJvSGFC4UXUOLghKlzcilXbL48s-c,8346
@@ -39,14 +39,14 @@ dstack/_internal/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 dstack/_internal/cli/utils/common.py,sha256=xgq2Rdx6wGS1NLx8FlmkjKrGaNYD428oeBB4VMjvJ5c,3447
 dstack/_internal/cli/utils/fleet.py,sha256=qgyZnhKSsdx4nWVnzcU30vhwV7dtrwn8C0yNKMQ1H4g,4044
 dstack/_internal/cli/utils/gateway.py,sha256=qMYa1NTAT_O98x2_mSyWDRbiHj5fqt6xUXFh9NIUwAM,1502
-dstack/_internal/cli/utils/gpu.py,sha256=5LrOFo2bWSNZcyf_vCxhInT1AFYZLbPUvOqJgWP45vM,7688
+dstack/_internal/cli/utils/gpu.py,sha256=sgkSZbK50i4MdVAP44b0Xg1QX631YBDOxi_8PCOQI_U,7667
 dstack/_internal/cli/utils/rich.py,sha256=Gx1MJU929kMKsbdo9qF7XHARNta2426Ssb-xMLVhwbQ,5710
 dstack/_internal/cli/utils/run.py,sha256=-hZI0q09qnTeUFJF5zkJ4v3NLS2y2mCDrYHjplZs8RA,11789
 dstack/_internal/cli/utils/secrets.py,sha256=c7an9e8HwB24oiP1q225G1dhWcvCKw4aZtN0rN38_PE,654
 dstack/_internal/cli/utils/updates.py,sha256=pGr5keEmHojnbY0ayjWL4GWTz4ParY6fP85tHpJ11pI,3043
 dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14rHudNQN4,1975
 dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/core/consts.py,sha256=c1Yd5UY6Qx7KeuYgloXWncWhMsYj6TqwlElda7NtB98,254
+dstack/_internal/core/consts.py,sha256=gN6AKBhvR-o6unEH66Kg3Sehqd06TO7uxunSvThvqzY,373
 dstack/_internal/core/errors.py,sha256=VTEmJ6C0zWL3fUL8ObRjB4rNL7_NxwEuWf0hZs6orWo,3377
 dstack/_internal/core/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/configurators.py,sha256=Fy1FpiS_2aV8IWT1nkCb2CUI7fDVVRK-O2YFcHSpkTo,4573
@@ -59,21 +59,21 @@ dstack/_internal/core/backends/amddevcloud/configurator.py,sha256=WF4e-Yh2jMuTKQ
 dstack/_internal/core/backends/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/aws/auth.py,sha256=BAe3HAedoAO42Fbv7YhpODKWlyil8M0PuRJsL-_u_kw,978
 dstack/_internal/core/backends/aws/backend.py,sha256=pjROH-S9pgrSMm-Eox_ocL7cTU6mIMRxvURq7Vi-2J8,876
-dstack/_internal/core/backends/aws/compute.py,sha256=mxwNXg1sNjF1bGNxt42DfXj66bbEi-Yh92lKS6EMxkg,45224
+dstack/_internal/core/backends/aws/compute.py,sha256=wmeFCzgAfTZznpUOPlxAy4eGYhgVVixbqt2_q4RmQfU,45444
 dstack/_internal/core/backends/aws/configurator.py,sha256=X4CS-DnvsbdYCEVo6HBKLmH0YQ-zh5RucOjyhEcRca8,7634
 dstack/_internal/core/backends/aws/models.py,sha256=EUCHXHmZnshe3rwI9UtjilwcUMu1Z7MO4Y4-nlZ_IcA,4404
-dstack/_internal/core/backends/aws/resources.py,sha256=ccvdQd2H1DuUupdzMtqf86nkrenYPBJb1k3VgHu5Gp4,23610
+dstack/_internal/core/backends/aws/resources.py,sha256=vwfy2lz9E0RVl3Lhs8dsuWMhhJOjv1NDckqSNucs7f0,24025
 dstack/_internal/core/backends/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/azure/auth.py,sha256=CHp3QRNx3tIwkhyOY_l8WU-ElY4Pxhavoy9JSdwr15g,1259
 dstack/_internal/core/backends/azure/backend.py,sha256=XySTor8v_tLkZctDBryrFHrbVzQgNf_RUPkXQYplvwM,746
-dstack/_internal/core/backends/azure/compute.py,sha256=cbR0nSQfSIiR1x2J35So9xJbDkSjfYTEcYGQ-_1pceo,24695
+dstack/_internal/core/backends/azure/compute.py,sha256=tuMYTAti3RU58ueBnfvznfc66k55aniNNzWBsXmFEz4,25372
 dstack/_internal/core/backends/azure/configurator.py,sha256=tKovBgMvyKmCogZ2OMxOJ-Sh_TCBqYXBvbBM-kgyeBI,18367
 dstack/_internal/core/backends/azure/models.py,sha256=ap0WsYBSFdDCxY0G3W1sg2DBN2k0xknz6bp8Ypf6phs,3356
 dstack/_internal/core/backends/azure/resources.py,sha256=cJWS2Yjxdg-XtjpdBxfVDZaX_gTbuoEpmdjTxdA9uIw,3431
 dstack/_internal/core/backends/azure/utils.py,sha256=taHMJq6UHRzUXLUcO2P5VCKy3wJaye2bG-6QdkEPNdY,1741
 dstack/_internal/core/backends/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/base/backend.py,sha256=hdFMHED1RMV9GVfLSU0yGhGE-cXlbLvt1coDA885PMM,505
-dstack/_internal/core/backends/base/compute.py,sha256=bK4whPes7mCQ698Fu1XtgOUGOpiBhx9EnhTI9KkG_R0,33666
+dstack/_internal/core/backends/base/compute.py,sha256=XsyNrbTvHb8_DKDnM--mTiLXqOQW0pIdn3Ug-afRcP4,34358
 dstack/_internal/core/backends/base/configurator.py,sha256=TKG5iJlOXxVT2agE_ruMW8JGt2dtwXY7ghmJN8bEWlw,4299
 dstack/_internal/core/backends/base/models.py,sha256=Ij0osOl-T-ABsKLoVg2eY81DMkwdWkevAnjXj2QnLXI,532
 dstack/_internal/core/backends/base/offers.py,sha256=1688KlzBWgQWJFAEQMtqOud4Z8CwpvufNolX1zm1UTI,7487
@@ -109,7 +109,7 @@ dstack/_internal/core/backends/dstack/models.py,sha256=Jbe0Syez43N5i97TlIc8l-5Vn
 dstack/_internal/core/backends/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/gcp/auth.py,sha256=VWCn34hapaZ3E0OHCoiAVL97o3s64RF9w85CZ6xdtmk,2235
 dstack/_internal/core/backends/gcp/backend.py,sha256=OvTv1c7j4LTPCIEtkwD3-q6Eo1QET8xlM8CzbY0hLmI,572
-dstack/_internal/core/backends/gcp/compute.py,sha256=LycsCO9gCI1WEhsjUNnn-A4PrjciSU28BDsG3BEGNb4,43239
+dstack/_internal/core/backends/gcp/compute.py,sha256=0OByUucYMMzhLTvohvsVRV98fB8SOXmkBoPYG-NN0IU,43724
 dstack/_internal/core/backends/gcp/configurator.py,sha256=eVPr6pD4g66ej5qxhFd94lb_R11CIBA-py9PQS9EOGE,6865
 dstack/_internal/core/backends/gcp/models.py,sha256=biLA3rlFcoPatAZpKycuIl-8PdnNSAFiDCJjov65_zo,4612
 dstack/_internal/core/backends/gcp/resources.py,sha256=2jpg24NH4Ov4J4Of1F3DPkwEWkRxEi-x4ZTx8XSNGIQ,16926
@@ -146,12 +146,12 @@ dstack/_internal/core/backends/nebius/resources.py,sha256=ttgwdqokvXF8BH_IDPFZxW
 dstack/_internal/core/backends/oci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/oci/auth.py,sha256=8Cr18y_LOsyRP-16yfFpT70Cofpm0clB3KawS_7aRl4,717
 dstack/_internal/core/backends/oci/backend.py,sha256=yXjVCt7n6BVLH0byYFbNFf-P9J0FwlNfxsYbKGMdoI4,536
-dstack/_internal/core/backends/oci/compute.py,sha256=jB3A6s4Hxh-LYrqQR8wJYqYCxLt31cbeMxrEkBxRfys,7605
+dstack/_internal/core/backends/oci/compute.py,sha256=njttE6pwrSq5itqUmpzfKjl1NFQ4Qgr5SAOMO4gEu_c,7733
 dstack/_internal/core/backends/oci/configurator.py,sha256=_mi093BzV8Xdfw81wkiqoY5d5x4CCCtni7J01CZeDCQ,5768
 dstack/_internal/core/backends/oci/exceptions.py,sha256=GxlUf7itcAQgY3m-U2ul_ZXzfUOcEcPEmisWFaoWSkA,288
 dstack/_internal/core/backends/oci/models.py,sha256=fDlY7-A0WDQeiwJzwrQl6D5ENOc-677Bz8LXyvBvF0c,2969
 dstack/_internal/core/backends/oci/region.py,sha256=a41dmlv-7jh2vP2z-0wnf0Yzdh2Donjm9aXuMgFXXKo,2935
-dstack/_internal/core/backends/oci/resources.py,sha256=VfTDuVTEO8MUvD3i2DFoe2du1qP4MSQb1yEaDgHbUp8,28982
+dstack/_internal/core/backends/oci/resources.py,sha256=-47u2vWvRnpNQmgz3A12vu1hSIQ-dqawDl5ndyF0nwY,29392
 dstack/_internal/core/backends/runpod/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/runpod/api_client.py,sha256=Oet7CwQTu3sE57unqh-JvuMCBDEDPruAbGB4GyM0dM4,13427
 dstack/_internal/core/backends/runpod/backend.py,sha256=GcLbySsGY29ZGHXSqTXCxUUoTK9pxVScKly0_fkNr8c,566
@@ -219,7 +219,7 @@ dstack/_internal/core/services/api_client.py,sha256=HTQ0fcZciUh-nmfV09hUt__Z4N2z
 dstack/_internal/core/services/diff.py,sha256=xtJzlSLIw9pnECpMQTy-a2zcfnSd4Q9qi_Y0W00D65A,2073
 dstack/_internal/core/services/logs.py,sha256=7_eJdH4MD-3rVb4A6rIJfjj_p4jzUOCmjRVlPD-UDsg,2166
 dstack/_internal/core/services/profiles.py,sha256=ZFY6VILW6urm6l3u6xpORJ2z48eGJ2jcmRgjVrxr2tw,1612
-dstack/_internal/core/services/repos.py,sha256=Cq-t9RvdmEKjQIgQA6giQirqopdrPT-WzUoCoKLt0ws,6251
+dstack/_internal/core/services/repos.py,sha256=Lw-4MXFBaE_BogRgtYra1H3KPkRGeVWoIfU8hTyJ7GM,10552
 dstack/_internal/core/services/configs/__init__.py,sha256=jMAHvZphEgZJldEiQhkaNSW8lVrVAxuYbu9RtLysUGU,5645
 dstack/_internal/core/services/ssh/__init__.py,sha256=UhgC3Lv3CPSGqSPEQZIKOfLKUlCFnaB0uqPQhfKCFt0,878
 dstack/_internal/core/services/ssh/attach.py,sha256=6YZ_HaimKrjdLzcDLksGi0_9rlVyLeBgPmmWHC7dYxo,9096
@@ -297,9 +297,9 @@ dstack/_internal/server/background/tasks/process_metrics.py,sha256=yKXe9J7m3oleK
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=lgYIzjHG9EITK31yG6uQjlIcSwW5jsP9ZOBBZqW_eNs,4263
 dstack/_internal/server/background/tasks/process_probes.py,sha256=dmug-_rmYiVLLF-imto-Ju1gPtENvHvCjHyilqgYuJw,6457
 dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=_UZm37FVV4rhdd0So7HtcKbIgrSdAr5Vx-Uen_xizec,5459
-dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=ulDrWgwnhosbK0Mtt0yNS4E38QSKXSh6ermNt-3jUTs,44965
+dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=IoQi7mm4upEZgujTkWYrXDKrC5rSZ5Q4_jAR4OpajaM,44973
 dstack/_internal/server/background/tasks/process_runs.py,sha256=Cx7Z1B7pZVlvCl-OsIaAiIMFG_aZDdn3nlZeha6k2x4,25041
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=vcu1JRJ0-p_VB7HZfYkVf9d_yclazrcW1U7zHQltjcA,40935
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=XxPapMdCsuA_H_X27SIwIZFd0Y5jzwvIABnhqa-kwyQ,41098
 dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=S7ZSDVMX-N0XMaMgwFa1QG_RAi48BP432s9AqHw4PMM,4066
 dstack/_internal/server/background/tasks/process_volumes.py,sha256=_fMmkwLYsyX-kpW9pDrZVJvFTZEOPp0gpjyKBMW-zw0,5204
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -628,7 +628,7 @@ dstack/_internal/utils/nested_list.py,sha256=ba4fmMuaz807Bf6KRve-9_8zm4dxlNMQiFb
 dstack/_internal/utils/network.py,sha256=vvIQulBapL6Zw50jsit7sNNbGJVKTzooqWhkVWqaAzQ,1488
 dstack/_internal/utils/path.py,sha256=FSKOBLDZdT64KJ-lqpRMoo6YzqR2jwCgdEmcodFQz-E,1342
 dstack/_internal/utils/random_names.py,sha256=lVyzAmXhaSf2bAuvRFQ_ihLy42in1ydN4gJYsMw4GG8,3537
-dstack/_internal/utils/ssh.py,sha256=FxOA85-NTYEGhnW_PL3-uVYGZbVZfyvcA8uvD7BcN2Q,12131
+dstack/_internal/utils/ssh.py,sha256=SE10JKbnUyocoAhMs0dsLIUDOQmUtQGArZEUAFIXBPQ,12852
 dstack/_internal/utils/tags.py,sha256=Gx4aZM85E_UUCGuFZbCEFxHB5DNgrtJP15_1tFFEd1c,1292
 dstack/_internal/utils/typing.py,sha256=tHrhtkr3G6jkk-_FZyobdjLYvP5_YIgXbKsN-13dfvg,386
 dstack/api/__init__.py,sha256=7qbG42qEHMyNDFRCz9FgfuimAKws3hV8eMXs6oRgh5c,1607
@@ -662,8 +662,8 @@ dstack/plugins/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 dstack/plugins/builtin/rest_plugin/__init__.py,sha256=lgTsq8Z6Km2F2UhPRChVB4vDM5ZpWtdk1iB1aa20ypA,440
 dstack/plugins/builtin/rest_plugin/_models.py,sha256=9hgVuU6OGSxidar88XhQnNo9izYWeQvVH45ciErv-Es,1910
 dstack/plugins/builtin/rest_plugin/_plugin.py,sha256=h3r3Yc3h22i93fifPTgTm9Oojd1sN1O4DP7ZTV-kWpM,5386
-dstack-0.19.29.dist-info/METADATA,sha256=HcciEQov78Oe3HcLD7vquMFJHimJXPOx_AANTEW33gM,20586
-dstack-0.19.29.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dstack-0.19.29.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.29.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.29.dist-info/RECORD,,
+dstack-0.19.30.dist-info/METADATA,sha256=1Eek1ucesAzIWosGAWsdivcQu0_mV0FKW00DVi6R4nk,20919
+dstack-0.19.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dstack-0.19.30.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.30.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.30.dist-info/RECORD,,