dstack 0.19.30rc1__py3-none-any.whl → 0.19.31__py3-none-any.whl

dstack/_internal/core/backends/kubernetes/models.py

@@ -5,12 +5,14 @@ from pydantic import Field, root_validator
 from dstack._internal.core.backends.base.models import fill_data
 from dstack._internal.core.models.common import CoreModel
 
+DEFAULT_NAMESPACE = "default"
 
-class KubernetesNetworkingConfig(CoreModel):
-    ssh_host: Annotated[
-        Optional[str], Field(description="The external IP address of any node")
+
+class KubernetesProxyJumpConfig(CoreModel):
+    hostname: Annotated[
+        Optional[str], Field(description="The external IP address or hostname of any node")
     ] = None
-    ssh_port: Annotated[
+    port: Annotated[
         Optional[int], Field(description="Any port accessible outside of the cluster")
     ] = None
 
@@ -22,16 +24,15 @@ class KubeconfigConfig(CoreModel):
 
 class KubernetesBackendConfig(CoreModel):
     type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
+    proxy_jump: Annotated[
+        Optional[KubernetesProxyJumpConfig], Field(description="The SSH proxy jump configuration")
     ] = None
+    namespace: Annotated[
+        str, Field(description="The namespace for resources managed by `dstack`")
+    ] = DEFAULT_NAMESPACE
 
 
-class KubernetesBackendConfigWithCreds(CoreModel):
-    type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ] = None
+class KubernetesBackendConfigWithCreds(KubernetesBackendConfig):
     kubeconfig: Annotated[KubeconfigConfig, Field(description="The kubeconfig configuration")]
 
 
@@ -53,11 +54,7 @@ class KubeconfigFileConfig(CoreModel):
         return fill_data(values)
 
 
-class KubernetesBackendFileConfigWithCreds(CoreModel):
-    type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ] = None
+class KubernetesBackendFileConfigWithCreds(KubernetesBackendConfig):
     kubeconfig: Annotated[KubeconfigFileConfig, Field(description="The kubeconfig configuration")]
 
 

dstack/_internal/core/backends/kubernetes/utils.py

@@ -1,20 +1,157 @@
-from typing import Dict, List, Optional
+import ast
+from typing import Any, Callable, List, Literal, Optional, TypeVar, Union, get_origin, overload
 
-import kubernetes
 import yaml
+from kubernetes import client as kubernetes_client
+from kubernetes import config as kubernetes_config
+from typing_extensions import ParamSpec
 
+T = TypeVar("T")
+P = ParamSpec("P")
 
-def get_api_from_config_data(kubeconfig_data: str) -> kubernetes.client.CoreV1Api:
+
+def get_api_from_config_data(kubeconfig_data: str) -> kubernetes_client.CoreV1Api:
     config_dict = yaml.load(kubeconfig_data, yaml.FullLoader)
     return get_api_from_config_dict(config_dict)
 
 
-def get_api_from_config_dict(kubeconfig: Dict) -> kubernetes.client.CoreV1Api:
-    api_client = kubernetes.config.new_client_from_config_dict(config_dict=kubeconfig)
-    return kubernetes.client.CoreV1Api(api_client=api_client)
+def get_api_from_config_dict(kubeconfig: dict) -> kubernetes_client.CoreV1Api:
+    api_client = kubernetes_config.new_client_from_config_dict(config_dict=kubeconfig)
+    return kubernetes_client.CoreV1Api(api_client=api_client)
+
+
+@overload
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: None = None,
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> T: ...
+
+
+@overload
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: Union[int, tuple[int, ...], list[int]],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> Optional[T]: ...
+
+
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: Optional[Union[int, tuple[int, ...], list[int]]] = None,
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> Optional[T]:
+    """
+    Returns the result of the API method call, optionally ignoring specified HTTP status codes.
+
+    Args:
+        method: the `CoreV1Api` bound method.
+        type_: The expected type of the return value, used for runtime type checking and
+            as a type hint for a static type checker (as kubernetes package is not type-annotated).
+            NB: For composite types, only "origin" type is checked, e.g., list, not list[Node]
+        expected: Expected error statuses, e.g., 404.
+        args: positional arguments of the method.
+        kwargs: keyword arguments of the method.
+    Returns:
+        The return value or `None` in case of the expected error.
+    """
+    if isinstance(expected, int):
+        expected = (expected,)
+    result: T
+    try:
+        result = method(*args, **kwargs)
+    except kubernetes_client.ApiException as e:
+        if expected is None or e.status not in expected:
+            raise
+        return None
+    if not isinstance(result, get_origin(type_) or type_):
+        raise TypeError(
+            f"{method.__name__} returned {type(result).__name__}, expected {type_.__name__}"
+        )
+    return result
+
+
+@overload
+def get_value(
+    obj: object, path: str, type_: type[T], *, required: Literal[False] = False
+) -> Optional[T]: ...
+
+
+@overload
+def get_value(obj: object, path: str, type_: type[T], *, required: Literal[True]) -> T: ...
+
+
+def get_value(obj: object, path: str, type_: type[T], *, required: bool = False) -> Optional[T]:
+    """
+    Returns the value at a given path.
+    Supports object attributes, sequence indices, and mapping keys.
+
+    Args:
+        obj: The object to traverse.
+        path: The path to the value, regular Python syntax. The leading dot is optional, all the
+            following are correct: `.attr`, `attr`, `.[0]`, `[0]`, `.['key']`, `['key']`.
+        type_: The expected type of the value, used for runtime type checking and as a type hint
+            for a static type checker (as kubernetes package is not type-annotated).
+            NB: For composite types, only "origin" type is checked, e.g., list, not list[Node]
+        required: If `True`, the value must exist and must not be `None`. If `False` (safe
+            navigation mode), the may not exist and may be `None`.
+
+    Returns:
+        The requested value or `None` in case of failed traverse when required=False.
+    """
+    _path = path.removeprefix(".")
+    if _path.startswith("["):
+        src = f"obj{_path}"
+    else:
+        src = f"obj.{_path}"
+    module = ast.parse(src)
+    assert len(module.body) == 1, ast.dump(module, indent=4)
+    root_expr = module.body[0]
+    assert isinstance(root_expr, ast.Expr), ast.dump(module, indent=4)
+    varname: Optional[str] = None
+    expr = root_expr.value
+    while True:
+        if isinstance(expr, ast.Name):
+            varname = expr.id
+            break
+        if __debug__:
+            if isinstance(expr, ast.Subscript):
+                if isinstance(expr.slice, ast.UnaryOp):
+                    # .items[-1]
+                    assert isinstance(expr.slice.op, ast.USub), ast.dump(expr, indent=4)
+                    assert isinstance(expr.slice.operand, ast.Constant), ast.dump(expr, indent=4)
+                    assert isinstance(expr.slice.operand.value, int), ast.dump(expr, indent=4)
+                else:
+                    # .items[0], .labels["name"]
+                    assert isinstance(expr.slice, ast.Constant), ast.dump(expr, indent=4)
+            else:
+                assert isinstance(expr, ast.Attribute), ast.dump(expr, indent=4)
+        else:
+            assert isinstance(expr, (ast.Attribute, ast.Subscript))
+        expr = expr.value
+    assert varname is not None, ast.dump(module)
+    try:
+        value = eval(src, {"__builtins__": {}}, {"obj": obj})
+    except (AttributeError, KeyError, IndexError, TypeError) as e:
+        if required:
+            raise type(e)(f"Failed to traverse {path}: {e}") from e
+        return None
+    if value is None:
+        if required:
+            raise TypeError(f"Required {path} is None")
+        return value
+    if not isinstance(value, get_origin(type_) or type_):
+        raise TypeError(f"{path} value is {type(value).__name__}, expected {type_.__name__}")
+    return value
 
 
-def get_cluster_public_ip(api_client: kubernetes.client.CoreV1Api) -> Optional[str]:
+def get_cluster_public_ip(api_client: kubernetes_client.CoreV1Api) -> Optional[str]:
     """
     Returns public IP of any cluster node.
     """
@@ -24,7 +161,7 @@ def get_cluster_public_ip(api_client: kubernetes.client.CoreV1Api) -> Optional[s
     return public_ips[0]
 
 
-def get_cluster_public_ips(api_client: kubernetes.client.CoreV1Api) -> List[str]:
+def get_cluster_public_ips(api_client: kubernetes_client.CoreV1Api) -> List[str]:
     """
     Returns public IPs of all cluster nodes.
     """

dstack/_internal/core/backends/lambdalabs/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_shim_commands,
 )
@@ -31,6 +32,7 @@ MAX_INSTANCE_NAME_LEN = 60
 class LambdaCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     Compute,
 ):
     def __init__(self, config: LambdaConfig):

dstack/_internal/core/backends/local/compute.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithVolumeSupport,
 )
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
@@ -25,6 +26,7 @@ logger = get_logger(__name__)
 
 class LocalCompute(
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithVolumeSupport,
     Compute,
 ):

dstack/_internal/core/backends/nebius/compute.py

@@ -16,6 +16,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
 )
@@ -79,6 +80,7 @@ SUPPORTED_PLATFORMS = [
 class NebiusCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     Compute,

dstack/_internal/core/backends/oci/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
 )
@@ -50,6 +51,7 @@ CONFIGURABLE_DISK_SIZE = Range[Memory](min=Memory.parse("50GB"), max=Memory.pars
 class OCICompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     Compute,
 ):
@@ -118,7 +120,11 @@ class OCICompute(
         availability_domain = instance_offer.availability_zones[0]
 
         listing, package = resources.get_marketplace_listing_and_package(
-            cuda=len(instance_offer.instance.resources.gpus) > 0,
+            gpu_name=(
+                instance_offer.instance.resources.gpus[0].name
+                if len(instance_offer.instance.resources.gpus) > 0
+                else None
+            ),
             client=region.marketplace_client,
         )
         resources.accept_marketplace_listing_agreements(

dstack/_internal/core/backends/oci/resources.py

@@ -23,7 +23,9 @@ import oci
 from oci.object_storage.models import CreatePreauthenticatedRequestDetails
 
 from dstack import version
+from dstack._internal.core.backends.base.compute import requires_nvidia_proprietary_kernel_modules
 from dstack._internal.core.backends.oci.region import OCIRegionClient
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import BackendError
 from dstack._internal.core.models.instances import InstanceOffer
 from dstack._internal.utils.common import batched
@@ -352,11 +354,14 @@ def terminate_instance_if_exists(client: oci.core.ComputeClient, instance_id: st
 
 
 def get_marketplace_listing_and_package(
-    cuda: bool, client: oci.marketplace.MarketplaceClient
+    gpu_name: Optional[str], client: oci.marketplace.MarketplaceClient
 ) -> Tuple[oci.marketplace.models.Listing, oci.marketplace.models.ImageListingPackage]:
     listing_name = f"dstack-{version.base_image}"
-    if cuda:
-        listing_name = f"dstack-cuda-{version.base_image}"
+    if gpu_name is not None:
+        if not requires_nvidia_proprietary_kernel_modules(gpu_name):
+            listing_name = f"dstack-cuda-{version.base_image}"
+        else:
+            listing_name = f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
 
     listing_summaries = list_marketplace_listings(listing_name, client)
     if len(listing_summaries) != 1:

dstack/_internal/core/backends/template/compute.py.jinja

@@ -8,6 +8,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivateGatewaySupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithReservationSupport,
     ComputeWithVolumeSupport,
 )
@@ -31,6 +32,7 @@ class {{ backend_name }}Compute(
     # TODO: Choose ComputeWith* classes to extend and implement
     # ComputeWithAllOffersCached,
     # ComputeWithCreateInstanceSupport,
+    # ComputeWithPrivilegedSupport,
     # ComputeWithMultinodeSupport,
     # ComputeWithReservationSupport,
     # ComputeWithPlacementGroupSupport,

dstack/_internal/core/backends/tensordock/compute.py

@@ -6,6 +6,7 @@ import requests
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_shim_commands,
 )
@@ -32,6 +33,7 @@ MAX_INSTANCE_NAME_LEN = 60
 
 class TensorDockCompute(
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     Compute,
 ):
     def __init__(self, config: TensorDockConfig):

dstack/_internal/core/backends/vultr/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
 )
@@ -35,6 +36,7 @@ MAX_INSTANCE_NAME_LEN = 64
 class VultrCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     Compute,
 ):

dstack/_internal/core/consts.py

@@ -4,3 +4,5 @@ DSTACK_SHIM_HTTP_PORT = 10998
 DSTACK_RUNNER_HTTP_PORT = 10999
 # ssh server (runs alongside the runner inside a container) listen port
 DSTACK_RUNNER_SSH_PORT = 10022
+# legacy AWS, Azure, GCP, and OCI image for older GPUs
+DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES = "0.10"

dstack/_internal/core/services/repos.py

@@ -36,24 +36,59 @@ def get_repo_creds_and_default_branch(
 
     # no auth
     with suppress(InvalidRepoCredentialsError):
-        return _get_repo_creds_and_default_branch_https(url)
+        creds, default_branch = _get_repo_creds_and_default_branch_https(url)
+        logger.debug(
+            "Git repo %s is public. Using no auth. Default branch: %s", repo_url, default_branch
+        )
+        return creds, default_branch
 
     # ssh key provided by the user or pulled from the server
     if identity_file is not None or private_key is not None:
         if identity_file is not None:
             private_key = _read_private_key(identity_file)
-            return _get_repo_creds_and_default_branch_ssh(url, identity_file, private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, identity_file, private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using identity file: %s. Default branch: %s",
+                repo_url,
+                identity_file,
+                default_branch,
+            )
+            return creds, default_branch
         elif private_key is not None:
             with NamedTemporaryFile("w+", 0o600) as f:
                 f.write(private_key)
                 f.flush()
-                return _get_repo_creds_and_default_branch_ssh(url, f.name, private_key)
+                creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                    url, f.name, private_key
+                )
+                masked_key = "***" + private_key[-10:] if len(private_key) > 10 else "***MASKED***"
+                logger.debug(
+                    "Git repo %s is private. Using private key: %s. Default branch: %s",
+                    repo_url,
+                    masked_key,
+                    default_branch,
+                )
+                return creds, default_branch
         else:
             assert False, "should not reach here"
 
     # oauth token provided by the user or pulled from the server
     if oauth_token is not None:
-        return _get_repo_creds_and_default_branch_https(url, oauth_token)
+        creds, default_branch = _get_repo_creds_and_default_branch_https(url, oauth_token)
+        masked_token = (
+            len(oauth_token[:-4]) * "*" + oauth_token[-4:]
+            if len(oauth_token) > 4
+            else "***MASKED***"
+        )
+        logger.debug(
+            "Git repo %s is private. Using provided OAuth token: %s. Default branch: %s",
+            repo_url,
+            masked_token,
+            default_branch,
+        )
+        return creds, default_branch
 
     # key from ssh config
     identities = get_host_config(url.original_host).get("identityfile")
@@ -61,7 +96,16 @@ def get_repo_creds_and_default_branch(
         _identity_file = identities[0]
         with suppress(InvalidRepoCredentialsError):
             _private_key = _read_private_key(_identity_file)
-            return _get_repo_creds_and_default_branch_ssh(url, _identity_file, _private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, _identity_file, _private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using SSH config identity file: %s. Default branch: %s",
+                repo_url,
+                _identity_file,
+                default_branch,
+            )
+            return creds, default_branch
 
     # token from gh config
     if os.path.exists(gh_config_path):
@@ -70,13 +114,35 @@ def get_repo_creds_and_default_branch(
         _oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
         if _oauth_token is not None:
             with suppress(InvalidRepoCredentialsError):
-                return _get_repo_creds_and_default_branch_https(url, _oauth_token)
+                creds, default_branch = _get_repo_creds_and_default_branch_https(url, _oauth_token)
+                masked_token = (
+                    len(_oauth_token[:-4]) * "*" + _oauth_token[-4:]
+                    if len(_oauth_token) > 4
+                    else "***MASKED***"
+                )
+                logger.debug(
+                    "Git repo %s is private. Using GitHub config token: %s from %s. Default branch: %s",
+                    repo_url,
+                    masked_token,
+                    gh_config_path,
+                    default_branch,
+                )
+                return creds, default_branch
 
     # default user key
     if os.path.exists(default_ssh_key):
         with suppress(InvalidRepoCredentialsError):
             _private_key = _read_private_key(default_ssh_key)
-            return _get_repo_creds_and_default_branch_ssh(url, default_ssh_key, _private_key)
+            creds, default_branch = _get_repo_creds_and_default_branch_ssh(
+                url, default_ssh_key, _private_key
+            )
+            logger.debug(
+                "Git repo %s is private. Using default identity file: %s. Default branch: %s",
+                repo_url,
+                default_ssh_key,
+                default_branch,
+            )
+            return creds, default_branch
 
     raise InvalidRepoCredentialsError(
         "No valid default Git credentials found. Pass valid `--token` or `--git-identity`."
@@ -87,8 +153,9 @@ def _get_repo_creds_and_default_branch_ssh(
     url: GitRepoURL, identity_file: PathLike, private_key: str
 ) -> tuple[RemoteRepoCreds, Optional[str]]:
     _url = url.as_ssh()
+    env = _make_git_env_for_creds_check(identity_file=identity_file)
     try:
-        default_branch = _get_repo_default_branch(_url, make_git_env(identity_file=identity_file))
+        default_branch = _get_repo_default_branch(_url, env)
     except GitCommandError as e:
         message = f"Cannot access `{_url}` using the `{identity_file}` private SSH key"
         raise InvalidRepoCredentialsError(message) from e
@@ -104,8 +171,9 @@ def _get_repo_creds_and_default_branch_https(
     url: GitRepoURL, oauth_token: Optional[str] = None
 ) -> tuple[RemoteRepoCreds, Optional[str]]:
     _url = url.as_https()
+    env = _make_git_env_for_creds_check()
     try:
-        default_branch = _get_repo_default_branch(url.as_https(oauth_token), make_git_env())
+        default_branch = _get_repo_default_branch(url.as_https(oauth_token), env)
     except GitCommandError as e:
         message = f"Cannot access `{_url}`"
         if oauth_token is not None:
@@ -120,10 +188,32 @@ def _get_repo_creds_and_default_branch_https(
     return creds, default_branch
 
 
+def _make_git_env_for_creds_check(identity_file: Optional[PathLike] = None) -> dict[str, str]:
+    # Our goal is to check if _provided_ creds (if any) are correct, so we need to be sure that
+    # only the provided creds are used, without falling back to any additional mechanisms.
+    # To do this, we:
+    # 1. Disable all configs to ignore any stored creds
+    # 2. Disable askpass to avoid asking for creds interactively or fetching stored creds from
+    # a non-interactive askpass helper (for example, VS Code sets GIT_ASKPASS to its own helper,
+    # which silently provides creds to Git).
+    return make_git_env(disable_config=True, disable_askpass=True, identity_file=identity_file)
+
+
 def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]:
+    # Git shipped by Apple with XCode is patched to support an additional config scope
+    # above "system" called "xcode". There is no option in `git config list` to show this config,
+    # but you can list the merged config (`git config list` without options) and then exclude
+    # all settings listed in `git config list --{system,global,local,worktree}`.
+    # As of time of writing, there are only two settings in the "xcode" config, one of which breaks
+    # our "is repo public?" check, namely "credential.helper=osxkeychain".
+    # As there is no way to disable "xcode" config (no env variable, no CLI option, etc.),
+    # the only way to disable credential helper is to override this specific setting with an empty
+    # string via command line argument: `git -c credential.helper= COMMAND [ARGS ...]`.
+    # See: https://github.com/git/git/commit/3d4355712b9fe77a96ad4ad877d92dc7ff6e0874
+    # See: https://gist.github.com/ChrisTollefson/ab9c0a5d1dd4dd615217345c6936a307
+    _git = git.cmd.Git()(c="credential.helper=")
     # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD"
-    # Disable credential helpers to exclude any default credentials from being used
-    output: str = git.cmd.Git()(c="credential.helper=").ls_remote("--symref", url, "HEAD", env=env)
+    output: str = _git.ls_remote("--symref", url, "HEAD", env=env)
     for line in output.splitlines():
         # line format: `<oid> TAB <ref> LF`
         oid, _, ref = line.partition("\t")

dstack/_internal/server/background/tasks/common.py

@@ -19,4 +19,6 @@ def get_provisioning_timeout(backend_type: BackendType, instance_type_name: str)
         return timedelta(minutes=20)
     if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
         return timedelta(minutes=55)
+    if backend_type == BackendType.GCP and instance_type_name == "a4-highgpu-8g":
+        return timedelta(minutes=16)
     return timedelta(minutes=10)

dstack/_internal/server/background/tasks/process_instances.py

@@ -307,7 +307,7 @@ async def _add_remote(instance: InstanceModel) -> None:
             )
             deploy_timeout = 20 * 60  # 20 minutes
             result = await asyncio.wait_for(future, timeout=deploy_timeout)
-            health, host_info, cpu_arch = result
+            health, host_info, arch = result
         except (asyncio.TimeoutError, TimeoutError) as e:
             raise ProvisioningError(f"Deploy timeout: {e}") from e
         except Exception as e:
@@ -327,7 +327,7 @@ async def _add_remote(instance: InstanceModel) -> None:
         instance.status = InstanceStatus.PENDING
         return
 
-    instance_type = host_info_to_instance_type(host_info, cpu_arch)
+    instance_type = host_info_to_instance_type(host_info, arch)
     instance_network = None
     internal_ip = None
     try:

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -1139,7 +1139,7 @@ def _patch_base_image_for_aws_efa(
     efa_enabled_patterns = [
         # TODO: p6-b200 isn't supported yet in gpuhunt
         r"^p6-b200\.(48xlarge)$",
-        r"^p5\.(48xlarge)$",
+        r"^p5\.(4xlarge|48xlarge)$",
         r"^p5e\.(48xlarge)$",
         r"^p5en\.(48xlarge)$",
         r"^p4d\.(24xlarge)$",