dstack 0.19.30rc1__py3-none-any.whl → 0.19.32__py3-none-any.whl

dstack/_internal/core/backends/kubernetes/models.py

@@ -5,12 +5,14 @@ from pydantic import Field, root_validator
 from dstack._internal.core.backends.base.models import fill_data
 from dstack._internal.core.models.common import CoreModel
 
+DEFAULT_NAMESPACE = "default"
 
-class KubernetesNetworkingConfig(CoreModel):
-    ssh_host: Annotated[
-        Optional[str], Field(description="The external IP address of any node")
+
+class KubernetesProxyJumpConfig(CoreModel):
+    hostname: Annotated[
+        Optional[str], Field(description="The external IP address or hostname of any node")
     ] = None
-    ssh_port: Annotated[
+    port: Annotated[
         Optional[int], Field(description="Any port accessible outside of the cluster")
     ] = None
 
@@ -22,16 +24,15 @@ class KubeconfigConfig(CoreModel):
 
 class KubernetesBackendConfig(CoreModel):
     type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
+    proxy_jump: Annotated[
+        Optional[KubernetesProxyJumpConfig], Field(description="The SSH proxy jump configuration")
     ] = None
+    namespace: Annotated[
+        str, Field(description="The namespace for resources managed by `dstack`")
+    ] = DEFAULT_NAMESPACE
 
 
-class KubernetesBackendConfigWithCreds(CoreModel):
-    type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ] = None
+class KubernetesBackendConfigWithCreds(KubernetesBackendConfig):
     kubeconfig: Annotated[KubeconfigConfig, Field(description="The kubeconfig configuration")]
 
 
@@ -53,11 +54,7 @@ class KubeconfigFileConfig(CoreModel):
         return fill_data(values)
 
 
-class KubernetesBackendFileConfigWithCreds(CoreModel):
-    type: Annotated[Literal["kubernetes"], Field(description="The type of backend")] = "kubernetes"
-    networking: Annotated[
-        Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ] = None
+class KubernetesBackendFileConfigWithCreds(KubernetesBackendConfig):
     kubeconfig: Annotated[KubeconfigFileConfig, Field(description="The kubeconfig configuration")]
 
 

dstack/_internal/core/backends/kubernetes/utils.py

@@ -1,20 +1,157 @@
-from typing import Dict, List, Optional
+import ast
+from typing import Any, Callable, List, Literal, Optional, TypeVar, Union, get_origin, overload
 
-import kubernetes
 import yaml
+from kubernetes import client as kubernetes_client
+from kubernetes import config as kubernetes_config
+from typing_extensions import ParamSpec
 
+T = TypeVar("T")
+P = ParamSpec("P")
 
-def get_api_from_config_data(kubeconfig_data: str) -> kubernetes.client.CoreV1Api:
+
+def get_api_from_config_data(kubeconfig_data: str) -> kubernetes_client.CoreV1Api:
     config_dict = yaml.load(kubeconfig_data, yaml.FullLoader)
     return get_api_from_config_dict(config_dict)
 
 
-def get_api_from_config_dict(kubeconfig: Dict) -> kubernetes.client.CoreV1Api:
-    api_client = kubernetes.config.new_client_from_config_dict(config_dict=kubeconfig)
-    return kubernetes.client.CoreV1Api(api_client=api_client)
+def get_api_from_config_dict(kubeconfig: dict) -> kubernetes_client.CoreV1Api:
+    api_client = kubernetes_config.new_client_from_config_dict(config_dict=kubeconfig)
+    return kubernetes_client.CoreV1Api(api_client=api_client)
+
+
+@overload
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: None = None,
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> T: ...
+
+
+@overload
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: Union[int, tuple[int, ...], list[int]],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> Optional[T]: ...
+
+
+def call_api_method(
+    method: Callable[P, Any],
+    type_: type[T],
+    expected: Optional[Union[int, tuple[int, ...], list[int]]] = None,
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> Optional[T]:
+    """
+    Returns the result of the API method call, optionally ignoring specified HTTP status codes.
+
+    Args:
+        method: the `CoreV1Api` bound method.
+        type_: The expected type of the return value, used for runtime type checking and
+            as a type hint for a static type checker (as kubernetes package is not type-annotated).
+            NB: For composite types, only "origin" type is checked, e.g., list, not list[Node]
+        expected: Expected error statuses, e.g., 404.
+        args: positional arguments of the method.
+        kwargs: keyword arguments of the method.
+    Returns:
+        The return value or `None` in case of the expected error.
+    """
+    if isinstance(expected, int):
+        expected = (expected,)
+    result: T
+    try:
+        result = method(*args, **kwargs)
+    except kubernetes_client.ApiException as e:
+        if expected is None or e.status not in expected:
+            raise
+        return None
+    if not isinstance(result, get_origin(type_) or type_):
+        raise TypeError(
+            f"{method.__name__} returned {type(result).__name__}, expected {type_.__name__}"
+        )
+    return result
+
+
+@overload
+def get_value(
+    obj: object, path: str, type_: type[T], *, required: Literal[False] = False
+) -> Optional[T]: ...
+
+
+@overload
+def get_value(obj: object, path: str, type_: type[T], *, required: Literal[True]) -> T: ...
+
+
+def get_value(obj: object, path: str, type_: type[T], *, required: bool = False) -> Optional[T]:
+    """
+    Returns the value at a given path.
+    Supports object attributes, sequence indices, and mapping keys.
+
+    Args:
+        obj: The object to traverse.
+        path: The path to the value, regular Python syntax. The leading dot is optional, all the
+            following are correct: `.attr`, `attr`, `.[0]`, `[0]`, `.['key']`, `['key']`.
+        type_: The expected type of the value, used for runtime type checking and as a type hint
+            for a static type checker (as kubernetes package is not type-annotated).
+            NB: For composite types, only "origin" type is checked, e.g., list, not list[Node]
+        required: If `True`, the value must exist and must not be `None`. If `False` (safe
+            navigation mode), the may not exist and may be `None`.
+
+    Returns:
+        The requested value or `None` in case of failed traverse when required=False.
+    """
+    _path = path.removeprefix(".")
+    if _path.startswith("["):
+        src = f"obj{_path}"
+    else:
+        src = f"obj.{_path}"
+    module = ast.parse(src)
+    assert len(module.body) == 1, ast.dump(module, indent=4)
+    root_expr = module.body[0]
+    assert isinstance(root_expr, ast.Expr), ast.dump(module, indent=4)
+    varname: Optional[str] = None
+    expr = root_expr.value
+    while True:
+        if isinstance(expr, ast.Name):
+            varname = expr.id
+            break
+        if __debug__:
+            if isinstance(expr, ast.Subscript):
+                if isinstance(expr.slice, ast.UnaryOp):
+                    # .items[-1]
+                    assert isinstance(expr.slice.op, ast.USub), ast.dump(expr, indent=4)
+                    assert isinstance(expr.slice.operand, ast.Constant), ast.dump(expr, indent=4)
+                    assert isinstance(expr.slice.operand.value, int), ast.dump(expr, indent=4)
+                else:
+                    # .items[0], .labels["name"]
+                    assert isinstance(expr.slice, ast.Constant), ast.dump(expr, indent=4)
+            else:
+                assert isinstance(expr, ast.Attribute), ast.dump(expr, indent=4)
+        else:
+            assert isinstance(expr, (ast.Attribute, ast.Subscript))
+        expr = expr.value
+    assert varname is not None, ast.dump(module)
+    try:
+        value = eval(src, {"__builtins__": {}}, {"obj": obj})
+    except (AttributeError, KeyError, IndexError, TypeError) as e:
+        if required:
+            raise type(e)(f"Failed to traverse {path}: {e}") from e
+        return None
+    if value is None:
+        if required:
+            raise TypeError(f"Required {path} is None")
+        return value
+    if not isinstance(value, get_origin(type_) or type_):
+        raise TypeError(f"{path} value is {type(value).__name__}, expected {type_.__name__}")
+    return value
 
 
-def get_cluster_public_ip(api_client: kubernetes.client.CoreV1Api) -> Optional[str]:
+def get_cluster_public_ip(api_client: kubernetes_client.CoreV1Api) -> Optional[str]:
     """
     Returns public IP of any cluster node.
     """
@@ -24,7 +161,7 @@ def get_cluster_public_ip(api_client: kubernetes.client.CoreV1Api) -> Optional[s
     return public_ips[0]
 
 
-def get_cluster_public_ips(api_client: kubernetes.client.CoreV1Api) -> List[str]:
+def get_cluster_public_ips(api_client: kubernetes_client.CoreV1Api) -> List[str]:
     """
     Returns public IPs of all cluster nodes.
     """

dstack/_internal/core/backends/lambdalabs/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_shim_commands,
 )
@@ -31,6 +32,7 @@ MAX_INSTANCE_NAME_LEN = 60
 class LambdaCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     Compute,
 ):
     def __init__(self, config: LambdaConfig):

dstack/_internal/core/backends/local/compute.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithVolumeSupport,
 )
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
@@ -25,6 +26,7 @@ logger = get_logger(__name__)
 
 class LocalCompute(
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithVolumeSupport,
     Compute,
 ):

dstack/_internal/core/backends/nebius/compute.py

@@ -16,8 +16,10 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
+    merge_tags,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers, get_offers_disk_modifier
 from dstack._internal.core.backends.nebius import resources
@@ -79,6 +81,7 @@ SUPPORTED_PLATFORMS = [
 class NebiusCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     Compute,
@@ -148,6 +151,18 @@ class NebiusCompute(
             if backend_data.cluster is not None:
                 cluster_id = backend_data.cluster.id
 
+        labels = {
+            "owner": "dstack",
+            "dstack_project": instance_config.project_name.lower(),
+            "dstack_name": instance_config.instance_name,
+            "dstack_user": instance_config.user.lower(),
+        }
+        labels = merge_tags(
+            base_tags=labels,
+            backend_tags=self.config.tags,
+            resource_tags=instance_config.tags,
+        )
+        labels = resources.filter_invalid_labels(labels)
         gpus = instance_offer.instance.resources.gpus
         create_disk_op = resources.create_disk(
             sdk=self._sdk,
@@ -157,6 +172,7 @@ class NebiusCompute(
             image_family="ubuntu24.04-cuda12"
             if gpus and gpus[0].name == "B200"
             else "ubuntu22.04-cuda12",
+            labels=labels,
         )
         create_instance_op = None
         try:
@@ -182,6 +198,7 @@ class NebiusCompute(
                 disk_id=create_disk_op.resource_id,
                 subnet_id=self._get_subnet_id(instance_offer.region),
                 preemptible=instance_offer.instance.resources.spot,
+                labels=labels,
             )
             _wait_for_instance(self._sdk, create_instance_op)
         except BaseException:

dstack/_internal/core/backends/nebius/configurator.py

@@ -3,6 +3,7 @@ import json
 from nebius.aio.service_error import RequestError
 
 from dstack._internal.core.backends.base.configurator import (
+    TAGS_MAX_NUM,
     BackendRecord,
     Configurator,
     raise_invalid_credentials_error,
@@ -18,6 +19,7 @@ from dstack._internal.core.backends.nebius.models import (
     NebiusServiceAccountCreds,
     NebiusStoredConfig,
 )
+from dstack._internal.core.errors import BackendError, ServerClientError
 from dstack._internal.core.models.backends.base import BackendType
 
 
@@ -53,6 +55,19 @@ class NebiusConfigurator(
                     f" some of the valid options: {sorted(valid_fabrics)}"
                 ),
             )
+        self._check_config_tags(config)
+
+    def _check_config_tags(self, config: NebiusBackendConfigWithCreds):
+        if not config.tags:
+            return
+        if len(config.tags) > TAGS_MAX_NUM:
+            raise ServerClientError(
+                f"Maximum number of tags exceeded. Up to {TAGS_MAX_NUM} tags is allowed."
+            )
+        try:
+            resources.validate_labels(config.tags)
+        except BackendError as e:
+            raise ServerClientError(e.args[0])
 
     def create_backend(
         self, project_name: str, config: NebiusBackendConfigWithCreds

dstack/_internal/core/backends/nebius/models.py

@@ -1,4 +1,6 @@
-from typing import Annotated, Literal, Optional, Union
+import json
+from pathlib import Path
+from typing import Annotated, Dict, Literal, Optional, Union
 
 from pydantic import Field, root_validator
 
@@ -27,16 +29,38 @@ class NebiusServiceAccountCreds(CoreModel):
             )
         ),
     ]
+    filename: Annotated[
+        Optional[str], Field(description="The path to the service account credentials file")
+    ] = None
 
 
 class NebiusServiceAccountFileCreds(CoreModel):
     type: Annotated[Literal["service_account"], Field(description="The type of credentials")] = (
         "service_account"
     )
-    service_account_id: Annotated[str, Field(description="Service account ID")]
-    public_key_id: Annotated[str, Field(description="ID of the service account public key")]
+    service_account_id: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "Service account ID. Set automatically if `filename` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
+    ] = None
+    public_key_id: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "ID of the service account public key. Set automatically if `filename` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
+    ] = None
     private_key_file: Annotated[
-        Optional[str], Field(description=("Path to the service account private key"))
+        Optional[str],
+        Field(
+            description=(
+                "Path to the service account private key. Set automatically if `filename` or `private_key_content` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
     ] = None
     private_key_content: Annotated[
         Optional[str],
@@ -44,13 +68,35 @@ class NebiusServiceAccountFileCreds(CoreModel):
             description=(
                 "Content of the service account private key. When configuring via"
                 " `server/config.yml`, it's automatically filled from `private_key_file`."
-                " When configuring via UI, it has to be specified explicitly."
+                " When configuring via UI, it has to be specified explicitly"
             )
         ),
     ] = None
+    filename: Annotated[
+        Optional[str], Field(description="The path to the service account credentials file")
+    ] = None
 
     @root_validator
     def fill_data(cls, values):
+        if filename := values.get("filename"):
+            try:
+                with open(Path(filename).expanduser()) as f:
+                    data = json.load(f)
+                from nebius.base.service_account.credentials_file import (
+                    ServiceAccountCredentials,
+                )
+
+                credentials = ServiceAccountCredentials.from_json(data)
+                subject = credentials.subject_credentials
+                values["service_account_id"] = subject.sub
+                values["public_key_id"] = subject.kid
+                values["private_key_content"] = subject.private_key
+            except OSError:
+                raise ValueError(f"No such file {filename}")
+            except Exception as e:
+                raise ValueError(f"Failed to parse credentials file {filename}: {e}")
+            return values
+
         return fill_data(
             values, filename_field="private_key_file", data_field="private_key_content"
         )
@@ -95,6 +141,12 @@ class NebiusBackendConfig(CoreModel):
             )
         ),
     ] = None
+    tags: Annotated[
+        Optional[Dict[str, str]],
+        Field(
+            description="The tags (labels) that will be assigned to resources created by `dstack`"
+        ),
+    ] = None
 
 
 class NebiusBackendConfigWithCreds(NebiusBackendConfig):

dstack/_internal/core/backends/nebius/resources.py

@@ -1,11 +1,12 @@
 import logging
+import re
 import time
 from collections import defaultdict
 from collections.abc import Container as ContainerT
 from collections.abc import Generator, Iterable, Sequence
 from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
-from typing import Optional
+from typing import Dict, Optional
 
 from nebius.aio.authorization.options import options_to_metadata
 from nebius.aio.operation import Operation as SDKOperation
@@ -249,13 +250,14 @@ def get_default_subnet(sdk: SDK, project_id: str) -> Subnet:
 
 
 def create_disk(
-    sdk: SDK, name: str, project_id: str, size_mib: int, image_family: str
+    sdk: SDK, name: str, project_id: str, size_mib: int, image_family: str, labels: Dict[str, str]
 ) -> SDKOperation[Operation]:
     client = DiskServiceClient(sdk)
     request = CreateDiskRequest(
         metadata=ResourceMetadata(
             name=name,
             parent_id=project_id,
+            labels=labels,
         ),
         spec=DiskSpec(
             size_mebibytes=size_mib,
@@ -288,12 +290,14 @@ def create_instance(
     disk_id: str,
     subnet_id: str,
     preemptible: bool,
+    labels: Dict[str, str],
 ) -> SDKOperation[Operation]:
     client = InstanceServiceClient(sdk)
     request = CreateInstanceRequest(
         metadata=ResourceMetadata(
             name=name,
             parent_id=project_id,
+            labels=labels,
         ),
         spec=InstanceSpec(
             cloud_init_user_data=user_data,
@@ -367,3 +371,42 @@ def delete_cluster(sdk: SDK, cluster_id: str) -> None:
             metadata=REQUEST_MD,
         )
     )
+
+
+def filter_invalid_labels(labels: Dict[str, str]) -> Dict[str, str]:
+    filtered_labels = {}
+    for k, v in labels.items():
+        if not _is_valid_label(k, v):
+            logger.warning("Skipping invalid label '%s: %s'", k, v)
+            continue
+        filtered_labels[k] = v
+    return filtered_labels
+
+
+def validate_labels(labels: Dict[str, str]):
+    for k, v in labels.items():
+        if not _is_valid_label(k, v):
+            raise BackendError("Invalid resource labels")
+
+
+def _is_valid_label(key: str, value: str) -> bool:
+    # TODO: [Nebius] current validation logic reuses GCP's approach.
+    #   There is no public information on Nebius labels restrictions.
+    return is_valid_resource_name(key) and is_valid_label_value(value)
+
+
+MAX_RESOURCE_NAME_LEN = 63
+NAME_PATTERN = re.compile(r"^[a-z][_\-a-z0-9]{0,62}$")
+LABEL_VALUE_PATTERN = re.compile(r"^[_\-a-z0-9]{0,63}$")
+
+
+def is_valid_resource_name(name: str) -> bool:
+    if len(name) < 1 or len(name) > MAX_RESOURCE_NAME_LEN:
+        return False
+    match = re.match(NAME_PATTERN, name)
+    return match is not None
+
+
+def is_valid_label_value(value: str) -> bool:
+    match = re.match(LABEL_VALUE_PATTERN, value)
+    return match is not None

dstack/_internal/core/backends/oci/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
 )
@@ -50,6 +51,7 @@ CONFIGURABLE_DISK_SIZE = Range[Memory](min=Memory.parse("50GB"), max=Memory.pars
 class OCICompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     Compute,
 ):
@@ -118,7 +120,11 @@ class OCICompute(
         availability_domain = instance_offer.availability_zones[0]
 
         listing, package = resources.get_marketplace_listing_and_package(
-            cuda=len(instance_offer.instance.resources.gpus) > 0,
+            gpu_name=(
+                instance_offer.instance.resources.gpus[0].name
+                if len(instance_offer.instance.resources.gpus) > 0
+                else None
+            ),
             client=region.marketplace_client,
         )
         resources.accept_marketplace_listing_agreements(

dstack/_internal/core/backends/oci/resources.py

@@ -23,7 +23,9 @@ import oci
 from oci.object_storage.models import CreatePreauthenticatedRequestDetails
 
 from dstack import version
+from dstack._internal.core.backends.base.compute import requires_nvidia_proprietary_kernel_modules
 from dstack._internal.core.backends.oci.region import OCIRegionClient
+from dstack._internal.core.consts import DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES
 from dstack._internal.core.errors import BackendError
 from dstack._internal.core.models.instances import InstanceOffer
 from dstack._internal.utils.common import batched
@@ -352,11 +354,14 @@ def terminate_instance_if_exists(client: oci.core.ComputeClient, instance_id: st
 
 
 def get_marketplace_listing_and_package(
-    cuda: bool, client: oci.marketplace.MarketplaceClient
+    gpu_name: Optional[str], client: oci.marketplace.MarketplaceClient
 ) -> Tuple[oci.marketplace.models.Listing, oci.marketplace.models.ImageListingPackage]:
     listing_name = f"dstack-{version.base_image}"
-    if cuda:
-        listing_name = f"dstack-cuda-{version.base_image}"
+    if gpu_name is not None:
+        if not requires_nvidia_proprietary_kernel_modules(gpu_name):
+            listing_name = f"dstack-cuda-{version.base_image}"
+        else:
+            listing_name = f"dstack-cuda-{DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES}"
 
     listing_summaries = list_marketplace_listings(listing_name, client)
     if len(listing_summaries) != 1:

dstack/_internal/core/backends/template/compute.py.jinja

@@ -8,6 +8,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivateGatewaySupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithReservationSupport,
     ComputeWithVolumeSupport,
 )
@@ -31,6 +32,7 @@ class {{ backend_name }}Compute(
     # TODO: Choose ComputeWith* classes to extend and implement
     # ComputeWithAllOffersCached,
     # ComputeWithCreateInstanceSupport,
+    # ComputeWithPrivilegedSupport,
     # ComputeWithMultinodeSupport,
     # ComputeWithReservationSupport,
     # ComputeWithPlacementGroupSupport,

dstack/_internal/core/backends/tensordock/compute.py

@@ -6,6 +6,7 @@ import requests
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_shim_commands,
 )
@@ -32,6 +33,7 @@ MAX_INSTANCE_NAME_LEN = 60
 
 class TensorDockCompute(
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     Compute,
 ):
     def __init__(self, config: TensorDockConfig):

dstack/_internal/core/backends/vultr/compute.py

@@ -9,6 +9,7 @@ from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
+    ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
 )
@@ -35,6 +36,7 @@ MAX_INSTANCE_NAME_LEN = 64
 class VultrCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     Compute,
 ):

dstack/_internal/core/compatibility/runs.py

@@ -53,6 +53,10 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             }
         if all(js.exit_status is None for js in job_submissions):
             job_submissions_excludes["exit_status"] = True
+        if all(js.status_message == "" for js in job_submissions):
+            job_submissions_excludes["status_message"] = True
+        if all(js.error is None for js in job_submissions):
+            job_submissions_excludes["error"] = True
         if all(js.deployment_num == 0 for js in job_submissions):
             job_submissions_excludes["deployment_num"] = True
         if all(not js.probes for js in job_submissions):
@@ -71,6 +75,10 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
                 }
             if latest_job_submission.exit_status is None:
                 latest_job_submission_excludes["exit_status"] = True
+            if latest_job_submission.status_message == "":
+                latest_job_submission_excludes["status_message"] = True
+            if latest_job_submission.error is None:
+                latest_job_submission_excludes["error"] = True
             if latest_job_submission.deployment_num == 0:
                 latest_job_submission_excludes["deployment_num"] = True
             if not latest_job_submission.probes:

dstack/_internal/core/consts.py

@@ -4,3 +4,5 @@ DSTACK_SHIM_HTTP_PORT = 10998
 DSTACK_RUNNER_HTTP_PORT = 10999
 # ssh server (runs alongside the runner inside a container) listen port
 DSTACK_RUNNER_SSH_PORT = 10022
+# legacy AWS, Azure, GCP, and OCI image for older GPUs
+DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES = "0.10"