dstack 0.19.11rc1__py3-none-any.whl → 0.19.12__py3-none-any.whl

dstack/_internal/cli/commands/offer.py

@@ -84,6 +84,8 @@ class OfferCommand(APIBaseCommand):
         job_plan = run_plan.job_plans[0]
 
         if args.format == "json":
+            # FIXME: Should use effective_run_spec from run_plan,
+            # since the spec can be changed by the server and plugins
             output = {
                 "project": run_plan.project_name,
                 "user": run_plan.user,

dstack/_internal/cli/services/configurators/run.py

@@ -3,7 +3,7 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set
 
 import gpuhunt
 from pydantic import parse_obj_as
@@ -41,7 +41,7 @@ from dstack._internal.core.models.configurations import (
 )
 from dstack._internal.core.models.repos.base import Repo
 from dstack._internal.core.models.resources import CPUSpec
-from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus
+from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunStatus
 from dstack._internal.core.services.configs import ConfigManager
 from dstack._internal.core.services.diff import diff_models
 from dstack._internal.utils.common import local_time
@@ -105,7 +105,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
             changed_fields = []
             if run_plan.action == ApplyAction.UPDATE:
                 diff = diff_models(
-                    run_plan.run_spec.configuration,
+                    run_plan.get_effective_run_spec().configuration,
                     run_plan.current_resource.run_spec.configuration,
                 )
                 changed_fields = list(diff.keys())
@@ -553,35 +553,38 @@ def _print_service_urls(run: Run) -> None:
 
 
 def print_finished_message(run: Run):
+    status_message = (
+        run._run.latest_job_submission.status_message
+        if run._run.latest_job_submission
+        else run._run.status_message
+    )
+    error = (
+        run._run.latest_job_submission.error if run._run.latest_job_submission else run._run.error
+    )
+    termination_reason = (
+        run._run.latest_job_submission.termination_reason
+        if run._run.latest_job_submission
+        else None
+    )
+    termination_reason_message = (
+        run._run.latest_job_submission.termination_reason_message
+        if run._run.latest_job_submission
+        else None
+    )
     if run.status == RunStatus.DONE:
-        console.print("[code]Done[/]")
+        console.print(f"[code]{status_message.capitalize()}[/code]")
         return
+    else:
+        str = f"[error]{status_message.capitalize()}[/error]"
+        if error:
+            str += f" ([error]{error.capitalize()}[/error])"
+        console.print(str)
 
-    termination_reason, termination_reason_message, exit_status = (
-        _get_run_termination_reason_and_exit_status(run)
-    )
-    message = "Run failed due to unknown reason. Check CLI, server, and run logs."
-    if run.status == RunStatus.TERMINATED:
-        message = "Run terminated due to unknown reason. Check CLI, server, and run logs."
-
-    if termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY:
-        message = (
-            "All provisioning attempts failed. "
-            "This is likely due to cloud providers not having enough capacity. "
-            "Check CLI and server logs for more details."
-        )
-    elif termination_reason is not None:
-        exit_status_details = f"Exit status: {exit_status}.\n" if exit_status else ""
-        error_details = (
-            f"Error: {termination_reason_message}\n" if termination_reason_message else ""
-        )
-        message = (
-            f"Run failed with error code {termination_reason.name}.\n"
-            f"{exit_status_details}"
-            f"{error_details}"
-            f"Check [bold]dstack logs -d {run.name}[/bold] for more details."
-        )
-    console.print(f"[error]{message}[/]")
+        if termination_reason_message:
+            console.print(f"[error]{termination_reason_message}[/error]")
+
+        if termination_reason:
+            console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.")
 
 
 def get_run_exit_code(run: Run) -> int:
@@ -590,19 +593,17 @@ def get_run_exit_code(run: Run) -> int:
     return 1
 
 
-def _get_run_termination_reason_and_exit_status(
-    run: Run,
-) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]:
-    if len(run._run.jobs) == 0:
-        return None, None, None
-    job = run._run.jobs[0]
-    if len(job.job_submissions) == 0:
-        return None, None, None
-    job_submission = job.job_submissions[0]
-    return (
-        job_submission.termination_reason,
-        job_submission.termination_reason_message,
-        job_submission.exit_status,
+def _is_ready_to_attach(run: Run) -> bool:
+    return not (
+        run.status
+        in [
+            RunStatus.SUBMITTED,
+            RunStatus.PENDING,
+            RunStatus.PROVISIONING,
+            RunStatus.TERMINATING,
+        ]
+        or run._run.jobs[0].job_submissions[-1].status
+        in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING]
     )
 
 

dstack/_internal/cli/utils/run.py

@@ -12,7 +12,6 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
-    Job,
     RunPlan,
 )
 from dstack._internal.core.services.profiles import get_termination
@@ -154,8 +153,7 @@ def get_runs_table(
     table.add_column("BACKEND", style="grey58", ratio=2)
     table.add_column("RESOURCES", ratio=3 if not verbose else 2)
     if verbose:
-        table.add_column("INSTANCE", no_wrap=True, ratio=1)
-        table.add_column("RESERVATION", no_wrap=True, ratio=1)
+        table.add_column("INSTANCE TYPE", no_wrap=True, ratio=1)
     table.add_column("PRICE", style="grey58", ratio=1)
     table.add_column("STATUS", no_wrap=True, ratio=1)
     table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
@@ -163,14 +161,14 @@ def get_runs_table(
         table.add_column("ERROR", no_wrap=True, ratio=2)
 
     for run in runs:
-        run_error = _get_run_error(run)
         run = run._run  # TODO(egor-s): make public attribute
 
         run_row: Dict[Union[str, int], Any] = {
             "NAME": run.run_spec.run_name,
             "SUBMITTED": format_date(run.submitted_at),
-            "ERROR": run_error,
         }
+        if run.error:
+            run_row["ERROR"] = run.error
         if len(run.jobs) != 1:
             run_row["STATUS"] = run.status
             add_row_from_dict(table, run_row)
@@ -183,25 +181,26 @@ def get_runs_table(
                 status += f" (inactive for {inactive_for})"
             job_row: Dict[Union[str, int], Any] = {
                 "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-                "STATUS": status,
+                "STATUS": latest_job_submission.status_message,
                 "SUBMITTED": format_date(latest_job_submission.submitted_at),
-                "ERROR": _get_job_error(job),
+                "ERROR": latest_job_submission.error,
             }
             jpd = latest_job_submission.job_provisioning_data
             if jpd is not None:
                 resources = jpd.instance_type.resources
-                instance = jpd.instance_type.name
+                instance_type = jpd.instance_type.name
                 jrd = latest_job_submission.job_runtime_data
                 if jrd is not None and jrd.offer is not None:
                     resources = jrd.offer.instance.resources
                     if jrd.offer.total_blocks > 1:
-                        instance += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                        instance_type += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                if jpd.reservation:
+                    instance_type += f" ({jpd.reservation})"
                 job_row.update(
                     {
                         "BACKEND": f"{jpd.backend.value.replace('remote', 'ssh')} ({jpd.region})",
                         "RESOURCES": resources.pretty_format(include_spot=True),
-                        "INSTANCE": instance,
-                        "RESERVATION": jpd.reservation,
+                        "INSTANCE TYPE": instance_type,
                         "PRICE": f"${jpd.price:.4f}".rstrip("0").rstrip("."),
                     }
                 )
@@ -211,18 +210,3 @@ def get_runs_table(
             add_row_from_dict(table, job_row, style="secondary" if len(run.jobs) != 1 else None)
 
     return table
-
-
-def _get_run_error(run: Run) -> str:
-    return run._run.error or ""
-
-
-def _get_job_error(job: Job) -> str:
-    job_submission = job.job_submissions[-1]
-    termination_reason = job_submission.termination_reason
-    exit_status = job_submission.exit_status
-    if termination_reason is None:
-        return ""
-    if exit_status:
-        return f"{termination_reason.name} {exit_status}"
-    return termination_reason.name

dstack/_internal/cli/utils/updates.py

@@ -57,10 +57,22 @@ def _is_last_check_time_outdated() -> bool:
     )
 
 
+def is_update_available(current_version: str, latest_version: str) -> bool:
+    """
+    Return True if latest_version is newer than current_version.
+    Pre-releases are only considered if the current version is also a pre-release.
+    """
+    _current_version = pkg_version.parse(str(current_version))
+    _latest_version = pkg_version.parse(str(latest_version))
+    return _current_version < _latest_version and (
+        not _latest_version.is_prerelease or _current_version.is_prerelease
+    )
+
+
 def _check_version():
     latest_version = get_latest_version()
     if latest_version is not None:
-        if pkg_version.parse(str(version.__version__)) < pkg_version.parse(latest_version):
+        if is_update_available(version.__version__, latest_version):
             console.print(f"A new version of dstack is available: [code]{latest_version}[/]\n")
 
 

dstack/_internal/core/backends/aws/compute.py

@@ -611,9 +611,12 @@ class AWSCompute(
                 raise e
         logger.debug("Deleted EBS volume %s", volume.configuration.name)
 
-    def attach_volume(self, volume: Volume, instance_id: str) -> VolumeAttachmentData:
+    def attach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData
+    ) -> VolumeAttachmentData:
         ec2_client = self.session.client("ec2", region_name=volume.configuration.region)
 
+        instance_id = provisioning_data.instance_id
         device_names = aws_resources.list_available_device_names(
             ec2_client=ec2_client, instance_id=instance_id
         )
@@ -646,9 +649,12 @@ class AWSCompute(
         logger.debug("Attached EBS volume %s to instance %s", volume.volume_id, instance_id)
         return VolumeAttachmentData(device_name=device_name)
 
-    def detach_volume(self, volume: Volume, instance_id: str, force: bool = False):
+    def detach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False
+    ):
         ec2_client = self.session.client("ec2", region_name=volume.configuration.region)
 
+        instance_id = provisioning_data.instance_id
         logger.debug("Detaching EBS volume %s from instance %s", volume.volume_id, instance_id)
         attachment_data = get_or_error(volume.get_attachment_data_for_instance(instance_id))
         try:
@@ -667,9 +673,10 @@ class AWSCompute(
             raise e
         logger.debug("Detached EBS volume %s from instance %s", volume.volume_id, instance_id)
 
-    def is_volume_detached(self, volume: Volume, instance_id: str) -> bool:
+    def is_volume_detached(self, volume: Volume, provisioning_data: JobProvisioningData) -> bool:
         ec2_client = self.session.client("ec2", region_name=volume.configuration.region)
 
+        instance_id = provisioning_data.instance_id
         logger.debug("Getting EBS volume %s status", volume.volume_id)
         response = ec2_client.describe_volumes(VolumeIds=[volume.volume_id])
         volumes_infos = response.get("Volumes")
@@ -819,18 +826,23 @@ def _get_regions_to_zones(session: boto3.Session, regions: List[str]) -> Dict[st
 
 def _supported_instances(offer: InstanceOffer) -> bool:
     for family in [
+        "m7i.",
+        "c7i.",
+        "r7i.",
+        "t3.",
         "t2.small",
         "c5.",
         "m5.",
-        "g4dn.",
-        "g5.",
+        "p5.",
+        "p5e.",
+        "p4d.",
+        "p4de.",
+        "p3.",
         "g6.",
         "g6e.",
         "gr6.",
-        "p3.",
-        "p4d.",
-        "p4de.",
-        "p5.",
+        "g5.",
+        "g4dn.",
     ]:
         if offer.instance.name.startswith(family):
             return True

dstack/_internal/core/backends/base/compute.py

@@ -336,7 +336,9 @@ class ComputeWithVolumeSupport(ABC):
         """
         raise NotImplementedError()
 
-    def attach_volume(self, volume: Volume, instance_id: str) -> VolumeAttachmentData:
+    def attach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData
+    ) -> VolumeAttachmentData:
         """
         Attaches a volume to the instance.
         If the volume is not found, it should raise `ComputeError()`.
@@ -345,7 +347,9 @@ class ComputeWithVolumeSupport(ABC):
         """
         raise NotImplementedError()
 
-    def detach_volume(self, volume: Volume, instance_id: str, force: bool = False):
+    def detach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False
+    ):
         """
         Detaches a volume from the instance.
         Implement only if compute may return `VolumeProvisioningData.detachable`.
@@ -353,7 +357,7 @@ class ComputeWithVolumeSupport(ABC):
         """
         raise NotImplementedError()
 
-    def is_volume_detached(self, volume: Volume, instance_id: str) -> bool:
+    def is_volume_detached(self, volume: Volume, provisioning_data: JobProvisioningData) -> bool:
         """
         Checks if a volume was detached from the instance.
         If `detach_volume()` may fail to detach volume,

dstack/_internal/core/backends/gcp/compute.py

@@ -649,13 +649,24 @@ class GCPCompute(
             pass
         logger.debug("Deleted persistent disk for volume %s", volume.name)
 
-    def attach_volume(self, volume: Volume, instance_id: str) -> VolumeAttachmentData:
+    def attach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData
+    ) -> VolumeAttachmentData:
+        instance_id = provisioning_data.instance_id
         logger.debug(
             "Attaching persistent disk for volume %s to instance %s",
             volume.volume_id,
             instance_id,
         )
+        if not gcp_resources.instance_type_supports_persistent_disk(
+            provisioning_data.instance_type.name
+        ):
+            raise ComputeError(
+                f"Instance type {provisioning_data.instance_type.name} does not support Persistent disk volumes"
+            )
+
         zone = get_or_error(volume.provisioning_data).availability_zone
+        is_tpu = _is_tpu_provisioning_data(provisioning_data)
         try:
             disk = self.disk_client.get(
                 project=self.config.project_id,
@@ -663,18 +674,16 @@ class GCPCompute(
                 disk=volume.volume_id,
             )
             disk_url = disk.self_link
+        except google.api_core.exceptions.NotFound:
+            raise ComputeError("Persistent disk found")
 
-            # This method has no information if the instance is a TPU or a VM,
-            # so we first try to see if there is a TPU with such name
-            try:
+        try:
+            if is_tpu:
                 get_node_request = tpu_v2.GetNodeRequest(
                     name=f"projects/{self.config.project_id}/locations/{zone}/nodes/{instance_id}",
                 )
                 tpu_node = self.tpu_client.get_node(get_node_request)
-            except google.api_core.exceptions.NotFound:
-                tpu_node = None
 
-            if tpu_node is not None:
                 # Python API to attach a disk to a TPU is not documented,
                 # so we follow the code from the gcloud CLI:
                 # https://github.com/twistedpair/google-cloud-sdk/blob/26ab5a281d56b384cc25750f3279a27afe5b499f/google-cloud-sdk/lib/googlecloudsdk/command_lib/compute/tpus/tpu_vm/util.py#L113
@@ -711,7 +720,6 @@ class GCPCompute(
                 attached_disk.auto_delete = False
                 attached_disk.device_name = f"pd-{volume.volume_id}"
                 device_name = attached_disk.device_name
-
                 operation = self.instances_client.attach_disk(
                     project=self.config.project_id,
                     zone=zone,
@@ -720,13 +728,16 @@ class GCPCompute(
                 )
                 gcp_resources.wait_for_extended_operation(operation, "persistent disk attachment")
         except google.api_core.exceptions.NotFound:
-            raise ComputeError("Persistent disk or instance not found")
+            raise ComputeError("Disk or instance not found")
         logger.debug(
             "Attached persistent disk for volume %s to instance %s", volume.volume_id, instance_id
         )
         return VolumeAttachmentData(device_name=device_name)
 
-    def detach_volume(self, volume: Volume, instance_id: str, force: bool = False):
+    def detach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False
+    ):
+        instance_id = provisioning_data.instance_id
         logger.debug(
             "Detaching persistent disk for volume %s from instance %s",
             volume.volume_id,
@@ -734,17 +745,16 @@ class GCPCompute(
         )
         zone = get_or_error(volume.provisioning_data).availability_zone
         attachment_data = get_or_error(volume.get_attachment_data_for_instance(instance_id))
-        # This method has no information if the instance is a TPU or a VM,
-        # so we first try to see if there is a TPU with such name
-        try:
-            get_node_request = tpu_v2.GetNodeRequest(
-                name=f"projects/{self.config.project_id}/locations/{zone}/nodes/{instance_id}",
-            )
-            tpu_node = self.tpu_client.get_node(get_node_request)
-        except google.api_core.exceptions.NotFound:
-            tpu_node = None
+        is_tpu = _is_tpu_provisioning_data(provisioning_data)
+        if is_tpu:
+            try:
+                get_node_request = tpu_v2.GetNodeRequest(
+                    name=f"projects/{self.config.project_id}/locations/{zone}/nodes/{instance_id}",
+                )
+                tpu_node = self.tpu_client.get_node(get_node_request)
+            except google.api_core.exceptions.NotFound:
+                raise ComputeError("Instance not found")
 
-        if tpu_node is not None:
             source_disk = (
                 f"projects/{self.config.project_id}/zones/{zone}/disks/{volume.volume_id}"
             )
@@ -815,6 +825,11 @@ def _supported_instances_and_zones(
         if _is_tpu(offer.instance.name) and not _is_single_host_tpu(offer.instance.name):
             return False
         for family in [
+            "m4-",
+            "c4-",
+            "n4-",
+            "h3-",
+            "n2-",
             "e2-medium",
             "e2-standard-",
             "e2-highmem-",
@@ -1001,3 +1016,11 @@ def _get_tpu_data_disk_for_volume(project_id: str, volume: Volume) -> tpu_v2.Att
         mode=tpu_v2.AttachedDisk.DiskMode.READ_WRITE,
     )
     return attached_disk
+
+
+def _is_tpu_provisioning_data(provisioning_data: JobProvisioningData) -> bool:
+    is_tpu = False
+    if provisioning_data.backend_data:
+        backend_data_dict = json.loads(provisioning_data.backend_data)
+        is_tpu = backend_data_dict.get("is_tpu", False)
+    return is_tpu

dstack/_internal/core/backends/gcp/resources.py

@@ -140,7 +140,10 @@ def create_instance_struct(
     initialize_params = compute_v1.AttachedDiskInitializeParams()
     initialize_params.source_image = image_id
     initialize_params.disk_size_gb = disk_size
-    initialize_params.disk_type = f"zones/{zone}/diskTypes/pd-balanced"
+    if instance_type_supports_persistent_disk(machine_type):
+        initialize_params.disk_type = f"zones/{zone}/diskTypes/pd-balanced"
+    else:
+        initialize_params.disk_type = f"zones/{zone}/diskTypes/hyperdisk-balanced"
     disk.initialize_params = initialize_params
     instance.disks = [disk]
 
@@ -421,7 +424,7 @@ def wait_for_extended_operation(
 
     if operation.error_code:
         # Write only debug logs here.
-        # The unexpected errors will be propagated and logged appropriatly by the caller.
+        # The unexpected errors will be propagated and logged appropriately by the caller.
         logger.debug(
             "Error during %s: [Code: %s]: %s",
             verbose_name,
@@ -462,3 +465,16 @@ def get_placement_policy_resource_name(
     placement_policy: str,
 ) -> str:
     return f"projects/{project_id}/regions/{region}/resourcePolicies/{placement_policy}"
+
+
+def instance_type_supports_persistent_disk(instance_type_name: str) -> bool:
+    return not any(
+        instance_type_name.startswith(series)
+        for series in [
+            "m4-",
+            "c4-",
+            "n4-",
+            "h3-",
+            "v6e",
+        ]
+    )

dstack/_internal/core/backends/local/compute.py

@@ -110,8 +110,10 @@ class LocalCompute(
     def delete_volume(self, volume: Volume):
         pass
 
-    def attach_volume(self, volume: Volume, instance_id: str):
+    def attach_volume(self, volume: Volume, provisioning_data: JobProvisioningData):
         pass
 
-    def detach_volume(self, volume: Volume, instance_id: str, force: bool = False):
+    def detach_volume(
+        self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False
+    ):
         pass

dstack/_internal/core/backends/template/configurator.py.jinja

@@ -19,9 +19,6 @@ from dstack._internal.core.models.backends.base import (
     BackendType,
 )
 
-# TODO: Add all supported regions and default regions
-REGIONS = []
-
 
 class {{ backend_name }}Configurator(Configurator):
     TYPE = BackendType.{{ backend_name|upper }}
@@ -31,13 +28,11 @@ class {{ backend_name }}Configurator(Configurator):
         self, config: {{ backend_name }}BackendConfigWithCreds, default_creds_enabled: bool
     ):
         self._validate_creds(config.creds)
-        # TODO: Validate additional config parameters if any
+        # TODO: If possible, validate config.regions and any other config parameters
 
     def create_backend(
         self, project_name: str, config: {{ backend_name }}BackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config={{ backend_name }}StoredConfig(
                 **{{ backend_name }}BackendConfig.__response__.parse_obj(config).dict()

dstack/_internal/core/backends/template/models.py.jinja

@@ -22,6 +22,7 @@ class {{ backend_name }}BackendConfig(CoreModel):
     It also serves as a base class for other backend config models.
     Should not include creds.
     """
+
     type: Annotated[
         Literal["{{ backend_name|lower }}"],
         Field(description="The type of backend"),
@@ -37,6 +38,7 @@ class {{ backend_name }}BackendConfigWithCreds({{ backend_name }}BackendConfig):
     """
     Same as `{{ backend_name }}BackendConfig` but also includes creds.
     """
+
     creds: Annotated[Any{{ backend_name }}Creds, Field(description="The credentials")]
 
 
@@ -48,6 +50,7 @@ class {{ backend_name }}StoredConfig({{ backend_name }}BackendConfig):
     The backend config used for config parameters in the DB.
     Can extend `{{ backend_name }}BackendConfig` with additional parameters.
     """
+
     pass
 
 
@@ -55,4 +58,5 @@ class {{ backend_name }}Config({{ backend_name }}StoredConfig):
     """
     The backend config used by `{{ backend_name }}Backend` and `{{ backend_name }}Compute`.
     """
+
     creds: Any{{ backend_name }}Creds

dstack/_internal/core/models/configurations.py

@@ -440,7 +440,7 @@ class ServiceConfigurationParams(CoreModel):
             raise ValueError("The minimum number of replicas must be greater than or equal to 0")
         if v.max < v.min:
             raise ValueError(
-                "The maximum number of replicas must be greater than or equal to the minium number of replicas"
+                "The maximum number of replicas must be greater than or equal to the minimum number of replicas"
             )
         return v
 

dstack/_internal/core/models/fleets.py

@@ -20,6 +20,7 @@ from dstack._internal.core.models.profiles import (
     parse_idle_duration,
 )
 from dstack._internal.core.models.resources import Range, ResourcesSpec
+from dstack._internal.utils.common import list_enum_values_for_annotation
 from dstack._internal.utils.json_schema import add_extra_schema_types
 from dstack._internal.utils.tags import tags_validator
 
@@ -207,7 +208,11 @@ class InstanceGroupParams(CoreModel):
     spot_policy: Annotated[
         Optional[SpotPolicy],
         Field(
-            description="The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, or `auto`"
+            description=(
+                "The policy for provisioning spot or on-demand instances:"
+                f" {list_enum_values_for_annotation(SpotPolicy)}."
+                f" Defaults to `{SpotPolicy.ONDEMAND.value}`"
+            )
         ),
     ] = None
     retry: Annotated[