dstack 0.19.11rc2__py3-none-any.whl → 0.19.12__py3-none-any.whl

dstack/_internal/cli/commands/offer.py

@@ -84,6 +84,8 @@ class OfferCommand(APIBaseCommand):
         job_plan = run_plan.job_plans[0]
 
         if args.format == "json":
+            # FIXME: Should use effective_run_spec from run_plan,
+            # since the spec can be changed by the server and plugins
             output = {
                 "project": run_plan.project_name,
                 "user": run_plan.user,

dstack/_internal/cli/services/configurators/run.py

@@ -3,7 +3,7 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set
 
 import gpuhunt
 from pydantic import parse_obj_as
@@ -41,7 +41,7 @@ from dstack._internal.core.models.configurations import (
 )
 from dstack._internal.core.models.repos.base import Repo
 from dstack._internal.core.models.resources import CPUSpec
-from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus
+from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunStatus
 from dstack._internal.core.services.configs import ConfigManager
 from dstack._internal.core.services.diff import diff_models
 from dstack._internal.utils.common import local_time
@@ -105,7 +105,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
             changed_fields = []
             if run_plan.action == ApplyAction.UPDATE:
                 diff = diff_models(
-                    run_plan.run_spec.configuration,
+                    run_plan.get_effective_run_spec().configuration,
                     run_plan.current_resource.run_spec.configuration,
                 )
                 changed_fields = list(diff.keys())
@@ -553,35 +553,38 @@ def _print_service_urls(run: Run) -> None:
 
 
 def print_finished_message(run: Run):
+    status_message = (
+        run._run.latest_job_submission.status_message
+        if run._run.latest_job_submission
+        else run._run.status_message
+    )
+    error = (
+        run._run.latest_job_submission.error if run._run.latest_job_submission else run._run.error
+    )
+    termination_reason = (
+        run._run.latest_job_submission.termination_reason
+        if run._run.latest_job_submission
+        else None
+    )
+    termination_reason_message = (
+        run._run.latest_job_submission.termination_reason_message
+        if run._run.latest_job_submission
+        else None
+    )
     if run.status == RunStatus.DONE:
-        console.print("[code]Done[/]")
+        console.print(f"[code]{status_message.capitalize()}[/code]")
         return
+    else:
+        str = f"[error]{status_message.capitalize()}[/error]"
+        if error:
+            str += f" ([error]{error.capitalize()}[/error])"
+        console.print(str)
 
-    termination_reason, termination_reason_message, exit_status = (
-        _get_run_termination_reason_and_exit_status(run)
-    )
-    message = "Run failed due to unknown reason. Check CLI, server, and run logs."
-    if run.status == RunStatus.TERMINATED:
-        message = "Run terminated due to unknown reason. Check CLI, server, and run logs."
-
-    if termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY:
-        message = (
-            "All provisioning attempts failed. "
-            "This is likely due to cloud providers not having enough capacity. "
-            "Check CLI and server logs for more details."
-        )
-    elif termination_reason is not None:
-        exit_status_details = f"Exit status: {exit_status}.\n" if exit_status else ""
-        error_details = (
-            f"Error: {termination_reason_message}\n" if termination_reason_message else ""
-        )
-        message = (
-            f"Run failed with error code {termination_reason.name}.\n"
-            f"{exit_status_details}"
-            f"{error_details}"
-            f"Check [bold]dstack logs -d {run.name}[/bold] for more details."
-        )
-    console.print(f"[error]{message}[/]")
+        if termination_reason_message:
+            console.print(f"[error]{termination_reason_message}[/error]")
+
+        if termination_reason:
+            console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.")
 
 
 def get_run_exit_code(run: Run) -> int:
@@ -590,19 +593,17 @@ def get_run_exit_code(run: Run) -> int:
     return 1
 
 
-def _get_run_termination_reason_and_exit_status(
-    run: Run,
-) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]:
-    if len(run._run.jobs) == 0:
-        return None, None, None
-    job = run._run.jobs[0]
-    if len(job.job_submissions) == 0:
-        return None, None, None
-    job_submission = job.job_submissions[0]
-    return (
-        job_submission.termination_reason,
-        job_submission.termination_reason_message,
-        job_submission.exit_status,
+def _is_ready_to_attach(run: Run) -> bool:
+    return not (
+        run.status
+        in [
+            RunStatus.SUBMITTED,
+            RunStatus.PENDING,
+            RunStatus.PROVISIONING,
+            RunStatus.TERMINATING,
+        ]
+        or run._run.jobs[0].job_submissions[-1].status
+        in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING]
     )
 
 

dstack/_internal/cli/utils/run.py

@@ -12,7 +12,6 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
-    Job,
     RunPlan,
 )
 from dstack._internal.core.services.profiles import get_termination
@@ -154,8 +153,7 @@ def get_runs_table(
     table.add_column("BACKEND", style="grey58", ratio=2)
     table.add_column("RESOURCES", ratio=3 if not verbose else 2)
     if verbose:
-        table.add_column("INSTANCE", no_wrap=True, ratio=1)
-        table.add_column("RESERVATION", no_wrap=True, ratio=1)
+        table.add_column("INSTANCE TYPE", no_wrap=True, ratio=1)
     table.add_column("PRICE", style="grey58", ratio=1)
     table.add_column("STATUS", no_wrap=True, ratio=1)
     table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
@@ -163,14 +161,14 @@ def get_runs_table(
         table.add_column("ERROR", no_wrap=True, ratio=2)
 
     for run in runs:
-        run_error = _get_run_error(run)
         run = run._run  # TODO(egor-s): make public attribute
 
         run_row: Dict[Union[str, int], Any] = {
             "NAME": run.run_spec.run_name,
             "SUBMITTED": format_date(run.submitted_at),
-            "ERROR": run_error,
         }
+        if run.error:
+            run_row["ERROR"] = run.error
         if len(run.jobs) != 1:
             run_row["STATUS"] = run.status
             add_row_from_dict(table, run_row)
@@ -183,25 +181,26 @@ def get_runs_table(
                 status += f" (inactive for {inactive_for})"
             job_row: Dict[Union[str, int], Any] = {
                 "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-                "STATUS": status,
+                "STATUS": latest_job_submission.status_message,
                 "SUBMITTED": format_date(latest_job_submission.submitted_at),
-                "ERROR": _get_job_error(job),
+                "ERROR": latest_job_submission.error,
             }
             jpd = latest_job_submission.job_provisioning_data
             if jpd is not None:
                 resources = jpd.instance_type.resources
-                instance = jpd.instance_type.name
+                instance_type = jpd.instance_type.name
                 jrd = latest_job_submission.job_runtime_data
                 if jrd is not None and jrd.offer is not None:
                     resources = jrd.offer.instance.resources
                     if jrd.offer.total_blocks > 1:
-                        instance += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                        instance_type += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                if jpd.reservation:
+                    instance_type += f" ({jpd.reservation})"
                 job_row.update(
                     {
                         "BACKEND": f"{jpd.backend.value.replace('remote', 'ssh')} ({jpd.region})",
                         "RESOURCES": resources.pretty_format(include_spot=True),
-                        "INSTANCE": instance,
-                        "RESERVATION": jpd.reservation,
+                        "INSTANCE TYPE": instance_type,
                         "PRICE": f"${jpd.price:.4f}".rstrip("0").rstrip("."),
                     }
                 )
@@ -211,18 +210,3 @@ def get_runs_table(
             add_row_from_dict(table, job_row, style="secondary" if len(run.jobs) != 1 else None)
 
     return table
-
-
-def _get_run_error(run: Run) -> str:
-    return run._run.error or ""
-
-
-def _get_job_error(job: Job) -> str:
-    job_submission = job.job_submissions[-1]
-    termination_reason = job_submission.termination_reason
-    exit_status = job_submission.exit_status
-    if termination_reason is None:
-        return ""
-    if exit_status:
-        return f"{termination_reason.name} {exit_status}"
-    return termination_reason.name

dstack/_internal/core/backends/template/configurator.py.jinja

@@ -19,9 +19,6 @@ from dstack._internal.core.models.backends.base import (
     BackendType,
 )
 
-# TODO: Add all supported regions and default regions
-REGIONS = []
-
 
 class {{ backend_name }}Configurator(Configurator):
     TYPE = BackendType.{{ backend_name|upper }}
@@ -31,13 +28,11 @@ class {{ backend_name }}Configurator(Configurator):
         self, config: {{ backend_name }}BackendConfigWithCreds, default_creds_enabled: bool
     ):
         self._validate_creds(config.creds)
-        # TODO: Validate additional config parameters if any
+        # TODO: If possible, validate config.regions and any other config parameters
 
     def create_backend(
         self, project_name: str, config: {{ backend_name }}BackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config={{ backend_name }}StoredConfig(
                 **{{ backend_name }}BackendConfig.__response__.parse_obj(config).dict()

dstack/_internal/core/backends/template/models.py.jinja

@@ -22,6 +22,7 @@ class {{ backend_name }}BackendConfig(CoreModel):
     It also serves as a base class for other backend config models.
     Should not include creds.
     """
+
     type: Annotated[
         Literal["{{ backend_name|lower }}"],
         Field(description="The type of backend"),
@@ -37,6 +38,7 @@ class {{ backend_name }}BackendConfigWithCreds({{ backend_name }}BackendConfig):
     """
     Same as `{{ backend_name }}BackendConfig` but also includes creds.
     """
+
     creds: Annotated[Any{{ backend_name }}Creds, Field(description="The credentials")]
 
 
@@ -48,6 +50,7 @@ class {{ backend_name }}StoredConfig({{ backend_name }}BackendConfig):
     The backend config used for config parameters in the DB.
     Can extend `{{ backend_name }}BackendConfig` with additional parameters.
     """
+
     pass
 
 
@@ -55,4 +58,5 @@ class {{ backend_name }}Config({{ backend_name }}StoredConfig):
     """
     The backend config used by `{{ backend_name }}Backend` and `{{ backend_name }}Compute`.
     """
+
     creds: Any{{ backend_name }}Creds

dstack/_internal/core/models/configurations.py

@@ -440,7 +440,7 @@ class ServiceConfigurationParams(CoreModel):
             raise ValueError("The minimum number of replicas must be greater than or equal to 0")
         if v.max < v.min:
             raise ValueError(
-                "The maximum number of replicas must be greater than or equal to the minium number of replicas"
+                "The maximum number of replicas must be greater than or equal to the minimum number of replicas"
             )
         return v
 

dstack/_internal/core/models/fleets.py

@@ -20,6 +20,7 @@ from dstack._internal.core.models.profiles import (
     parse_idle_duration,
 )
 from dstack._internal.core.models.resources import Range, ResourcesSpec
+from dstack._internal.utils.common import list_enum_values_for_annotation
 from dstack._internal.utils.json_schema import add_extra_schema_types
 from dstack._internal.utils.tags import tags_validator
 
@@ -207,7 +208,11 @@ class InstanceGroupParams(CoreModel):
     spot_policy: Annotated[
         Optional[SpotPolicy],
         Field(
-            description="The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, or `auto`"
+            description=(
+                "The policy for provisioning spot or on-demand instances:"
+                f" {list_enum_values_for_annotation(SpotPolicy)}."
+                f" Defaults to `{SpotPolicy.ONDEMAND.value}`"
+            )
         ),
     ] = None
     retry: Annotated[

dstack/_internal/core/models/profiles.py

@@ -6,6 +6,7 @@ from typing_extensions import Annotated, Literal
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel, Duration
+from dstack._internal.utils.common import list_enum_values_for_annotation
 from dstack._internal.utils.tags import tags_validator
 
 DEFAULT_RETRY_DURATION = 3600
@@ -32,6 +33,17 @@ class TerminationPolicy(str, Enum):
     DESTROY_AFTER_IDLE = "destroy-after-idle"
 
 
+class StartupOrder(str, Enum):
+    ANY = "any"
+    MASTER_FIRST = "master-first"
+    WORKERS_FIRST = "workers-first"
+
+
+class StopCriteria(str, Enum):
+    ALL_DONE = "all-done"
+    MASTER_DONE = "master-done"
+
+
 @overload
 def parse_duration(v: None) -> None: ...
 
@@ -102,7 +114,7 @@ class ProfileRetry(CoreModel):
         Field(
             description=(
                 "The list of events that should be handled with retry."
-                " Supported events are `no-capacity`, `interruption`, and `error`."
+                f" Supported events are {list_enum_values_for_annotation(RetryEvent)}."
                 " Omit to retry on all events"
             )
         ),
@@ -190,7 +202,11 @@ class ProfileParams(CoreModel):
     spot_policy: Annotated[
         Optional[SpotPolicy],
         Field(
-            description="The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, or `auto`. Defaults to `on-demand`"
+            description=(
+                "The policy for provisioning spot or on-demand instances:"
+                f" {list_enum_values_for_annotation(SpotPolicy)}."
+                f" Defaults to `{SpotPolicy.ONDEMAND.value}`"
+            )
         ),
     ] = None
     retry: Annotated[
@@ -225,7 +241,11 @@ class ProfileParams(CoreModel):
     creation_policy: Annotated[
         Optional[CreationPolicy],
         Field(
-            description="The policy for using instances from fleets. Defaults to `reuse-or-create`"
+            description=(
+                "The policy for using instances from fleets:"
+                f" {list_enum_values_for_annotation(CreationPolicy)}."
+                f" Defaults to `{CreationPolicy.REUSE_OR_CREATE.value}`"
+            )
         ),
     ] = None
     idle_duration: Annotated[
@@ -241,6 +261,26 @@ class ProfileParams(CoreModel):
         Optional[UtilizationPolicy],
         Field(description="Run termination policy based on utilization"),
     ] = None
+    startup_order: Annotated[
+        Optional[StartupOrder],
+        Field(
+            description=(
+                f"The order in which master and workers jobs are started:"
+                f" {list_enum_values_for_annotation(StartupOrder)}."
+                f" Defaults to `{StartupOrder.ANY.value}`"
+            )
+        ),
+    ] = None
+    stop_criteria: Annotated[
+        Optional[StopCriteria],
+        Field(
+            description=(
+                "The criteria determining when a multi-node run should be considered finished:"
+                f" {list_enum_values_for_annotation(StopCriteria)}."
+                f" Defaults to `{StopCriteria.ALL_DONE.value}`"
+            )
+        ),
+    ] = None
     fleets: Annotated[
         Optional[list[str]], Field(description="The fleets considered for reuse")
     ] = None

dstack/_internal/core/models/repos/local.py

@@ -2,13 +2,18 @@ import tarfile
 from pathlib import Path
 from typing import BinaryIO, Optional
 
+import ignore
+import ignore.overrides
 from typing_extensions import Literal
 
 from dstack._internal.core.models.repos.base import BaseRepoInfo, Repo
+from dstack._internal.utils.common import sizeof_fmt
 from dstack._internal.utils.hash import get_sha256, slugify
-from dstack._internal.utils.ignore import GitIgnore
+from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
 
+logger = get_logger(__name__)
+
 
 class LocalRepoInfo(BaseRepoInfo):
     repo_type: Literal["local"] = "local"
@@ -69,22 +74,23 @@ class LocalRepo(Repo):
         self.run_repo_data = repo_data
 
     def write_code_file(self, fp: BinaryIO) -> str:
+        repo_path = Path(self.run_repo_data.repo_dir)
         with tarfile.TarFile(mode="w", fileobj=fp) as t:
-            t.add(
-                self.run_repo_data.repo_dir,
-                arcname="",
-                filter=TarIgnore(self.run_repo_data.repo_dir, globs=[".git"]),
-            )
+            for entry in (
+                ignore.WalkBuilder(repo_path)
+                .overrides(ignore.overrides.OverrideBuilder(repo_path).add("!/.git/").build())
+                .hidden(False)  # do not ignore files that start with a dot
+                .require_git(False)  # respect git ignore rules even if not a git repo
+                .add_custom_ignore_filename(".dstackignore")
+                .build()
+            ):
+                entry_path_within_repo = entry.path().relative_to(repo_path)
+                if entry_path_within_repo != Path("."):
+                    t.add(entry.path(), arcname=entry_path_within_repo, recursive=False)
+        logger.debug("Code file size: %s", sizeof_fmt(fp.tell()))
         return get_sha256(fp)
 
     def get_repo_info(self) -> LocalRepoInfo:
         return LocalRepoInfo(
             repo_dir=self.run_repo_data.repo_dir,
         )
-
-
-class TarIgnore(GitIgnore):
-    def __call__(self, tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
-        if self.ignore(tarinfo.path):
-            return None
-        return tarinfo

dstack/_internal/core/models/runs.py

@@ -148,9 +148,6 @@ class JobTerminationReason(str, Enum):
         }
         return mapping[self]
 
-    def pretty_repr(self) -> str:
-        return " ".join(self.value.split("_")).capitalize()
-
 
 class Requirements(CoreModel):
     # TODO: Make requirements' fields required
@@ -289,6 +286,9 @@ class JobSubmission(CoreModel):
     exit_status: Optional[int]
     job_provisioning_data: Optional[JobProvisioningData]
     job_runtime_data: Optional[JobRuntimeData]
+    # TODO: make status_message and error a computed field after migrating to pydanticV2
+    status_message: Optional[str]
+    error: Optional[str] = None
 
     @property
     def age(self) -> timedelta:
@@ -301,6 +301,71 @@ class JobSubmission(CoreModel):
             end_time = self.finished_at
         return end_time - self.submitted_at
 
+    @root_validator
+    def _status_message(cls, values) -> Dict:
+        try:
+            status = values["status"]
+            termination_reason = values["termination_reason"]
+            exit_code = values["exit_status"]
+        except KeyError:
+            return values
+        values["status_message"] = JobSubmission._get_status_message(
+            status=status,
+            termination_reason=termination_reason,
+            exit_status=exit_code,
+        )
+        return values
+
+    @staticmethod
+    def _get_status_message(
+        status: JobStatus,
+        termination_reason: Optional[JobTerminationReason],
+        exit_status: Optional[int],
+    ) -> str:
+        if status == JobStatus.DONE:
+            return "exited (0)"
+        elif status == JobStatus.FAILED:
+            if termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR:
+                return f"exited ({exit_status})"
+            elif termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY:
+                return "no offers"
+            elif termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
+                return "interrupted"
+            else:
+                return "error"
+        elif status == JobStatus.TERMINATED:
+            if termination_reason == JobTerminationReason.TERMINATED_BY_USER:
+                return "stopped"
+            elif termination_reason == JobTerminationReason.ABORTED_BY_USER:
+                return "aborted"
+        return status.value
+
+    @root_validator
+    def _error(cls, values) -> Dict:
+        try:
+            termination_reason = values["termination_reason"]
+        except KeyError:
+            return values
+        values["error"] = JobSubmission._get_error(termination_reason=termination_reason)
+        return values
+
+    @staticmethod
+    def _get_error(termination_reason: Optional[JobTerminationReason]) -> Optional[str]:
+        error_mapping = {
+            JobTerminationReason.INSTANCE_UNREACHABLE: "instance unreachable",
+            JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED: "waiting instance limit exceeded",
+            JobTerminationReason.VOLUME_ERROR: "waiting runner limit exceeded",
+            JobTerminationReason.GATEWAY_ERROR: "gateway error",
+            JobTerminationReason.SCALED_DOWN: "scaled down",
+            JobTerminationReason.INACTIVITY_DURATION_EXCEEDED: "inactivity duration exceeded",
+            JobTerminationReason.TERMINATED_DUE_TO_UTILIZATION_POLICY: "utilization policy",
+            JobTerminationReason.PORTS_BINDING_FAILED: "ports binding failed",
+            JobTerminationReason.CREATING_CONTAINER_ERROR: "runner error",
+            JobTerminationReason.EXECUTOR_ERROR: "executor error",
+            JobTerminationReason.MAX_DURATION_EXCEEDED: "max duration exceeded",
+        }
+        return error_mapping.get(termination_reason)
+
 
 class Job(CoreModel):
     job_spec: JobSpec
@@ -445,15 +510,20 @@ class Run(CoreModel):
     def _error(cls, values) -> Dict:
         try:
             termination_reason = values["termination_reason"]
-            jobs = values["jobs"]
         except KeyError:
             return values
-        values["error"] = _get_run_error(
-            run_termination_reason=termination_reason,
-            run_jobs=jobs,
-        )
+        values["error"] = Run._get_error(termination_reason=termination_reason)
         return values
 
+    @staticmethod
+    def _get_error(termination_reason: Optional[RunTerminationReason]) -> Optional[str]:
+        if termination_reason == RunTerminationReason.RETRY_LIMIT_EXCEEDED:
+            return "retry limit exceeded"
+        elif termination_reason == RunTerminationReason.SERVER_ERROR:
+            return "server error"
+        else:
+            return None
+
 
 class JobPlan(CoreModel):
     job_spec: JobSpec
@@ -502,40 +572,3 @@ def get_policy_map(spot_policy: Optional[SpotPolicy], default: SpotPolicy) -> Op
         SpotPolicy.ONDEMAND: False,
     }
     return policy_map[spot_policy]
-
-
-def _get_run_error(
-    run_termination_reason: Optional[RunTerminationReason],
-    run_jobs: List[Job],
-) -> str:
-    if run_termination_reason is None:
-        return ""
-    if len(run_jobs) > 1:
-        return run_termination_reason.name
-    run_job_termination_reason, exit_status = _get_run_job_termination_reason_and_exit_status(
-        run_jobs
-    )
-    # For failed runs, also show termination reason to provide more context.
-    # For other run statuses, the job termination reason will duplicate run status.
-    if run_job_termination_reason is not None and run_termination_reason in [
-        RunTerminationReason.JOB_FAILED,
-        RunTerminationReason.SERVER_ERROR,
-        RunTerminationReason.RETRY_LIMIT_EXCEEDED,
-    ]:
-        if exit_status:
-            return (
-                f"{run_termination_reason.name}\n({run_job_termination_reason.name} {exit_status})"
-            )
-        return f"{run_termination_reason.name}\n({run_job_termination_reason.name})"
-    return run_termination_reason.name
-
-
-def _get_run_job_termination_reason_and_exit_status(
-    run_jobs: List[Job],
-) -> tuple[Optional[JobTerminationReason], Optional[int]]:
-    for job in run_jobs:
-        if len(job.job_submissions) > 0:
-            job_submission = job.job_submissions[-1]
-            if job_submission.termination_reason is not None:
-                return job_submission.termination_reason, job_submission.exit_status
-    return None, None