dstack 0.19.12rc1__py3-none-any.whl → 0.19.14__py3-none-any.whl

dstack/_internal/cli/commands/attach.py

@@ -52,9 +52,8 @@ class AttachCommand(APIBaseCommand):
         )
         self._parser.add_argument(
             "--replica",
-            help="The replica number. Defaults to 0.",
+            help="The replica number. Defaults to any running replica.",
             type=int,
-            default=0,
         )
         self._parser.add_argument(
             "--job",
@@ -129,14 +128,15 @@ _IGNORED_PORTS = [DSTACK_RUNNER_HTTP_PORT]
 def _print_attached_message(
     run: Run,
     bind_address: Optional[str],
-    replica_num: int,
+    replica_num: Optional[int],
     job_num: int,
 ):
     if bind_address is None:
         bind_address = "localhost"
 
-    output = f"Attached to run [code]{run.name}[/] (replica={replica_num} job={job_num})\n"
     job = get_or_error(run._find_job(replica_num=replica_num, job_num=job_num))
+    replica_num = job.job_spec.replica_num
+    output = f"Attached to run [code]{run.name}[/] (replica={replica_num} job={job_num})\n"
     name = run.name
     if replica_num != 0 or job_num != 0:
         name = job.job_spec.job_name

dstack/_internal/cli/services/configurators/run.py

@@ -3,7 +3,7 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set
 
 import gpuhunt
 from pydantic import parse_obj_as
@@ -41,7 +41,7 @@ from dstack._internal.core.models.configurations import (
 )
 from dstack._internal.core.models.repos.base import Repo
 from dstack._internal.core.models.resources import CPUSpec
-from dstack._internal.core.models.runs import JobSubmission, JobTerminationReason, RunStatus
+from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunStatus
 from dstack._internal.core.services.configs import ConfigManager
 from dstack._internal.core.services.diff import diff_models
 from dstack._internal.utils.common import local_time
@@ -166,12 +166,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
             # We can attach to run multiple times if it goes from running to pending (retried).
             while True:
                 with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live:
-                    while run.status in (
-                        RunStatus.SUBMITTED,
-                        RunStatus.PENDING,
-                        RunStatus.PROVISIONING,
-                        RunStatus.TERMINATING,
-                    ):
+                    while not _is_ready_to_attach(run):
                         table = get_runs_table([run])
                         live.update(table)
                         time.sleep(5)
@@ -553,35 +548,38 @@ def _print_service_urls(run: Run) -> None:
 
 
 def print_finished_message(run: Run):
+    status_message = (
+        run._run.latest_job_submission.status_message
+        if run._run.latest_job_submission
+        else run._run.status_message
+    )
+    error = (
+        run._run.latest_job_submission.error if run._run.latest_job_submission else run._run.error
+    )
+    termination_reason = (
+        run._run.latest_job_submission.termination_reason
+        if run._run.latest_job_submission
+        else None
+    )
+    termination_reason_message = (
+        run._run.latest_job_submission.termination_reason_message
+        if run._run.latest_job_submission
+        else None
+    )
     if run.status == RunStatus.DONE:
-        console.print("[code]Done[/]")
+        console.print(f"[code]{status_message.capitalize()}[/code]")
         return
+    else:
+        str = f"[error]{status_message.capitalize()}[/error]"
+        if error:
+            str += f" ([error]{error.capitalize()}[/error])"
+        console.print(str)
 
-    termination_reason, termination_reason_message, exit_status = (
-        _get_run_termination_reason_and_exit_status(run)
-    )
-    message = "Run failed due to unknown reason. Check CLI, server, and run logs."
-    if run.status == RunStatus.TERMINATED:
-        message = "Run terminated due to unknown reason. Check CLI, server, and run logs."
-
-    if termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY:
-        message = (
-            "All provisioning attempts failed. "
-            "This is likely due to cloud providers not having enough capacity. "
-            "Check CLI and server logs for more details."
-        )
-    elif termination_reason is not None:
-        exit_status_details = f"Exit status: {exit_status}.\n" if exit_status else ""
-        error_details = (
-            f"Error: {termination_reason_message}\n" if termination_reason_message else ""
-        )
-        message = (
-            f"Run failed with error code {termination_reason.name}.\n"
-            f"{exit_status_details}"
-            f"{error_details}"
-            f"Check [bold]dstack logs -d {run.name}[/bold] for more details."
-        )
-    console.print(f"[error]{message}[/]")
+        if termination_reason_message:
+            console.print(f"[error]{termination_reason_message}[/error]")
+
+        if termination_reason:
+            console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.")
 
 
 def get_run_exit_code(run: Run) -> int:
@@ -590,19 +588,18 @@ def get_run_exit_code(run: Run) -> int:
     return 1
 
 
-def _get_run_termination_reason_and_exit_status(
-    run: Run,
-) -> Tuple[Optional[JobTerminationReason], Optional[str], Optional[int]]:
-    if len(run._run.jobs) == 0:
-        return None, None, None
-    job = run._run.jobs[0]
-    if len(job.job_submissions) == 0:
-        return None, None, None
-    job_submission = job.job_submissions[0]
-    return (
-        job_submission.termination_reason,
-        job_submission.termination_reason_message,
-        job_submission.exit_status,
+def _is_ready_to_attach(run: Run) -> bool:
+    return not (
+        run.status
+        in [
+            RunStatus.SUBMITTED,
+            RunStatus.PENDING,
+            RunStatus.PROVISIONING,
+            RunStatus.TERMINATING,
+        ]
+        or run._run.jobs[0].job_submissions[-1].status
+        in [JobStatus.SUBMITTED, JobStatus.PROVISIONING, JobStatus.PULLING]
+        or run._run.is_deployment_in_progress()
     )
 
 

dstack/_internal/cli/utils/run.py

@@ -12,7 +12,6 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
-    Job,
     RunPlan,
 )
 from dstack._internal.core.services.profiles import get_termination
@@ -154,8 +153,7 @@ def get_runs_table(
     table.add_column("BACKEND", style="grey58", ratio=2)
     table.add_column("RESOURCES", ratio=3 if not verbose else 2)
     if verbose:
-        table.add_column("INSTANCE", no_wrap=True, ratio=1)
-        table.add_column("RESERVATION", no_wrap=True, ratio=1)
+        table.add_column("INSTANCE TYPE", no_wrap=True, ratio=1)
     table.add_column("PRICE", style="grey58", ratio=1)
     table.add_column("STATUS", no_wrap=True, ratio=1)
     table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
@@ -163,16 +161,27 @@ def get_runs_table(
         table.add_column("ERROR", no_wrap=True, ratio=2)
 
     for run in runs:
-        run_error = _get_run_error(run)
         run = run._run  # TODO(egor-s): make public attribute
+        show_deployment_num = (
+            verbose
+            and run.run_spec.configuration.type == "service"
+            or run.is_deployment_in_progress()
+        )
+        merge_job_rows = len(run.jobs) == 1 and not show_deployment_num
 
         run_row: Dict[Union[str, int], Any] = {
-            "NAME": run.run_spec.run_name,
+            "NAME": run.run_spec.run_name
+            + (f" [secondary]deployment={run.deployment_num}[/]" if show_deployment_num else ""),
             "SUBMITTED": format_date(run.submitted_at),
-            "ERROR": run_error,
+            "STATUS": (
+                run.latest_job_submission.status_message
+                if run.status.is_finished() and run.latest_job_submission
+                else run.status_message
+            ),
         }
-        if len(run.jobs) != 1:
-            run_row["STATUS"] = run.status
+        if run.error:
+            run_row["ERROR"] = run.error
+        if not merge_job_rows:
             add_row_from_dict(table, run_row)
 
         for job in run.jobs:
@@ -182,47 +191,38 @@ def get_runs_table(
                 inactive_for = format_duration_multiunit(latest_job_submission.inactivity_secs)
                 status += f" (inactive for {inactive_for})"
             job_row: Dict[Union[str, int], Any] = {
-                "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-                "STATUS": status,
+                "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}"
+                + (
+                    f" deployment={latest_job_submission.deployment_num}"
+                    if show_deployment_num
+                    else ""
+                ),
+                "STATUS": latest_job_submission.status_message,
                 "SUBMITTED": format_date(latest_job_submission.submitted_at),
-                "ERROR": _get_job_error(job),
+                "ERROR": latest_job_submission.error,
             }
             jpd = latest_job_submission.job_provisioning_data
             if jpd is not None:
                 resources = jpd.instance_type.resources
-                instance = jpd.instance_type.name
+                instance_type = jpd.instance_type.name
                 jrd = latest_job_submission.job_runtime_data
                 if jrd is not None and jrd.offer is not None:
                     resources = jrd.offer.instance.resources
                     if jrd.offer.total_blocks > 1:
-                        instance += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                        instance_type += f" ({jrd.offer.blocks}/{jrd.offer.total_blocks})"
+                if jpd.reservation:
+                    instance_type += f" ({jpd.reservation})"
                 job_row.update(
                     {
                         "BACKEND": f"{jpd.backend.value.replace('remote', 'ssh')} ({jpd.region})",
                         "RESOURCES": resources.pretty_format(include_spot=True),
-                        "INSTANCE": instance,
-                        "RESERVATION": jpd.reservation,
+                        "INSTANCE TYPE": instance_type,
                         "PRICE": f"${jpd.price:.4f}".rstrip("0").rstrip("."),
                     }
                 )
-            if len(run.jobs) == 1:
+            if merge_job_rows:
                 # merge rows
                 job_row.update(run_row)
             add_row_from_dict(table, job_row, style="secondary" if len(run.jobs) != 1 else None)
 
     return table
-
-
-def _get_run_error(run: Run) -> str:
-    return run._run.error or ""
-
-
-def _get_job_error(job: Job) -> str:
-    job_submission = job.job_submissions[-1]
-    termination_reason = job_submission.termination_reason
-    exit_status = job_submission.exit_status
-    if termination_reason is None:
-        return ""
-    if exit_status:
-        return f"{termination_reason.name} {exit_status}"
-    return termination_reason.name

dstack/_internal/core/backends/aws/compute.py

@@ -132,7 +132,8 @@ class AWSCompute(
         availability_offers = []
         for offer in offers:
             availability = InstanceAvailability.UNKNOWN
-            if not _has_quota(regions_to_quotas[offer.region], offer.instance.name):
+            quota = _has_quota(regions_to_quotas[offer.region], offer.instance.name)
+            if quota is not None and not quota:
                 availability = InstanceAvailability.NO_QUOTA
             availability_offers.append(
                 InstanceOfferWithAvailability(
@@ -231,6 +232,7 @@ class AWSCompute(
                 image_id, username = aws_resources.get_image_id_and_username(
                     ec2_client=ec2_client,
                     cuda=len(instance_offer.instance.resources.gpus) > 0,
+                    instance_type=instance_offer.instance.name,
                     image_config=self.config.os_images,
                 )
                 response = ec2_resource.create_instances(
@@ -781,10 +783,18 @@ def _get_regions_to_quotas(
 ) -> Dict[str, Dict[str, int]]:
     def get_region_quotas(client: botocore.client.BaseClient) -> Dict[str, int]:
         region_quotas = {}
-        for page in client.get_paginator("list_service_quotas").paginate(ServiceCode="ec2"):
-            for q in page["Quotas"]:
-                if "On-Demand" in q["QuotaName"]:
-                    region_quotas[q["UsageMetric"]["MetricDimensions"]["Class"]] = q["Value"]
+        try:
+            for page in client.get_paginator("list_service_quotas").paginate(ServiceCode="ec2"):
+                for q in page["Quotas"]:
+                    if "On-Demand" in q["QuotaName"]:
+                        region_quotas[q["UsageMetric"]["MetricDimensions"]["Class"]] = q["Value"]
+        except botocore.exceptions.ClientError as e:
+            if len(e.args) > 0 and "TooManyRequestsException" in e.args[0]:
+                logger.warning(
+                    "Failed to get quotas due to rate limits. Quotas won't be accounted for."
+                )
+            else:
+                logger.exception(e)
         return region_quotas
 
     regions_to_quotas = {}
@@ -800,12 +810,15 @@ def _get_regions_to_quotas(
     return regions_to_quotas
 
 
-def _has_quota(quotas: Dict[str, int], instance_name: str) -> bool:
+def _has_quota(quotas: Dict[str, int], instance_name: str) -> Optional[bool]:
+    quota = quotas.get("Standard/OnDemand")
     if instance_name.startswith("p"):
-        return quotas.get("P/OnDemand", 0) > 0
+        quota = quotas.get("P/OnDemand")
     if instance_name.startswith("g"):
-        return quotas.get("G/OnDemand", 0) > 0
-    return quotas.get("Standard/OnDemand", 0) > 0
+        quota = quotas.get("G/OnDemand")
+    if quota is None:
+        return None
+    return quota > 0
 
 
 def _get_regions_to_zones(session: boto3.Session, regions: List[str]) -> Dict[str, List[str]]:

dstack/_internal/core/backends/aws/resources.py

@@ -12,11 +12,13 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 DSTACK_ACCOUNT_ID = "142421590066"
+DLAMI_OWNER_ACCOUNT_ID = "898082745236"
 
 
 def get_image_id_and_username(
     ec2_client: botocore.client.BaseClient,
     cuda: bool,
+    instance_type: str,
     image_config: Optional[AWSOSImageConfig] = None,
 ) -> tuple[str, str]:
     if image_config is not None:
@@ -27,6 +29,11 @@ def get_image_id_and_username(
         image_name = image.name
         image_owner = image.owner
         username = image.user
+    elif _supported_by_dlami(instance_type):
+        # TODO: Update DLAMI image version from time to time
+        image_name = "Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04) 20250516"
+        image_owner = DLAMI_OWNER_ACCOUNT_ID
+        username = "ubuntu"
     else:
         image_name = (
             f"dstack-{version.base_image}" if not cuda else f"dstack-cuda-{version.base_image}"
@@ -628,6 +635,25 @@ def _is_private_subnet_with_internet_egress(
     return False
 
 
+def _supported_by_dlami(instance_type: str) -> bool:
+    # Currently only p3. instances are not supported by DLAMI among GPU instances.
+    return any(
+        instance_type.startswith(family)
+        for family in [
+            "g4dn.",
+            "g5.",
+            "g6.",
+            "gr6.",
+            "g6e.",
+            "p4d.",
+            "p4de.",
+            "p5.",
+            "p5e.",
+            "p6-b200.",
+        ]
+    )
+
+
 def get_reservation(
     ec2_client: botocore.client.BaseClient,
     reservation_id: str,

dstack/_internal/core/backends/base/offers.py

@@ -80,7 +80,6 @@ def catalog_item_to_offer(
         spot=item.spot,
         disk=Disk(size_mib=disk_size_mib),
     )
-    resources.description = resources.pretty_format()
     return InstanceOffer(
         backend=backend,
         instance=InstanceType(

dstack/_internal/core/backends/template/configurator.py.jinja

@@ -19,9 +19,6 @@ from dstack._internal.core.models.backends.base import (
     BackendType,
 )
 
-# TODO: Add all supported regions and default regions
-REGIONS = []
-
 
 class {{ backend_name }}Configurator(Configurator):
     TYPE = BackendType.{{ backend_name|upper }}
@@ -31,13 +28,11 @@ class {{ backend_name }}Configurator(Configurator):
         self, config: {{ backend_name }}BackendConfigWithCreds, default_creds_enabled: bool
     ):
         self._validate_creds(config.creds)
-        # TODO: Validate additional config parameters if any
+        # TODO: If possible, validate config.regions and any other config parameters
 
     def create_backend(
         self, project_name: str, config: {{ backend_name }}BackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config={{ backend_name }}StoredConfig(
                 **{{ backend_name }}BackendConfig.__response__.parse_obj(config).dict()

dstack/_internal/core/backends/template/models.py.jinja

@@ -22,6 +22,7 @@ class {{ backend_name }}BackendConfig(CoreModel):
     It also serves as a base class for other backend config models.
     Should not include creds.
     """
+
     type: Annotated[
         Literal["{{ backend_name|lower }}"],
         Field(description="The type of backend"),
@@ -37,6 +38,7 @@ class {{ backend_name }}BackendConfigWithCreds({{ backend_name }}BackendConfig):
     """
     Same as `{{ backend_name }}BackendConfig` but also includes creds.
     """
+
     creds: Annotated[Any{{ backend_name }}Creds, Field(description="The credentials")]
 
 
@@ -48,6 +50,7 @@ class {{ backend_name }}StoredConfig({{ backend_name }}BackendConfig):
     The backend config used for config parameters in the DB.
     Can extend `{{ backend_name }}BackendConfig` with additional parameters.
     """
+
     pass
 
 
@@ -55,4 +58,5 @@ class {{ backend_name }}Config({{ backend_name }}StoredConfig):
     """
     The backend config used by `{{ backend_name }}Backend` and `{{ backend_name }}Compute`.
     """
+
     creds: Any{{ backend_name }}Creds

dstack/_internal/core/compatibility/fleets.py

@@ -0,0 +1,72 @@
+from typing import Any, Dict, Optional
+
+from dstack._internal.core.models.fleets import ApplyFleetPlanInput, FleetSpec
+from dstack._internal.core.models.instances import Instance
+
+
+def get_get_plan_excludes(fleet_spec: FleetSpec) -> Dict:
+    get_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        get_plan_excludes["spec"] = spec_excludes
+    return get_plan_excludes
+
+
+def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
+    apply_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(plan_input.spec)
+    if spec_excludes:
+        apply_plan_excludes["spec"] = spec_excludes
+    current_resource = plan_input.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        if all(map(_should_exclude_instance_cpu_arch, current_resource.instances)):
+            current_resource_excludes["instances"] = {
+                "__all__": {"instance_type": {"resources": {"cpu_arch"}}}
+            }
+    return {"plan": apply_plan_excludes}
+
+
+def get_create_fleet_excludes(fleet_spec: FleetSpec) -> Dict:
+    create_fleet_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        create_fleet_excludes["spec"] = spec_excludes
+    return create_fleet_excludes
+
+
+def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[Dict]:
+    """
+    Returns `fleet_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes: Dict[str, Any] = {}
+    configuration_excludes: Dict[str, Any] = {}
+    profile_excludes: set[str] = set()
+    profile = fleet_spec.profile
+    if profile.fleets is None:
+        profile_excludes.add("fleets")
+    if fleet_spec.configuration.tags is None:
+        configuration_excludes["tags"] = True
+    if profile.tags is None:
+        profile_excludes.add("tags")
+    if profile.startup_order is None:
+        profile_excludes.add("startup_order")
+    if profile.stop_criteria is None:
+        profile_excludes.add("stop_criteria")
+    if configuration_excludes:
+        spec_excludes["configuration"] = configuration_excludes
+    if profile_excludes:
+        spec_excludes["profile"] = profile_excludes
+    if spec_excludes:
+        return spec_excludes
+    return None
+
+
+def _should_exclude_instance_cpu_arch(instance: Instance) -> bool:
+    try:
+        return instance.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True

dstack/_internal/core/compatibility/gateways.py

@@ -0,0 +1,34 @@
+from typing import Dict
+
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewaySpec
+
+
+def get_gateway_spec_excludes(gateway_spec: GatewaySpec) -> Dict:
+    """
+    Returns `gateway_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes = {}
+    spec_excludes["configuration"] = _get_gateway_configuration_excludes(
+        gateway_spec.configuration
+    )
+    return spec_excludes
+
+
+def get_create_gateway_excludes(configuration: GatewayConfiguration) -> Dict:
+    """
+    Returns an exclude mapping to exclude certain fields from the create gateway request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    create_gateway_excludes = {}
+    create_gateway_excludes["configuration"] = _get_gateway_configuration_excludes(configuration)
+    return create_gateway_excludes
+
+
+def _get_gateway_configuration_excludes(configuration: GatewayConfiguration) -> Dict:
+    configuration_excludes = {}
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    return configuration_excludes