dstack 0.19.7__py3-none-any.whl → 0.19.9__py3-none-any.whl

dstack/_internal/server/testing/common.py

@@ -2,7 +2,7 @@ import json
 import uuid
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Literal, Optional, Union
 from uuid import UUID
 
 import gpuhunt
@@ -25,7 +25,12 @@ from dstack._internal.core.models.configurations import (
     DevEnvironmentConfiguration,
 )
 from dstack._internal.core.models.envs import Env
-from dstack._internal.core.models.fleets import FleetConfiguration, FleetSpec, FleetStatus
+from dstack._internal.core.models.fleets import (
+    FleetConfiguration,
+    FleetSpec,
+    FleetStatus,
+    InstanceGroupPlacement,
+)
 from dstack._internal.core.models.gateways import GatewayComputeConfiguration, GatewayStatus
 from dstack._internal.core.models.instances import (
     Disk,
@@ -51,7 +56,7 @@ from dstack._internal.core.models.profiles import (
 )
 from dstack._internal.core.models.repos.base import RepoType
 from dstack._internal.core.models.repos.local import LocalRunRepoData
-from dstack._internal.core.models.resources import Memory, Range, ResourcesSpec
+from dstack._internal.core.models.resources import CPUSpec, Memory, Range, ResourcesSpec
 from dstack._internal.core.models.runs import (
     JobProvisioningData,
     JobRuntimeData,
@@ -297,6 +302,7 @@ async def create_job(
     job_num: int = 0,
     replica_num: int = 0,
     instance_assigned: bool = False,
+    disconnected_at: Optional[datetime] = None,
 ) -> JobModel:
     run_spec = RunSpec.parse_raw(run.run_spec)
     job_spec = (await get_job_specs_from_run_spec(run_spec, replica_num=replica_num))[0]
@@ -318,6 +324,7 @@ async def create_job(
         instance=instance,
         instance_assigned=instance_assigned,
         used_instance_id=instance.id if instance is not None else None,
+        disconnected_at=disconnected_at,
     )
     session.add(job)
     await session.commit()
@@ -497,10 +504,12 @@ def get_fleet_spec(conf: Optional[FleetConfiguration] = None) -> FleetSpec:
 def get_fleet_configuration(
     name: str = "test-fleet",
     nodes: Range[int] = Range(min=1, max=1),
+    placement: Optional[InstanceGroupPlacement] = None,
 ) -> FleetConfiguration:
     return FleetConfiguration(
         name=name,
         nodes=nodes,
+        placement=placement,
     )
 
 
@@ -519,13 +528,13 @@ async def create_instance(
     instance_id: Optional[UUID] = None,
     job: Optional[JobModel] = None,
     instance_num: int = 0,
-    backend: BackendType = BackendType.DATACRUNCH,
+    backend: Optional[BackendType] = BackendType.DATACRUNCH,
     termination_policy: Optional[TerminationPolicy] = None,
     termination_idle_time: int = DEFAULT_FLEET_TERMINATION_IDLE_TIME,
-    region: str = "eu-west",
+    region: Optional[str] = "eu-west",
     remote_connection_info: Optional[RemoteConnectionInfo] = None,
-    offer: Optional[InstanceOfferWithAvailability] = None,
-    job_provisioning_data: Optional[JobProvisioningData] = None,
+    offer: Optional[Union[InstanceOfferWithAvailability, Literal["auto"]]] = "auto",
+    job_provisioning_data: Optional[Union[JobProvisioningData, Literal["auto"]]] = "auto",
     total_blocks: Optional[int] = 1,
     busy_blocks: int = 0,
     name: str = "test_instance",
@@ -534,7 +543,7 @@ async def create_instance(
 ) -> InstanceModel:
     if instance_id is None:
         instance_id = uuid.uuid4()
-    if job_provisioning_data is None:
+    if job_provisioning_data == "auto":
         job_provisioning_data = get_job_provisioning_data(
             dockerized=True,
             backend=backend,
@@ -543,13 +552,13 @@ async def create_instance(
             hostname="running_instance.ip",
             internal_ip=None,
         )
-    if offer is None:
+    if offer == "auto":
         offer = get_instance_offer_with_availability(backend=backend, region=region, spot=spot)
     if profile is None:
         profile = Profile(name="test_name")
 
     if requirements is None:
-        requirements = Requirements(resources=ResourcesSpec(cpu=1))
+        requirements = Requirements(resources=ResourcesSpec(cpu=CPUSpec.parse("1")))
 
     if instance_configuration is None:
         instance_configuration = get_instance_configuration()
@@ -571,8 +580,8 @@ async def create_instance(
         created_at=created_at,
         started_at=created_at,
         finished_at=finished_at,
-        job_provisioning_data=job_provisioning_data.json(),
-        offer=offer.json(),
+        job_provisioning_data=job_provisioning_data.json() if job_provisioning_data else None,
+        offer=offer.json() if offer else None,
         price=price,
         region=region,
         backend=backend,
@@ -659,20 +668,7 @@ def get_remote_connection_info(
     env: Optional[Union[Env, dict]] = None,
 ):
     if ssh_keys is None:
-        ssh_keys = [
-            SSHKey(
-                public="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6mJxVbNtm0zXgMLvByrhXJCmJRveSrJxLB5/OzcyCk",
-                private="""
-                    -----BEGIN OPENSSH PRIVATE KEY-----
-                    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
-                    QyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpAAAAJCiWa5Volmu
-                    VQAAAAtzc2gtZWQyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpA
-                    AAAEAncHi4AhS6XdMp5Gzd+IMse/4ekyQ54UngByf0Sp0uH+6mJxVbNtm0zXgMLvByrhXJ
-                    CmJRveSrJxLB5/OzcyCkAAAACWRlZkBkZWZwYwECAwQ=
-                    -----END OPENSSH PRIVATE KEY-----
-                """,
-            )
-        ]
+        ssh_keys = [get_ssh_key()]
     if env is None:
         env = Env()
     elif isinstance(env, dict):
@@ -686,6 +682,21 @@ def get_remote_connection_info(
     )
 
 
+def get_ssh_key() -> SSHKey:
+    return SSHKey(
+        public="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6mJxVbNtm0zXgMLvByrhXJCmJRveSrJxLB5/OzcyCk",
+        private="""
+                    -----BEGIN OPENSSH PRIVATE KEY-----
+                    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+                    QyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpAAAAJCiWa5Volmu
+                    VQAAAAtzc2gtZWQyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpA
+                    AAAEAncHi4AhS6XdMp5Gzd+IMse/4ekyQ54UngByf0Sp0uH+6mJxVbNtm0zXgMLvByrhXJ
+                    CmJRveSrJxLB5/OzcyCkAAAACWRlZkBkZWZwYwECAwQ=
+                    -----END OPENSSH PRIVATE KEY-----
+                """,
+    )
+
+
 async def create_volume(
     session: AsyncSession,
     project: ProjectModel,

dstack/_internal/utils/common.py

@@ -1,4 +1,5 @@
 import asyncio
+import enum
 import itertools
 import re
 import time
@@ -83,6 +84,8 @@ def pretty_date(time: datetime) -> str:
 
 
 def pretty_resources(
+    *,
+    cpu_arch: Optional[Any] = None,
     cpus: Optional[Any] = None,
     memory: Optional[Any] = None,
     gpu_count: Optional[Any] = None,
@@ -110,7 +113,16 @@ def pretty_resources(
     """
     parts = []
     if cpus is not None:
-        parts.append(f"cpu={cpus}")
+        cpu_arch_lower: Optional[str] = None
+        if isinstance(cpu_arch, enum.Enum):
+            cpu_arch_lower = str(cpu_arch.value).lower()
+        elif isinstance(cpu_arch, str):
+            cpu_arch_lower = cpu_arch.lower()
+        if cpu_arch_lower == "arm":
+            cpu_arch_prefix = "arm:"
+        else:
+            cpu_arch_prefix = ""
+        parts.append(f"cpu={cpu_arch_prefix}{cpus}")
     if memory is not None:
         parts.append(f"mem={memory}")
     if disk_size:

dstack/_internal/utils/json_schema.py

@@ -1,6 +1,9 @@
 def add_extra_schema_types(schema_property: dict, extra_types: list[dict]):
     if "allOf" in schema_property:
-        ref = schema_property.pop("allOf")[0]
+        refs = [schema_property.pop("allOf")[0]]
+    elif "anyOf" in schema_property:
+        refs = schema_property.pop("anyOf")
     else:
-        ref = {"type": schema_property.pop("type")}
-    schema_property["anyOf"] = [ref, *extra_types]
+        refs = [{"type": schema_property.pop("type")}]
+    refs.extend(extra_types)
+    schema_property["anyOf"] = refs

dstack/api/__init__.py

@@ -14,6 +14,7 @@ from dstack._internal.core.models.repos.local import LocalRepo
 from dstack._internal.core.models.repos.remote import RemoteRepo
 from dstack._internal.core.models.repos.virtual import VirtualRepo
 from dstack._internal.core.models.resources import ComputeCapability, Memory, Range
+from dstack._internal.core.models.resources import CPUSpec as CPU
 from dstack._internal.core.models.resources import DiskSpec as Disk
 from dstack._internal.core.models.resources import GPUSpec as GPU
 from dstack._internal.core.models.resources import ResourcesSpec as Resources

dstack/api/server/_fleets.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Union
 from pydantic import parse_obj_as
 
 from dstack._internal.core.models.fleets import ApplyFleetPlanInput, Fleet, FleetPlan, FleetSpec
+from dstack._internal.core.models.instances import Instance
 from dstack._internal.server.schemas.fleets import (
     ApplyFleetPlanRequest,
     CreateFleetRequest,
@@ -83,9 +84,24 @@ def _get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
     spec_excludes = _get_fleet_spec_excludes(plan_input.spec)
     if spec_excludes:
         apply_plan_excludes["spec"] = apply_plan_excludes
+    current_resource = plan_input.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        if all(map(_should_exclude_instance_cpu_arch, current_resource.instances)):
+            current_resource_excludes["instances"] = {
+                "__all__": {"instance_type": {"resources": {"cpu_arch"}}}
+            }
     return {"plan": apply_plan_excludes}
 
 
+def _should_exclude_instance_cpu_arch(instance: Instance) -> bool:
+    try:
+        return instance.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True
+
+
 def _get_create_fleet_excludes(fleet_spec: FleetSpec) -> Dict:
     create_fleet_excludes = {}
     spec_excludes = _get_fleet_spec_excludes(fleet_spec)

dstack/api/server/_runs.py

@@ -7,6 +7,7 @@ from pydantic import parse_obj_as
 from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.runs import (
     ApplyRunPlanInput,
+    JobSubmission,
     Run,
     RunPlan,
     RunSpec,
@@ -96,13 +97,57 @@ def _get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]:
     run_spec_excludes = _get_run_spec_excludes(plan.run_spec)
     if run_spec_excludes is not None:
         apply_plan_excludes["run_spec"] = run_spec_excludes
-    if plan.current_resource is not None:
-        apply_plan_excludes["current_resource"] = {
-            "run_spec": _get_run_spec_excludes(plan.current_resource.run_spec)
+    current_resource = plan.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        current_resource_excludes["run_spec"] = _get_run_spec_excludes(current_resource.run_spec)
+        job_submissions_excludes = {}
+        current_resource_excludes["jobs"] = {
+            "__all__": {"job_submissions": {"__all__": job_submissions_excludes}}
         }
+        job_submissions = [js for j in current_resource.jobs for js in j.job_submissions]
+        if all(map(_should_exclude_job_submission_jpd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_provisioning_data"] = {
+                "instance_type": {"resources": {"cpu_arch"}}
+            }
+        if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_runtime_data"] = {
+                "offer": {"instance": {"resources": {"cpu_arch"}}}
+            }
+        if all(js.exit_status is None for js in job_submissions):
+            job_submissions_excludes["exit_status"] = True
+        latest_job_submission = current_resource.latest_job_submission
+        if latest_job_submission is not None:
+            latest_job_submission_excludes = {}
+            current_resource_excludes["latest_job_submission"] = latest_job_submission_excludes
+            if _should_exclude_job_submission_jpd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_provisioning_data"] = {
+                    "instance_type": {"resources": {"cpu_arch"}}
+                }
+            if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_runtime_data"] = {
+                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                }
+            if latest_job_submission.exit_status is None:
+                latest_job_submission_excludes["exit_status"] = True
     return {"plan": apply_plan_excludes}
 
 
+def _should_exclude_job_submission_jpd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_provisioning_data.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True
+
+
+def _should_exclude_job_submission_jrd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_runtime_data.offer.instance.resources.cpu_arch is None
+    except AttributeError:
+        return True
+
+
 def _get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[Dict]:
     """
     Excludes new fields when they are not set to keep

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.7"
+__version__ = "0.19.9"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.19.7.dist-info → dstack-0.19.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.7
+Version: 0.19.9
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -49,11 +49,12 @@ Requires-Dist: asyncpg; extra == 'all'
 Requires-Dist: azure-identity>=1.12.0; extra == 'all'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-compute>=29.1.0; extra == 'all'
+Requires-Dist: azure-mgmt-msi>=7.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-network<28.0.0,>=23.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-resource>=22.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == 'all'
 Requires-Dist: backports-entry-points-selectable; extra == 'all'
-Requires-Dist: boto3; extra == 'all'
+Requires-Dist: boto3>=1.38.13; extra == 'all'
 Requires-Dist: botocore; extra == 'all'
 Requires-Dist: datacrunch; extra == 'all'
 Requires-Dist: docker>=6.0.0; extra == 'all'
@@ -89,7 +90,7 @@ Requires-Dist: alembic>=1.10.2; extra == 'aws'
 Requires-Dist: apscheduler<4; extra == 'aws'
 Requires-Dist: asyncpg; extra == 'aws'
 Requires-Dist: backports-entry-points-selectable; extra == 'aws'
-Requires-Dist: boto3; extra == 'aws'
+Requires-Dist: boto3>=1.38.13; extra == 'aws'
 Requires-Dist: botocore; extra == 'aws'
 Requires-Dist: docker>=6.0.0; extra == 'aws'
 Requires-Dist: fastapi; extra == 'aws'
@@ -116,6 +117,7 @@ Requires-Dist: asyncpg; extra == 'azure'
 Requires-Dist: azure-identity>=1.12.0; extra == 'azure'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-compute>=29.1.0; extra == 'azure'
+Requires-Dist: azure-mgmt-msi>=7.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-network<28.0.0,>=23.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-resource>=22.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == 'azure'
@@ -229,7 +231,7 @@ Requires-Dist: alembic>=1.10.2; extra == 'lambda'
 Requires-Dist: apscheduler<4; extra == 'lambda'
 Requires-Dist: asyncpg; extra == 'lambda'
 Requires-Dist: backports-entry-points-selectable; extra == 'lambda'
-Requires-Dist: boto3; extra == 'lambda'
+Requires-Dist: boto3>=1.38.13; extra == 'lambda'
 Requires-Dist: botocore; extra == 'lambda'
 Requires-Dist: docker>=6.0.0; extra == 'lambda'
 Requires-Dist: fastapi; extra == 'lambda'
@@ -336,24 +338,27 @@ orchestration for ML teams across top clouds and on-prem clusters.
 
 #### Accelerators
 
-`dstack` supports `NVIDIA`, `AMD`, `Google TPU`, and `Intel Gaudi` accelerators out of the box.
+`dstack` supports `NVIDIA`, `AMD`, `Google TPU`, `Intel Gaudi`, and `Tenstorrent` accelerators out of the box.
 
-## Major news ✨
+## Latest news ✨
 
-- [2025/02] [dstack 0.18.41: GPU blocks, Proxy jump, inactivity duration, and more](https://github.com/dstackai/dstack/releases/tag/0.18.41)
-- [2025/01] [dstack 0.18.38: Intel Gaudi](https://github.com/dstackai/dstack/releases/tag/0.18.38)
-- [2025/01] [dstack 0.18.35: Vultr](https://github.com/dstackai/dstack/releases/tag/0.18.35)
-- [2024/12] [dstack 0.18.30: AWS Capacity Reservations and Capacity Blocks](https://github.com/dstackai/dstack/releases/tag/0.18.30)
-- [2024/10] [dstack 0.18.21: Instance volumes](https://github.com/dstackai/dstack/releases/tag/0.18.21)
-- [2024/10] [dstack 0.18.18: Hardware metrics monitoring](https://github.com/dstackai/dstack/releases/tag/0.18.18)
+- [2025/05] [dstack 0.19.8: Nebius clusters, GH200 on Lambda](https://github.com/dstackai/dstack/releases/tag/0.19.8)
+- [2025/04] [dstack 0.19.6: Tenstorrent, Plugins](https://github.com/dstackai/dstack/releases/tag/0.19.6)
+- [2025/04] [dstack 0.19.5: GCP A3 High clusters](https://github.com/dstackai/dstack/releases/tag/0.19.5)
+- [2025/04] [dstack 0.19.3: GCP A3 Mega clusters](https://github.com/dstackai/dstack/releases/tag/0.19.3)
+- [2025/03] [dstack 0.19.0: Prometheus](https://github.com/dstackai/dstack/releases/tag/0.19.0)
 
-## Installation
+## How does it work?
+
+<img src="https://dstack.ai/static-assets/static-assets/images/dstack-architecture-diagram-v8.svg" width="750" />
+
+### Installation
 
 > Before using `dstack` through CLI or API, set up a `dstack` server. If you already have a running `dstack` server, you only need to [set up the CLI](#set-up-the-cli).
 
-### Set up the server
+#### Set up the server
 
-#### (Optional) Configure backends
+##### (Optional) Configure backends
 
 To use `dstack` with cloud providers, configure backends
 via the `~/.dstack/server/config.yml` file.
@@ -363,21 +368,21 @@ For more details on how to configure backends, check [Backends](https://dstack.a
 > For using `dstack` with on-prem servers, create [SSH fleets](https://dstack.ai/docs/concepts/fleets#ssh) 
 > once the server is up.
 
-#### Start the server
+##### Start the server
 
 You can install the server on Linux, macOS, and Windows (via WSL 2). It requires Git and
 OpenSSH.
 
-##### pip
+##### uv
 
 ```shell
-$ pip install "dstack[all]" -U
+$ uv tool install "dstack[all]" -U
 ```
 
-##### uv
+##### pip
 
 ```shell
-$ uv tool install "dstack[all]" -U
+$ pip install "dstack[all]" -U
 ```
 
 Once it's installed, go ahead and start the server.
@@ -390,25 +395,28 @@ The admin token is "bbae0f28-d3dd-4820-bf61-8f4bb40815da"
 The server is running at http://127.0.0.1:3000/
 ```
 
-For more details on server configuration options, see the
+> For more details on server configuration options, see the
 [Server deployment](https://dstack.ai/docs/guides/server-deployment) guide.
 
-### Set up the CLI
+
+<details><summary>Set up the CLI</summary>
+
+#### Set up the CLI
 
 Once the server is up, you can access it via the `dstack` CLI. 
 
 The CLI can be installed on Linux, macOS, and Windows. It requires Git and OpenSSH.
 
-##### pip
+##### uv
 
 ```shell
-$ pip install dstack -U
+$ uv tool install dstack -U
 ```
 
-##### uv
+##### pip
 
 ```shell
-$ uv tool install dstack -U
+$ pip install dstack -U
 ```
 
 To point the CLI to the `dstack` server, configure it
@@ -423,9 +431,9 @@ $ dstack config \
 Configuration is updated at ~/.dstack/config.yml
 ```
 
-## How does it work?
+</details>
 
-### 1. Define configurations
+### Define configurations
 
 `dstack` supports the following configurations:
    
@@ -438,7 +446,7 @@ Configuration is updated at ~/.dstack/config.yml
 
 Configuration can be defined as YAML files within your repo.
 
-### 2. Apply configurations
+### Apply configurations
 
 Apply the configuration either via the `dstack apply` CLI command or through a programmatic API.
 
@@ -450,6 +458,7 @@ out-of-capacity errors, port-forwarding, and more &mdash; across clouds and on-p
 For additional information, see the following links:
 
 * [Docs](https://dstack.ai/docs)
+* [Examples](https://dstack.ai/examples)
 * [Discord](https://discord.gg/u8SmfwPpMd)
 
 ## Contributing