dstack 0.22.2__tar.gz → 0.22.3__tar.gz

{dstack-0.22.2 → dstack-0.22.3}/.github/workflows/vm-images.yml

@@ -19,6 +19,15 @@ on:
         description: "Build Azure images"
         type: boolean
         default: true
+      azure_variant:
+        description: "Azure image variants to build"
+        type: choice
+        options: ["all", "standard", "cuda", "grid"]
+        default: "all"
+      azure_new_gallery_version:
+        description: "Rebuild Azure images that already exist and publish them as a new gallery version"
+        type: boolean
+        default: false
       build_gcp:
         description: "Build GCP images"
         type: boolean
@@ -72,7 +81,7 @@ jobs:
       VERSION: ${{ github.run_number }}
     strategy:
       matrix:
-        variant: ["", "-cuda", "-grid"]
+        variant: ${{ fromJSON('{"all":["","-cuda","-grid"],"standard":[""],"cuda":["-cuda"],"grid":["-grid"]}')[inputs.azure_variant] }}
     steps:
       - uses: actions/checkout@v4
       - uses: Azure/login@v2
@@ -86,13 +95,13 @@ jobs:
           chmod +x packer
       - name: Run packer
         run: |
-          ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json
+          ./packer build ${{ inputs.azure_new_gallery_version && '-force' || '' }} -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json
       - name: Publish azure image
         if: ${{ !inputs.staging }}
         run: |
           IMAGE_DEFINITION=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
           IMAGE_NAME=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
-          ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME
+          ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME ${{ inputs.azure_new_gallery_version }}
 
   build-gcp-images:
     if: inputs.build_gcp

{dstack-0.22.2 → dstack-0.22.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dstack
-Version: 0.22.2
+Version: 0.22.3
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -22,7 +22,7 @@ Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt==0.1.31
+Requires-Dist: gpuhunt==0.1.33
 Requires-Dist: ignore-python>=0.2.0
 Requires-Dist: jsonschema
 Requires-Dist: packaging
@@ -70,6 +70,7 @@ Requires-Dist: google-cloud-logging>=2.0.0; extra == 'all'
 Requires-Dist: google-cloud-storage>=2.0.0; extra == 'all'
 Requires-Dist: google-cloud-tpu>=1.18.3; extra == 'all'
 Requires-Dist: grpcio>=1.81.0; extra == 'all'
+Requires-Dist: httpx2>=2.13.1; extra == 'all'
 Requires-Dist: httpx>=0.28.0; extra == 'all'
 Requires-Dist: jinja2; extra == 'all'
 Requires-Dist: kubernetes; extra == 'all'
@@ -96,6 +97,7 @@ Requires-Dist: sqlalchemy[asyncio]<2.1,>=2.0.0; extra == 'all'
 Requires-Dist: starlette>=0.26.0; extra == 'all'
 Requires-Dist: uvicorn; extra == 'all'
 Requires-Dist: uvicorn[standard]; extra == 'all'
+Requires-Dist: uvloop>=0.18.0; (sys_platform != 'win32' and platform_python_implementation != 'PyPy') and extra == 'all'
 Requires-Dist: verda>=1.23.0; extra == 'all'
 Requires-Dist: watchfiles; extra == 'all'
 Provides-Extra: aws
@@ -229,10 +231,12 @@ Provides-Extra: gateway
 Requires-Dist: aiocache; extra == 'gateway'
 Requires-Dist: aiorwlock; extra == 'gateway'
 Requires-Dist: fastapi; extra == 'gateway'
+Requires-Dist: httpx2>=2.13.1; extra == 'gateway'
 Requires-Dist: httpx>=0.28.0; extra == 'gateway'
 Requires-Dist: jinja2; extra == 'gateway'
 Requires-Dist: starlette>=0.26.0; extra == 'gateway'
 Requires-Dist: uvicorn; extra == 'gateway'
+Requires-Dist: uvloop>=0.18.0; (sys_platform != 'win32' and platform_python_implementation != 'PyPy') and extra == 'gateway'
 Provides-Extra: gcp
 Requires-Dist: aiocache; extra == 'gcp'
 Requires-Dist: aiorwlock; extra == 'gcp'

{dstack-0.22.2 → dstack-0.22.3}/mkdocs/docs/concepts/backends.md

@@ -976,24 +976,29 @@ projects:
 </div>
 
 ??? info "Required permissions"
-    The API key must have the following roles assigned:
+    The API key requires the `owner` role for your user and the `operator` and `user` roles for the team specified in `team_handle`.
 
-    * **Owner role for the user** - Required for creating and managing SSH keys
-    * **Operator role for the team** - Required for managing virtual machines within the team
+??? info "Bare metal"
+    Bare metal servers (`bm-mi300x-8`) are experimental and disabled by default. To enable them, set `bare_metal: true` in the backend settings:
 
-??? info "Pricing"
-    `dstack` shows the hourly price for Hot Aisle instances. Some instances also require an upfront payment for a minimum reservation period, which is usually a few hours. You will be charged for the full minimum period even if you stop the instance early.
-
-    See the Hot Aisle API for the minimum reservation period for each instance type:
-
-    <div class="termy">
+    <div editor-title="~/.dstack/server/config.yml">
 
-    ```shell
-    $ curl -H "Authorization: Token $API_KEY" https://admin.hotaisle.app/api/teams/$TEAM_HANDLE/virtual_machines/available/ | jq ".[] | {gpus: .Specs.gpus, MinimumReservationMinutes}"
+    ```yaml
+    projects:
+    - name: main
+      backends:
+        - type: hotaisle
+          team_handle: hotaisle-team-handle
+          creds:
+            type: api_key
+            api_key: 9c27a4bb7a8e472fae12ab34.3f2e3c1db75b9a0187fd2196c6b3e56d2b912e1c439ba08d89e7b6fcd4ef1d3f
+          bare_metal: true
     ```
 
     </div>
 
+    Bare metal servers are prepaid for 8 hours. To use the full period, use a fleet with a fixed number of [`nodes`](../concepts/fleets.md#nodes), or at least set [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) to cover it. Within the period, `dstack` doesn't terminate them but stops tracking them — terminate them manually in Hot Aisle. After the period, `dstack` terminates them automatically.
+
 ### JarvisLabs
 
 Log into your [JarvisLabs](https://cloud.jarvislabs.ai/) account and create an API key.

{dstack-0.22.2 → dstack-0.22.3}/mkdocs/docs/examples/inference/vllm.md

@@ -144,7 +144,7 @@ groups:
   - replicas: 1
     python: "3.12"
     commands:
-      - pip install smg
+      - pip install smg==1.11.0
       - |
         smg launch \
           --pd-disaggregation \
@@ -162,37 +162,47 @@ groups:
     scaling:
       metric: rps
       target: 3
-    image: ghcr.io/lightseekorg/smg:1.4.1-vllm-v0.18.0
+    image: ghcr.io/smg-project/smg:1.11.0-vllm-v0.27.1
     commands:
+      - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP
       - |
         python3 -m vllm.entrypoints.grpc_server \
           --model "$MODEL_ID" \
           --host 0.0.0.0 \
           --port 8000 \
+          --max-model-len 65536 \
           --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}'
     resources:
       gpu: H200
+      disk: 200GB..
 
   - replicas: 1..8
     scaling:
       metric: rps
       target: 2
-    image: ghcr.io/lightseekorg/smg:1.4.1-vllm-v0.18.0
+    image: ghcr.io/smg-project/smg:1.11.0-vllm-v0.27.1
     commands:
+      - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP
       - |
         python3 -m vllm.entrypoints.grpc_server \
           --model "$MODEL_ID" \
           --host 0.0.0.0 \
           --port 8000 \
+          --max-model-len 65536 \
           --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}'
     resources:
       gpu: H200
+      disk: 200GB..
 
 port: 8000
 ```
 
 </div>
 
+Use the same SMG version for the router and the worker images: the worker image bundles the SMG gRPC servicer, which must return the KV transfer parameters that the router relays from prefill to decode.
+
+`VLLM_NIXL_SIDE_CHANNEL_HOST` sets the address of the NIXL handshake listener, which the prefill worker also passes to the decode worker. It defaults to `localhost`, in which case the decode worker connects to itself instead of the prefill worker. `DSTACK_MASTER_NODE_IP` is the internal IP of the replica's instance.
+
 > To use the [Mooncake Transfer](https://github.com/kvcache-ai/Mooncake) backend, set `"kv_connector": "MooncakeConnector"` in `--kv-transfer-config`.
 
 Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics are coming soon.
@@ -202,6 +212,23 @@ Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics
 
     While the prefill and decode replicas run on GPUs, the router replica requires a CPU instance in the same cluster.
 
+    For production, use instances with RDMA (InfiniBand or RoCE). Without RDMA, NIXL transfers the KV cache over TCP, which works but may be slower than recomputing the prompt on the decode worker.
+
+??? info "Instances without RDMA"
+    If an instance has both a private and a public network interface and no RDMA, UCX (the default NIXL transport) may advertise the public address, which may be unreachable from other instances. In this case, restrict UCX to the interface that has the instance's internal IP by adding the following command before starting the worker:
+
+    ```yaml
+    - |
+      export UCX_NET_DEVICES=$(python3 -c '
+      import os, psutil
+      ip = os.environ["DSTACK_MASTER_NODE_IP"]
+      ifaces = psutil.net_if_addrs().items()
+      print(next(name for name, addrs in ifaces if any(a.address == ip for a in addrs)))
+      ')
+    ```
+
+    Don't use this on instances with RDMA, as it prevents UCX from using RDMA devices.
+
 ## What's next?
 
 1. Read about [services](../../concepts/services.md) and [gateways](../../concepts/gateways.md)

{dstack-0.22.2 → dstack-0.22.3}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "python-multipart>=0.0.16",
     "filelock",
     "psutil",
-    "gpuhunt==0.1.31",
+    "gpuhunt==0.1.33",
     "argcomplete>=3.5.0",
     "ignore-python>=0.2.0",
     "apscheduler<4",
@@ -217,9 +217,13 @@ gateway = [
     "fastapi",
     "starlette>=0.26.0",
     "uvicorn",
+    # Supervise long-lived SSH children without a waiting thread for each replica.
+    "uvloop>=0.18.0; sys_platform != 'win32' and platform_python_implementation != 'PyPy'",
     "aiorwlock",
     "aiocache",
     "httpx>=0.28.0",
+    # Gateway authorization needs the pool cancellation and recovery fixes in httpx2 (#4338).
+    "httpx2>=2.13.1",
     "jinja2",
 ]
 server = [

{dstack-0.22.2 → dstack-0.22.3}/scripts/packer/provisioners/install-nvidia-grid-driver-for-azure.sh

@@ -8,7 +8,7 @@ sudo apt-get update
 sudo DEBIAN_FRONTEND=noninteractive apt-get install build-essential linux-azure -y
 
 wget --no-verbose -O NVIDIA-Linux-x86_64-grid.run \
-    https://download.microsoft.com/download/2a04ca6a-9eec-40d9-9564-9cdea1ab795f/NVIDIA-Linux-x86_64-570.211.01-grid-azure.run
+    https://download.microsoft.com/download/4efc1974-7b6f-431f-ab2d-b8d99bb8a512/NVIDIA-Linux-x86_64-580.178.04-grid-azure.run
 chmod +x NVIDIA-Linux-x86_64-grid.run
 sudo ./NVIDIA-Linux-x86_64-grid.run --silent --disable-nouveau
 rm NVIDIA-Linux-x86_64-grid.run

{dstack-0.22.2 → dstack-0.22.3}/scripts/publish_azure_image.sh

@@ -1,7 +1,11 @@
 #!/bin/bash
 
+set -o pipefail
+
 image_definition=$1
 image_name=$2
+# If true, publish a new gallery version when the image definition already has versions
+new_gallery_version=${3:-false}
 subscription_id=$AZURE_SUBSCRIPTION_ID
 
 if [ -z "$subscription_id" ]; then
@@ -37,17 +41,39 @@ function create_image_definition() {
         --features DiskControllerTypes=SCSI,NVMe
 }
 
+function get_latest_image_version() {
+    az sig image-version list \
+        --resource-group $resource_group \
+        --gallery-name $gallery_name \
+        --gallery-image-definition $image_definition \
+        --query "[].name" \
+        --output tsv |
+        sort -V |
+        tail -1
+}
+
 function create_image_version() {
-    echo Creating image version...
+    echo Creating image version $1...
     az sig image-version create \
         --resource-group $resource_group \
         --gallery-name $gallery_name \
         --gallery-image-definition $image_definition \
-        --gallery-image-version "0.0.1" \
+        --gallery-image-version "$1" \
         --target-regions "australiaeast" "brazilsouth" "canadacentral" "centralindia" "centralus" "eastasia" "eastus" "eastus2" "francecentral" "germanywestcentral" "japaneast" "koreacentral" "northeurope" "norwayeast" "qatarcentral" "southafricanorth" "southcentralus" "southeastasia" "swedencentral" "switzerlandnorth" "uaenorth" "uksouth" "westeurope" "westus2" "westus3" \
         --replica-count 1 \
         --managed-image "/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.Compute/images/${image_name}"
 }
 
 get_image_definition > /dev/null || create_image_definition
-create_image_version
+
+latest_version=$(get_latest_image_version) || exit 1
+if [ -z "$latest_version" ]; then
+    image_version=0.0.1
+elif [ "$new_gallery_version" = true ]; then
+    image_version="${latest_version%.*}.$((${latest_version##*.} + 1))"
+else
+    echo "$image_definition already has gallery version $latest_version." \
+        "Enable azure_new_gallery_version to publish a new one."
+    exit 1
+fi
+create_image_version $image_version

{dstack-0.22.2 → dstack-0.22.3}/skills/dstack/SKILL.md

@@ -172,6 +172,7 @@ Unless you use **Multi-node tasks** (see `### 2. Tasks`) or **Multi-replica serv
 - Prefer giving configurations a `name` property for easier management
 - When configurations need credentials (API keys, tokens), list only env var names in the `env` section (e.g., `- HF_TOKEN`), not values. Recommend storing actual values in a `.envrc` file alongside the configuration, applied via `source .envrc && dstack apply`.
 - `python` and `image` are mutually exclusive in run configurations. If `image` is set, do not set `python`.
+- `commands` items are joined into one shell line with ` && `, so keep multi-line shell such as heredocs inside a single item (a `|` block). A heredoc that ends an item swallows every item after it, and the run exits 0 without running them. The same applies to a trailing `# comment` in an item, which comments out the rest of the line.
 
 ### `files` and `repos` intent policy
 

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/aws/compute.py

@@ -528,6 +528,25 @@ class AWSCompute(
             return False
         return placement_group.configuration.region == instance_offer.region
 
+    def are_placement_groups_compatible_with_reservation(
+        self,
+        instance_offer: InstanceOffer,
+        reservation: str,
+    ) -> bool:
+        # AWS rejects launches into Capacity Blocks that specify a placement group.
+        # Capacity Block instances are already placed close together in EC2 UltraClusters.
+        try:
+            capacity_block = aws_resources.get_reservation(
+                ec2_client=self.session.client("ec2", region_name=instance_offer.region),
+                reservation_id=reservation,
+                is_capacity_block=True,
+                active_only=False,
+            )
+        except botocore.exceptions.ClientError as e:
+            logger.warning("Failed to get reservation %s: %s", reservation, e)
+            return True
+        return capacity_block is None
+
     def create_gateway_replica(
         self,
         configuration: GatewayReplicaConfiguration,

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/aws/resources.py

@@ -678,8 +678,11 @@ def get_reservation(
     instance_count: int = 0,
     instance_types: Optional[List[str]] = None,
     is_capacity_block: bool = False,
+    active_only: bool = True,
 ) -> Optional[Dict[str, Any]]:
-    filters = [{"Name": "state", "Values": ["active"]}]
+    filters = []
+    if active_only:
+        filters.append({"Name": "state", "Values": ["active"]})
     if instance_types:
         filters.append({"Name": "instance-type", "Values": instance_types})
     try:
@@ -707,7 +710,7 @@ def get_reservation(
     if instance_count > 0 and reservation["AvailableInstanceCount"] < instance_count:
         return None
 
-    if is_capacity_block and reservation["ReservationType"] != "capacity-block":
+    if is_capacity_block and reservation.get("ReservationType") != "capacity-block":
         return None
 
     return reservation

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/base/compute.py

@@ -26,7 +26,6 @@ from dstack._internal.core.consts import (
     DSTACK_RUNNER_SSH_PORT,
     DSTACK_SHIM_HTTP_PORT,
 )
-from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.compute_groups import ComputeGroup, ComputeGroupProvisioningData
 from dstack._internal.core.models.gateways import (
     GatewayLoadBalancerConfiguration,
@@ -415,7 +414,7 @@ class ComputeWithCreateInstanceSupport(ABC):
             user=run.user,
             ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
             volumes=volumes,
-            reservation=job.job_spec.requirements.reservation,
+            reservation=requirements.reservation,
             tags=run.run_spec.merged_profile.tags,
         )
         instance_offer = instance_offer.model_copy()
@@ -559,13 +558,20 @@ class ComputeWithPlacementGroupSupport(ABC):
         """
         pass
 
-    def are_placement_groups_compatible_with_reservations(self, backend_type: BackendType) -> bool:
+    def are_placement_groups_compatible_with_reservation(
+        self,
+        instance_offer: InstanceOffer,
+        reservation: str,
+    ) -> bool:
         """
-        Whether placement groups can be used for instances provisioned in reservations.
+        Whether a placement group can be used for an instance provisioned in the reservation.
+
+        May perform API calls.
 
         Arguments:
-            backend_type: matches the backend type of this compute, unless this compute is a proxy
-                for other backends (dstack Sky)
+            instance_offer: the offer to provision. Its backend matches the backend type of this
+                compute, unless this compute is a proxy for other backends (dstack Sky)
+            reservation: the reservation to provision the instance in
         """
         return True
 

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/base/offers.py

@@ -30,6 +30,7 @@ SUPPORTED_GPUHUNT_FLAGS = [
     "gcp-dws-calendar-mode",
     "runpod-cpu",
     "runpod-cluster",
+    "hotaisle-bm",
 ]
 
 

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/gcp/compute.py

@@ -594,7 +594,11 @@ class GCPCompute(
     ) -> bool:
         return placement_group.configuration.region == instance_offer.region
 
-    def are_placement_groups_compatible_with_reservations(self, backend_type: BackendType) -> bool:
+    def are_placement_groups_compatible_with_reservation(
+        self,
+        instance_offer: InstanceOffer,
+        reservation: str,
+    ) -> bool:
         # Cannot use our own placement policies when provisioning in a reservation.
         # Instead, we use the placement policy defined in reservation settings.
         return False

{dstack-0.22.2 → dstack-0.22.3}/src/dstack/_internal/core/backends/hotaisle/api_client.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, Optional
 import requests
 
 from dstack._internal.core.backends.base.configurator import raise_invalid_credentials_error
+from dstack._internal.core.errors import BackendError, NoCapacityError
 from dstack._internal.utils.logging import get_logger
 
 API_URL = "https://admin.hotaisle.app/api"
@@ -88,6 +89,38 @@ class HotAisleAPIClient:
             return
         response.raise_for_status()
 
+    def reserve_bare_metal_server(self, specs: Dict[str, Any], description: str) -> Dict[str, Any]:
+        url = f"{API_URL}/teams/{self.team_handle}/bare_metal/"
+        payload = {"specs": specs, "description": description}
+        response = self._make_request("POST", url, json=payload)
+        # 403: the team's bare metal server limit is reached or the API key lacks permissions.
+        # 404: no available server matches the specs, e.g. another team reserved it.
+        if response.status_code in [403, 404]:
+            raise NoCapacityError(response.text)
+        response.raise_for_status()
+        return response.json()
+
+    def get_bare_metal_server(self, server_id: str) -> Dict[str, Any]:
+        url = f"{API_URL}/teams/{self.team_handle}/bare_metal/{server_id}/"
+        response = self._make_request("GET", url)
+        response.raise_for_status()
+        return response.json()
+
+    def release_bare_metal_server(self, server_id: str, force: bool = True) -> None:
+        url = f"{API_URL}/teams/{self.team_handle}/bare_metal/{server_id}/"
+        # force releases even if min reservation time not met
+        params = {"force": "true"} if force else None
+        response = self._make_request("DELETE", url, params=params)
+        if response.status_code == 404:
+            logger.debug("Hot Aisle bare metal server %s not found", server_id)
+            return
+        if response.status_code == 400 and not force:
+            raise BackendError(
+                f"Hot Aisle refused to release bare metal server {server_id}"
+                f" before its minimum reservation period ends: {response.text}"
+            )
+        response.raise_for_status()
+
     def _make_request(
         self,
         method: str,