dstack 0.19.13__tar.gz → 0.19.15rc1__tar.gz

{dstack-0.19.13 → dstack-0.19.15rc1}/.github/workflows/docker.yml

@@ -51,8 +51,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-        flavor: ["base", "devel"]
+        flavor: ["base", "devel", "devel-efa"]
+        ubuntu_version: ["22"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -67,7 +67,21 @@ jobs:
         uses: docker/setup-qemu-action@v3
       - name: Build and upload to DockerHub
         run: |
-          docker buildx build --platform linux/amd64 --build-arg FLAVOR=${{ matrix.flavor }} --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/${{ env.BUILD_DOCKER_REPO }}:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1${{ matrix.flavor == 'devel' && '-devel' || '' }} -f base/Dockerfile .
+         if [ "${{ matrix.flavor }}" = "base" ]; then
+            FILE="base/Dockerfile"
+          elif [ "${{ matrix.flavor }}" = "devel" ]; then
+            FILE="base/Dockerfile"
+          else
+            FILE="base/efa/Dockerfile"
+          fi
+          docker buildx build \
+            --platform linux/amd64 \
+            --tag dstackai/${{ env.BUILD_DOCKER_REPO }}:${{ inputs.image_version }}-${{ matrix.flavor }}-ubuntu${{ matrix.ubuntu_version }}.04 \
+            --build-arg FLAVOR=${{ matrix.flavor }} \
+            --build-arg UBUNTU_VERSION=${{ matrix.ubuntu_version }} \
+            --provenance=false \
+            --push \
+            -f $FILE .
 
   build-aws-images:
     needs: build-docker

{dstack-0.19.13 → dstack-0.19.15rc1}/.github/workflows/release.yml

@@ -230,10 +230,12 @@ jobs:
       - name: Upload pip package
         run: |
           VERSION=${GITHUB_REF#refs/tags/}
-          BASE_IMAGE=$(cat src/dstack/version.py | grep base_image)
+          BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ")
+          BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ")
           echo "__version__ = \"$VERSION\"" > src/dstack/version.py
           echo "__is_release__ = True" >> src/dstack/version.py
           echo $BASE_IMAGE >> src/dstack/version.py
+          echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py
           cp README.md src
           uv build
           uv publish --username ${{ secrets.PYPI_USERNAME }} --password ${{ secrets.PYPI_PASSWORD }}   

{dstack-0.19.13 → dstack-0.19.15rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.13
+Version: 0.19.15rc1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack

dstack-0.19.15rc1/docker/base/Dockerfile

@@ -0,0 +1,79 @@
+# syntax = edrevo/dockerfile-plus
+ARG UBUNTU_VERSION
+
+# Build stage
+FROM nvidia/cuda:12.1.1-base-ubuntu${UBUNTU_VERSION}.04 AS builder
+
+ENV NCCL_HOME=/opt/nccl
+ENV CUDA_HOME=/usr/local/cuda
+ENV OPEN_MPI_PATH=/usr/lib/x86_64-linux-gnu/openmpi
+
+# Prerequisites
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}04/x86_64/3bf863cc.pub \
+    && apt-get update --fix-missing \
+    && apt-get upgrade -y \
+    && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
+    && apt-get install -y tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && cuda_version=$(echo ${CUDA_VERSION} | awk -F . '{ print $1"-"$2 }') \
+    && apt-get install -y --no-install-recommends \
+        cuda-libraries-dev-${cuda_version} \
+        cuda-nvcc-${cuda_version} \
+        libhwloc-dev \
+        autoconf \
+        automake \
+        libtool \
+        libopenmpi-dev \
+        git \
+        curl \
+        python3 \
+        build-essential
+
+# NCCL
+
+ARG NCCL_VERSION=2.26.2-1
+
+RUN cd /tmp \
+    && git clone https://github.com/NVIDIA/nccl.git -b v${NCCL_VERSION} \
+    && cd nccl \
+    && make -j$(nproc) src.build BUILDDIR=${NCCL_HOME}
+
+# NCCL tests
+
+RUN cd /opt \
+    && git clone https://github.com/NVIDIA/nccl-tests \
+    && cd nccl-tests \
+    && make -j$(nproc) \
+        MPI=1 \
+        MPI_HOME=${OPEN_MPI_PATH} \
+        CUDA_HOME=${CUDA_HOME} \
+        NCCL_HOME=${NCCL_HOME}
+
+# Final stage
+
+INCLUDE+ base/Dockerfile.common
+
+ENV NCCL_HOME=/opt/nccl
+
+COPY --from=builder ${NCCL_HOME} ${NCCL_HOME}
+COPY --from=builder /opt/nccl-tests/build /opt/nccl-tests/build
+
+ARG FLAVOR
+
+# MPI, NVCC, and /etc/ld.so.conf.d
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        openmpi-bin \
+    && if [ "$FLAVOR" = "devel" ]; then \
+        cuda_version=$(echo ${CUDA_VERSION} | awk -F . '{ print $1"-"$2 }') \
+        && apt-get install -y --no-install-recommends \
+            cuda-libraries-dev-${cuda_version} \
+            cuda-nvcc-${cuda_version} \
+            libhwloc-dev; \
+    fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && echo "${NCCL_HOME}/lib" >> /etc/ld.so.conf.d/nccl.conf \
+    && ldconfig

dstack-0.19.15rc1/docker/base/Dockerfile.common

@@ -0,0 +1,35 @@
+ARG UBUNTU_VERSION
+
+FROM nvidia/cuda:12.1.1-base-ubuntu${UBUNTU_VERSION}.04
+
+ARG _UV_HOME="/opt/uv"
+
+ENV UV_INSTALL_DIR="${_UV_HOME}/bin"
+ENV UV_MANAGED_PYTHON=1
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+ENV PATH="${UV_INSTALL_DIR}:${PATH}"
+
+ENV OMPI_MCA_pml=^cm,ucx
+ENV OMPI_MCA_btl=tcp,self
+ENV OMPI_MCA_btl_tcp_if_exclude=lo,docker0
+ENV NCCL_SOCKET_IFNAME=^docker,lo
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}04/x86_64/3bf863cc.pub \
+    && apt-get update --fix-missing \
+    && apt-get upgrade -y \
+    && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
+    && apt-get install -y tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && apt-get install -y bzip2 ca-certificates curl build-essential git libglib2.0-0 libsm6 libxext6 libxrender1 mercurial openssh-server subversion wget \
+        libibverbs1 ibverbs-providers ibverbs-utils libibverbs-dev infiniband-diags \
+    && rm -rf /var/lib/apt/lists/* \
+    && sed -i "s/.*PasswordAuthentication.*/PasswordAuthentication no/g" /etc/ssh/sshd_config \
+    && mkdir /run/sshd \
+    && mkdir ~/.ssh && chmod 700 ~/.ssh && touch ~/.ssh/authorized_keys \
+    && chmod 600 ~/.ssh/authorized_keys \
+    && rm /etc/ssh/ssh_host_*
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | INSTALLER_NO_MODIFY_PATH=1 sh \
+    && uv python install --preview --default

{dstack-0.19.13/docker → dstack-0.19.15rc1/docker/base}/efa/Dockerfile

@@ -1,15 +1,15 @@
-ARG BASE_IMAGE=dstackai/base:py3.12-0.7-cuda-12.1
+# syntax = edrevo/dockerfile-plus
 
-FROM ${BASE_IMAGE}
+INCLUDE+ base/Dockerfile.common
 
-ENV PREFIX=/usr/local
-ENV CUDA_PATH=/usr/local/cuda
+ENV NCCL_HOME=/usr/local
+ENV CUDA_HOME=/usr/local/cuda
 ENV LIBFABRIC_PATH=/opt/amazon/efa
 ENV OPEN_MPI_PATH=/opt/amazon/openmpi
 ENV PATH="${LIBFABRIC_PATH}/bin:${OPEN_MPI_PATH}/bin:${PATH}"
 ENV LD_LIBRARY_PATH="${OPEN_MPI_PATH}/lib:${LD_LIBRARY_PATH}"
 
-# prerequisites
+# Prerequisites
 
 RUN cuda_version=$(echo ${CUDA_VERSION} | awk -F . '{ print $1"-"$2 }') \
     && apt-get update \
@@ -19,61 +19,58 @@ RUN cuda_version=$(echo ${CUDA_VERSION} | awk -F . '{ print $1"-"$2 }') \
         libhwloc-dev \
         autoconf \
         automake \
-        libtool
+        libtool \
+    && rm -rf /var/lib/apt/lists/*
 
 # EFA
 
 ARG EFA_VERSION=1.38.1
 
-RUN cd $HOME \
+RUN cd /tmp \
+    && apt-get update \
     && curl -O https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-${EFA_VERSION}.tar.gz \
     && tar -xf aws-efa-installer-${EFA_VERSION}.tar.gz \
     && cd aws-efa-installer \
-    && ./efa_installer.sh -y --skip-kmod -g
+    && ./efa_installer.sh -y --skip-kmod -g \
+    && rm -rf /tmp/aws-efa-installer /var/lib/apt/lists/*
 
 # NCCL
 
 ARG NCCL_VERSION=2.26.2-1
 
-RUN cd $HOME \
+RUN cd /tmp \
     && git clone https://github.com/NVIDIA/nccl.git -b v${NCCL_VERSION} \
     && cd nccl \
-    && make -j$(nproc) src.build BUILDDIR=${PREFIX}
+    && make -j$(nproc) src.build BUILDDIR=${NCCL_HOME} \
+    && rm -rf /tmp/nccl
 
 # AWS OFI NCCL
 
 ARG OFI_VERSION=1.14.0
 
-RUN cd $HOME \
+RUN cd /tmp \
     && git clone https://github.com/aws/aws-ofi-nccl.git -b v${OFI_VERSION} \
     && cd aws-ofi-nccl \
     && ./autogen.sh \
     && ./configure \
-        --with-cuda=${CUDA_PATH} \
+        --with-cuda=${CUDA_HOME} \
         --with-libfabric=${LIBFABRIC_PATH} \
         --with-mpi=${OPEN_MPI_PATH} \
-        --with-cuda=${CUDA_PATH} \
-        --with-nccl=${PREFIX} \
+        --with-cuda=${CUDA_HOME} \
+        --with-nccl=${NCCL_HOME} \
         --disable-tests \
-        --prefix=${PREFIX} \
-    && make -j$(numproc) \
-    && make install
+        --prefix=${NCCL_HOME} \
+    && make -j$(nproc) \
+    && make install \
+    && rm -rf /tmp/aws-ofi-nccl /var/lib/apt/lists/*
 
 # NCCL Tests
 
-RUN cd $HOME \
+RUN cd /opt \
     && git clone https://github.com/NVIDIA/nccl-tests \
     && cd nccl-tests \
-    && make -j$(numproc) \
+    && make -j$(nproc) \
         MPI=1 \
         MPI_HOME=${OPEN_MPI_PATH} \
-        CUDA_HOME=${CUDA_PATH} \
-        NCCL_HOME=${PREFIX}
-
-ARG BUILD_DATE
-ARG IMAGE_NAME
-ARG DSTACK_REVISION
-
-LABEL org.opencontainers.image.title="${IMAGE_NAME}"
-LABEL org.opencontainers.image.version="${EFA_VERSION}-${DSTACK_REVISION}"
-LABEL org.opencontainers.image.created="${BUILD_DATE}"
+        CUDA_HOME=${CUDA_HOME} \
+        NCCL_HOME=${NCCL_HOME}

{dstack-0.19.13 → dstack-0.19.15rc1}/docs/docs/concepts/fleets.md

@@ -70,7 +70,7 @@ This ensures all instances are provisioned with optimal inter-node connectivity.
     Note, EFA requires the `public_ips` to be set to `false` in the `aws` backend configuration.
     Otherwise, instances are only connected by the default VPC subnet.
 
-    Refer to the [EFA](../../blog/posts/efa.md) example for more details.
+    Refer to the [EFA](../../examples/clusters/efa/index.md) example for more details.
 
 ??? info "GCP"
     When you create a cloud fleet with GCP, for the A3 Mega and A3 High instance types, [GPUDirect-TCPXO and GPUDirect-TCPX :material-arrow-top-right-thin:{ .external }](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx-autopilot){:target="_blank"} networking is automatically configured.

{dstack-0.19.13 → dstack-0.19.15rc1}/docs/docs/guides/clusters.md

@@ -22,7 +22,7 @@ For cloud fleets, fast interconnect is currently supported only on the `aws`, `g
     
     !!! info "Backend configuration"    
         Note, EFA requires the `public_ips` to be set to `false` in the `aws` backend configuration.
-        Refer to the [EFA](../../blog/posts/efa.md) example for more details.
+        Refer to the [EFA](../../examples/clusters/efa/index.md) example for more details.
 
 === "GCP"
     When you create a cloud fleet with GCP, for the A3 Mega and A3 High instance types, [GPUDirect-TCPXO and GPUDirect-TCPX :material-arrow-top-right-thin:{ .external }](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx-autopilot){:target="_blank"} networking is automatically configured.

{dstack-0.19.13 → dstack-0.19.15rc1}/docs/docs/guides/metrics.md

@@ -176,3 +176,21 @@ telemetry, and more.
     | `dstack_run_type`     | *string*  | Run configuration type | `task`, `dev-environment`              |
     | `dstack_backend`      | *string*  | Backend                | `aws`, `runpod`                        |
     | `dstack_gpu`          | *string?* | GPU name               | `H100`                                 |
+
+### Server health metrics
+
+These are operational metrics to monitor the health of the dstack server. For now, these only include HTTP metrics, but more will be added later.
+
+=== "Metrics"
+    | Name                                     | Type      | Description                       | Examples     |
+    |------------------------------------------|-----------|-----------------------------------|--------------|
+    | `dstack_server_requests_total` | *counter* | Total number of HTTP requests | `100.0` |
+    | `dstack_server_request_duration_seconds` | *histogram*   | HTTP request duration in seconds  | `1.0`|
+
+=== "Labels"
+    | Name                   | Type      | Description   | Examples                               |
+    |------------------------|-----------|:--------------|----------------------------------------|
+    | `method`  | *string*  | HTTP method  | `POST`                                 |
+    | `endpoint`    | *string* | Endpoint path    | `/api/project/main/repos/get`                             |
+    | `http_status`      | *string* | HTTP status code      | `200` |
+    | `project_name` | *string?*  | Project name  | `main`                           |

{dstack-0.19.13 → dstack-0.19.15rc1}/docs/examples.md

@@ -103,7 +103,7 @@ hide:
     <a href="/examples/clusters/a3mega"
        class="feature-cell sky">
         <h3>
-            A3 Mega
+            GCP A3 Mega
         </h3>
 
         <p>
@@ -113,13 +113,23 @@ hide:
     <a href="/examples/clusters/a3high"
        class="feature-cell sky">
         <h3>
-            A3 High
+            GCP A3 High
         </h3>
 
         <p>
             Set up GCP A3 High clusters with optimized networking
         </p>
     </a>
+        <a href="/examples/clusters/efa"
+       class="feature-cell sky">
+        <h3>
+            AWS EFA
+        </h3>
+
+        <p>
+            Set up AWS EFA clusters with optimized networking
+        </p>
+    </a>
 </div>
 
 ## Inference

dstack-0.19.15rc1/examples/clusters/efa/README.md

@@ -0,0 +1,198 @@
+# AWS EFA
+
+In this guide, we’ll walk through how to run high-performance distributed training on AWS using [Amazon Elastic Fabric Adapter (EFA) :material-arrow-top-right-thin:{ .external }](https://aws.amazon.com/hpc/efa/){:target="_blank"} with `dstack`.
+
+## Overview
+
+EFA is a network interface for Amazon EC2 that enables low-latency, high-bandwidth inter-node communication — essential for scaling distributed deep learning. With `dstack`, EFA is automatically enabled when you create fleets with supported instance types.
+
+## Prerequisite
+
+Before you start, make sure the `aws` backend is properly configured.
+
+<div editor-title="~/.dstack/server/config.yml">
+
+```yaml
+projects:
+- name: main
+  backends:
+    - type: aws
+      creds:
+        type: default
+      regions: ["us-west-2"]
+
+      public_ips: false
+      vpc_name: my-custom-vpc
+```
+
+</div>
+
+!!! info "Multiple network interfaces"
+    To use P4, P5, or P6 instances, set `public_ips` to `false` — this allows AWS to attach multiple network interfaces for EFA. In this case, the `dstack` server can reach your VPC’s private subnets.
+
+!!! info "VPC"
+    If you use a custom VPC, verify that it permits all internal traffic between nodes for EFA to function properly
+
+## Create a fleet
+
+Once your backend is ready, define a fleet configuration.
+
+<div editor-title="examples/clusters/efa/fleet.dstack.yml">
+    
+    ```yaml
+    type: fleet
+    name: my-efa-fleet
+    
+    nodes: 2
+    placement: cluster
+    
+    resources:
+      gpu: H100:8
+    ```
+    
+</div>
+
+Provision the fleet with `dstack apply`:
+
+<div class="termy">
+
+```shell
+$ dstack apply -f examples/clusters/efa/fleet.dstack.yml
+
+Provisioning...
+---> 100%
+
+ FLEET         INSTANCE  BACKEND          INSTANCE TYPE  GPU          PRICE   STATUS  CREATED 
+ my-efa-fleet  0         aws (us-west-2)  p4d.24xlarge   H100:8:80GB  $98.32  idle    3 mins ago      
+               1         aws (us-west-2)  p4d.24xlarge   $98.32  idle    3 mins ago    
+```
+
+</div>
+
+??? info "Instance types"
+    `dstack` selects suitable instances automatically, but not
+    [all types support EFA :material-arrow-top-right-thin:{ .external }](https://aws.amazon.com/hpc/efa/){:target="_blank"}.
+    To enforce EFA, you can specify `instance_types` explicitly:
+
+    ```yaml
+    type: fleet
+    name: my-efa-fleet
+    
+    nodes: 2
+    placement: cluster
+    
+    resources:
+      gpu: L4
+
+    instance_types: ["g6.8xlarge"] # If not specified, g6.xlarge is used (won't have EFA)
+    ```
+      
+## Run NCCL tests
+
+To confirm that EFA is working, run NCCL tests:
+
+<div editor-title="examples/clusters/nccl-tests/.dstack.yml">
+
+```yaml
+type: task
+name: nccl-tests
+
+nodes: 2
+
+startup_order: workers-first
+stop_criteria: master-done
+
+env:
+  - NCCL_DEBUG=INFO
+commands:
+  - |
+    if [ $DSTACK_NODE_RANK -eq 0 ]; then
+      mpirun \
+        --allow-run-as-root \
+        --hostfile $DSTACK_MPI_HOSTFILE \
+        -n $DSTACK_GPUS_NUM \
+        -N $DSTACK_GPUS_PER_NODE \
+        --bind-to none \
+        all_reduce_perf -b 8 -e 8G -f 2 -g 1
+    else
+      sleep infinity
+    fi
+
+resources:
+  gpu: 1..8
+  shm_size: 16GB
+```
+
+</div>
+
+Run it with `dstack apply`:
+
+<div class="termy">
+
+```shell
+$ dstack apply -f examples/clusters/nccl-tests/.dstack.yml
+
+Provisioning...
+---> 100%
+```
+
+</div>
+
+!!! info "Docker image"
+    You can use your own container by setting `image`. If omitted, `dstack` uses its default image with drivers, NCCL tests, and tools pre-installed.
+
+## Run distributed training
+
+Here’s an example using `torchrun` for a simple multi-node PyTorch job:
+
+<div editor-title="examples/distributed-training/torchrun/.dstack.yml">
+
+```yaml
+type: task
+name: train-distrib
+
+nodes: 2
+
+python: 3.12
+env:
+  - NCCL_DEBUG=INFO
+commands:
+  - git clone https://github.com/pytorch/examples.git pytorch-examples
+  - cd pytorch-examples/distributed/ddp-tutorial-series
+  - uv pip install -r requirements.txt
+  - |
+    torchrun \
+      --nproc-per-node=$DSTACK_GPUS_PER_NODE \
+      --node-rank=$DSTACK_NODE_RANK \
+      --nnodes=$DSTACK_NODES_NUM \
+      --master-addr=$DSTACK_MASTER_NODE_IP \
+      --master-port=12345 \
+      multinode.py 50 10
+
+resources:
+  gpu: 1..8
+  shm_size: 16GB
+```
+
+</div>
+
+Provision and launch it via `dstack apply`.
+
+<div class="termy">
+
+```shell
+$ dstack apply -f examples/distributed-training/torchrun/.dstack.yml
+
+Provisioning...
+---> 100%
+```
+
+</div>
+
+Instead of setting `python`, you can specify your own Docker image using `image`. Make sure that the image is properly configured for EFA.
+
+!!! info "What's next"
+    1. Learn more about [distributed tasks](https://dstack.ai/docs/concepts/tasks#distributed-tasks) 
+    2. Check [dev environments](https://dstack.ai/docs/concepts/dev-environments),
+       [services](https://dstack.ai/docs/concepts/services), and [fleets](https://dstack.ai/docs/concepts/fleets)
+    3. Read the [Clusters](https://dstack.ai/docs/guides/clusters) guide

{dstack-0.19.13 → dstack-0.19.15rc1}/examples/clusters/nccl-tests/.dstack.yml

@@ -5,12 +5,9 @@ nodes: 2
 startup_order: workers-first
 stop_criteria: master-done
 
-# This image comes with MPI and NCCL tests pre-built
-image: dstackai/efa
 env:
   - NCCL_DEBUG=INFO
 commands:
-  - cd /root/nccl-tests/build
   - |
     if [ $DSTACK_NODE_RANK -eq 0 ]; then
       mpirun \
@@ -18,13 +15,12 @@ commands:
         --hostfile $DSTACK_MPI_HOSTFILE \
         -n $DSTACK_GPUS_NUM \
         -N $DSTACK_GPUS_PER_NODE \
-        --mca btl_tcp_if_exclude lo,docker0 \
         --bind-to none \
-        ./all_reduce_perf -b 8 -e 8G -f 2 -g 1
+        /opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1
     else
       sleep infinity
     fi
 
 resources:
-  gpu: nvidia:4:16GB
+  gpu: nvidia:1..8
   shm_size: 16GB

{dstack-0.19.13 → dstack-0.19.15rc1}/examples/clusters/nccl-tests/README.md

@@ -13,14 +13,13 @@ type: task
 name: nccl-tests
 
 nodes: 2
+
 startup_order: workers-first
 stop_criteria: master-done
 
-image: dstackai/efa
 env:
   - NCCL_DEBUG=INFO
 commands:
-  - cd /root/nccl-tests/build
   - |
     if [ $DSTACK_NODE_RANK -eq 0 ]; then
       mpirun \
@@ -28,15 +27,14 @@ commands:
         --hostfile $DSTACK_MPI_HOSTFILE \
         -n $DSTACK_GPUS_NUM \
         -N $DSTACK_GPUS_PER_NODE \
-        --mca btl_tcp_if_exclude lo,docker0 \
         --bind-to none \
-        ./all_reduce_perf -b 8 -e 8G -f 2 -g 1
+        /opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1
     else
       sleep infinity
     fi
 
 resources:
-  gpu: nvidia:4:16GB
+  gpu: nvidia:1..8
   shm_size: 16GB
 ```
 

{dstack-0.19.13 → dstack-0.19.15rc1}/examples/distributed-training/torchrun/.dstack.yml

@@ -1,7 +1,6 @@
 type: task
 name: train-distrib
 
-# The size of the cluster
 nodes: 2
 
 python: 3.12
@@ -21,6 +20,5 @@ commands:
       multinode.py 50 10
 
 resources:
-  gpu: 24GB:1..2
-  # Uncomment if using multiple GPUs
-  #shm_size: 24GB
+  gpu: 1..8
+  shm_size: 16GB

{dstack-0.19.13 → dstack-0.19.15rc1}/frontend/src/pages/User/Details/Billing/index.tsx

@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useParams, useSearchParams } from 'react-router-dom';
 
-import { Box, Button, Container, Header, Hotspot, Loader, Modal, SpaceBetween } from 'components';
+import { Box, Button, Header, Hotspot, Loader, Modal, SpaceBetween } from 'components';
 import { PermissionGuard } from 'components/PermissionGuard';
 import { HotspotIds } from 'layouts/AppLayout/TutorialPanel/constants';
 
@@ -102,7 +102,9 @@ export const Billing: React.FC = () => {
 
     return (
         <SpaceBetween size="l">
-            <Container header={<Header variant="h2">{t('billing.balance')}</Header>}>
+            <div>
+                <Header variant="h2">{t('billing.balance')}</Header>
+
                 {isLoading && <Loader />}
 
                 {data && (
@@ -126,7 +128,7 @@ export const Billing: React.FC = () => {
                         </SpaceBetween>
                     </SpaceBetween>
                 )}
-            </Container>
+            </div>
 
             <Payments
                 payments={data?.billing_history ?? []}

{dstack-0.19.13 → dstack-0.19.15rc1}/frontend/src/pages/User/Details/Payments/index.tsx

@@ -54,6 +54,7 @@ export const Payments: React.FC<IProps> = ({ payments, emptyMessageContent, isLo
         <Table
             {...collectionProps}
             columnDefinitions={columns}
+            variant="borderless"
             items={items}
             loading={isLoading}
             loadingText={t('common.loading')}