dstack 0.19.9__tar.gz → 0.19.10__tar.gz

{dstack-0.19.9 → dstack-0.19.10}/.github/workflows/docker.yml

@@ -8,11 +8,15 @@ on:
         description: "Docker image version"
         required: true
       staging:
-        description: "Staging build"
+        description: "Build staging images"
+        type: boolean
+        default: false
+      prepull_staging:
+        description: "Pre-pull staging Docker images"
         type: boolean
         default: false
       build_docker:
-        description: "Build docker images"
+        description: "Build Docker images"
         type: boolean
         default: true
       build_aws:
@@ -34,7 +38,9 @@ on:
 
 env:
   PACKER_VERSION: "1.9.2"
-  BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
+  BUILD_DOCKER_REPO: ${{ inputs.staging && 'base-stgn' || 'base' }}
+  PREPULL_DOCKER_REPO: ${{ inputs.prepull_staging && 'base-stgn' || 'base' }}
+  VM_IMAGE_BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
 
 jobs:
   build-docker:
@@ -46,6 +52,7 @@ jobs:
     strategy:
       matrix:
         python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        flavor: ["base", "devel"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -60,8 +67,7 @@ jobs:
         uses: docker/setup-qemu-action@v3
       - name: Build and upload to DockerHub
         run: |
-          docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1 -f base/Dockerfile .
-          docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --build-arg VERSION=${{ inputs.image_version }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1-devel -f base/devel.Dockerfile .
+          docker buildx build --platform linux/amd64 --build-arg FLAVOR=${{ matrix.flavor }} --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/${{ env.BUILD_DOCKER_REPO }}:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1${{ matrix.flavor == 'devel' && '-devel' || '' }} -f base/Dockerfile .
 
   build-aws-images:
     needs: build-docker
@@ -85,7 +91,7 @@ jobs:
           chmod +x packer
       - name: Run packer
         run: |
-          ./packer build -var-file=versions.json $PROD_VARS -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX aws-image${{ matrix.variant }}.json
+          ./packer build -var-file=versions.json $PROD_VARS -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX aws-image${{ matrix.variant }}.json
         env:
           PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }} # production ? var-file : ''
 
@@ -118,12 +124,12 @@ jobs:
           chmod +x packer
       - name: Run packer
         run: |
-          ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX azure-image${{ matrix.variant }}.json
+          ./packer build -var-file=versions.json -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json
       - name: Publish azure image
         if: ${{ !inputs.staging }}
         run: |
-          IMAGE_DEFINITION=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
-          IMAGE_NAME=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
+          IMAGE_DEFINITION=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
+          IMAGE_NAME=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
           ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME
 
   build-gcp-images:
@@ -156,11 +162,11 @@ jobs:
           chmod +x packer
       - name: Run packer
         run: |
-          ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX gcp-image${{ matrix.variant }}.json
+          ./packer build -var-file=versions.json -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX gcp-image${{ matrix.variant }}.json
       - name: Publish images
         run: |
           IMAGE_VERSION=${IMAGE_VERSION//./-}
-          gcloud compute images add-iam-policy-binding ${BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser'
+          gcloud compute images add-iam-policy-binding ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser'
         env:
           IMAGE_VERSION: ${{ inputs.image_version }}
 
@@ -205,8 +211,9 @@ jobs:
         run: |
           ./packer build \
             -var-file=versions.json \
+            -var image_repo=${{ env.PREPULL_DOCKER_REPO }} \
             -var image_version=${{ inputs.image_version }} \
-            -var build_prefix=$BUILD_PREFIX \
+            -var build_prefix=$VM_IMAGE_BUILD_PREFIX \
             -var oci_compartment_ocid=$OCI_COMPARTMENT \
             -var oci_subnet_ocid=$OCI_SUBNET \
             -var oci_availability_domain=$OCI_AVAILABILITY_DOMAIN \
@@ -221,14 +228,14 @@ jobs:
         if: ${{ !inputs.staging }}
         run: |
           uv run scripts/oci_image_tools.py copy \
-            --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
+            --image ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
             --from $OCI_REGION \
             --compartment $OCI_COMPARTMENT
       - name: Publish image in OCI Marketplace
         if: ${{ !inputs.staging }}
         run: |
           uv run scripts/oci_image_tools.py publish \
-            --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
+            --image ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
             --compartment $OCI_COMPARTMENT \
             --version ${{ inputs.image_version }} \
             --description "Image for running workloads with dstack - https://dstack.ai/" \

{dstack-0.19.9 → dstack-0.19.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.9
+Version: 0.19.10
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -21,7 +21,8 @@ Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt==0.1.5
+Requires-Dist: gpuhunt==0.1.6
+Requires-Dist: humanize>=4.12.3
 Requires-Dist: jsonschema
 Requires-Dist: packaging
 Requires-Dist: paramiko>=3.2.0
@@ -56,6 +57,7 @@ Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == 'all'
 Requires-Dist: backports-entry-points-selectable; extra == 'all'
 Requires-Dist: boto3>=1.38.13; extra == 'all'
 Requires-Dist: botocore; extra == 'all'
+Requires-Dist: cryptography>=44.0.3; extra == 'all'
 Requires-Dist: datacrunch; extra == 'all'
 Requires-Dist: docker>=6.0.0; extra == 'all'
 Requires-Dist: fastapi; extra == 'all'
@@ -71,8 +73,9 @@ Requires-Dist: httpx; extra == 'all'
 Requires-Dist: jinja2; extra == 'all'
 Requires-Dist: kubernetes; extra == 'all'
 Requires-Dist: nebius<0.3,>=0.2.19; (python_version >= '3.10') and extra == 'all'
-Requires-Dist: oci; extra == 'all'
+Requires-Dist: oci>=2.150.0; extra == 'all'
 Requires-Dist: prometheus-client; extra == 'all'
+Requires-Dist: pyopenssl>=23.2.0; extra == 'all'
 Requires-Dist: python-dxf==12.1.0; extra == 'all'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'all'
 Requires-Dist: sentry-sdk[fastapi]; extra == 'all'
@@ -280,13 +283,15 @@ Requires-Dist: alembic>=1.10.2; extra == 'oci'
 Requires-Dist: apscheduler<4; extra == 'oci'
 Requires-Dist: asyncpg; extra == 'oci'
 Requires-Dist: backports-entry-points-selectable; extra == 'oci'
+Requires-Dist: cryptography>=44.0.3; extra == 'oci'
 Requires-Dist: docker>=6.0.0; extra == 'oci'
 Requires-Dist: fastapi; extra == 'oci'
 Requires-Dist: grpcio>=1.50; extra == 'oci'
 Requires-Dist: httpx; extra == 'oci'
 Requires-Dist: jinja2; extra == 'oci'
-Requires-Dist: oci; extra == 'oci'
+Requires-Dist: oci>=2.150.0; extra == 'oci'
 Requires-Dist: prometheus-client; extra == 'oci'
+Requires-Dist: pyopenssl>=23.2.0; extra == 'oci'
 Requires-Dist: python-dxf==12.1.0; extra == 'oci'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'oci'
 Requires-Dist: sentry-sdk[fastapi]; extra == 'oci'
@@ -423,9 +428,9 @@ To point the CLI to the `dstack` server, configure it
 with the server address, user token, and project name:
 
 ```shell
-$ dstack config \
+$ dstack project add \
+    --name main \
     --url http://127.0.0.1:3000 \
-    --project main \
     --token bbae0f28-d3dd-4820-bf61-8f4bb40815da
     
 Configuration is updated at ~/.dstack/config.yml

{dstack-0.19.9 → dstack-0.19.10}/README.md

@@ -107,9 +107,9 @@ To point the CLI to the `dstack` server, configure it
 with the server address, user token, and project name:
 
 ```shell
-$ dstack config \
+$ dstack project add \
+    --name main \
     --url http://127.0.0.1:3000 \
-    --project main \
     --token bbae0f28-d3dd-4820-bf61-8f4bb40815da
     
 Configuration is updated at ~/.dstack/config.yml

dstack-0.19.10/docker/base/Dockerfile

@@ -0,0 +1,27 @@
+ARG FLAVOR
+FROM nvidia/cuda:12.1.1-${FLAVOR}-ubuntu20.04
+
+ARG PYTHON
+ARG _UV_HOME="/opt/uv"
+ENV UV_PYTHON="${PYTHON}"
+ENV UV_INSTALL_DIR="${_UV_HOME}/bin"
+ENV UV_PYTHON_INSTALL_DIR="${_UV_HOME}/python"
+ENV UV_PYTHON_BIN_DIR="${UV_PYTHON_INSTALL_DIR}/bin"
+ENV UV_MANAGED_PYTHON=1
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+ENV PATH="${UV_INSTALL_DIR}:${UV_PYTHON_BIN_DIR}:${PATH}"
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub && \
+    apt-get update --fix-missing && \
+    apt-get upgrade -y && \
+    ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    apt-get install -y tzdata && \
+    dpkg-reconfigure --frontend noninteractive tzdata && \
+    apt-get install -y bzip2 ca-certificates curl build-essential git libglib2.0-0 libsm6 libxext6 libxrender1 mercurial openssh-server subversion wget && \
+    sed -i "s/.*PasswordAuthentication.*/PasswordAuthentication no/g" /etc/ssh/sshd_config && mkdir /run/sshd && \
+    mkdir ~/.ssh && chmod 700 ~/.ssh && touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys && rm /etc/ssh/ssh_host_*
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | INSTALLER_NO_MODIFY_PATH=1 sh && \
+    uv python install --preview --default

{dstack-0.19.9 → dstack-0.19.10}/docker/server/README.md

@@ -29,8 +29,8 @@ with the server address, user token, and project name:
 
 ```shell
 $ pip install dstack
-$ dstack config --url http://127.0.0.1:3000 \
-    --project main \
+$ dstack project add --name main \
+    --url http://127.0.0.1:3000 \
     --token bbae0f28-d3dd-4820-bf61-8f4bb40815da
     
 Configuration is updated at ~/.dstack/config.yml

{dstack-0.19.9 → dstack-0.19.10}/docs/assets/stylesheets/extra.css

@@ -29,6 +29,10 @@
     margin-left: 32px;
 }
 
+[dir=ltr] .md-typeset :is(.md-typeset__scrollwrap,p,h4,h3,.tabbed-set,ul):not(.admonition-title) :is(.admonition,details, .termy) {
+    margin-left: 0;
+}
+
 @media screen and (max-width: 76.1875em) {
     .md-header {
         background-color: rgb(255, 255, 255);
@@ -790,6 +794,7 @@ body {
         display: flex;
     }
 
+    /* Comment to siwtch to sections in sidebar */
     .md-sidebar__inner > .md-nav--primary > .md-nav__list:not(.md-post__meta) > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item:not(.md-nav__item--section) {
         display: none;
     }
@@ -861,7 +866,7 @@ body {
 }
 
 .md-sidebar--primary .md-nav__link, .md-sidebar--post .md-nav__link {
-    padding: 5px 15px 5px;
+    padding: 5px 15px 4px;
     margin-top: 0;
 }
 

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/amd-on-runpod.md

@@ -42,7 +42,7 @@ you can now specify an AMD GPU under `resources`. Below are a few examples.
     Here's an example of a [service](../../docs/concepts/services.md) that deploys
     Llama 3.1 70B in FP16 using [TGI :material-arrow-top-right-thin:{ .external }](https://huggingface.co/docs/text-generation-inference/en/installation_amd){:target="_blank"}.
     
-    <div editor-title="examples/deployment/tgi/amd/service.dstack.yml"> 
+    <div editor-title="examples/inference/tgi/amd/service.dstack.yml"> 
     
     ```yaml
     type: service

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/beyond-kubernetes-2024-recap-and-whats-ahead.md

@@ -110,7 +110,7 @@ efficient manner.
 
 ### NVIDIA
 
-NVIDIA remains the top accelerator supported by `dstack`. Recently, we introduced a [NIM example](../../examples/deployment/nim/index.md) 
+NVIDIA remains the top accelerator supported by `dstack`. Recently, we introduced a [NIM example](../../examples/inference/nim/index.md) 
 for model deployment, and we continue to enhance support for the rest of NVIDIA's ecosystem.
 
 ### AMD

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/dstack-metrics.md

@@ -7,7 +7,7 @@ image: https://dstack.ai/static-assets/static-assets/images/dstack-stats-v2.png
 categories:
   - AMD
   - NVIDIA
-  - Monitoring
+  - Metrics
 ---
 
 # Monitoring essential GPU metrics via CLI

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/dstack-sky.md

@@ -40,8 +40,8 @@ set up with `dstack Sky`.
 <div class="termy">
 
 ```shell
-$ dstack config --url https://sky.dstack.ai \
-    --project my-awesome-project \
+$ dstack project add --name my-awesome-project \
+    --url https://sky.dstack.ai \
     --token ca1ee60b-7b3f-8943-9a25-6974c50efa75
 ```
 

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/metrics-ui.md

@@ -5,7 +5,7 @@ description: "TBA"
 slug: metrics-ui
 image: https://dstack.ai/static-assets/static-assets/images/dstack-metrics-ui-v3-min.png
 categories:
-  - Monitoring
+  - Metrics
   - AMD
   - NVIDIA
 ---
@@ -55,6 +55,6 @@ For persistent storage and long-term access to metrics, we still recommend setti
 metrics from `dstack`.
 
 !!! info "What's next?"
-    1. See [Monitoring](../../docs/guides/monitoring.md)
+    1. See [Metrics](../../docs/guides/metrics.md)
     2. Check [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md)
     3. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"}

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/mpi.md

@@ -86,10 +86,10 @@ resources:
 
 </div>
 
-The first worker node (`DSTACK_NODE_RANK=0`) generates a `hostfile` listing all node IPs and waits until all nodes are
+The master node (`DSTACK_NODE_RANK=0`) generates a `hostfile` listing all node IPs and waits until all nodes are
 reachable via MPI. Once confirmed, it launches the `/root/nccl-tests/build/all_reduce_perf` benchmark across all available GPUs in the cluster.
 
-The other worker nodes remain blocked until they receive a termination signal from the master node via a FIFO pipe.
+Non-master nodes remain blocked until they receive a termination signal from the master node via a FIFO pipe.
 
 With this, now you can use such a task to run both NCCL or RCCL tests on both cloud and SSH fleets, 
 as well as use MPI for other tasks.
@@ -101,5 +101,5 @@ as well as use MPI for other tasks.
 
 !!! info "What's next?"
     1. Learn more about [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md)
-    2. Check the [NCCL tests](../../examples/distributed-training/nccl-tests/index.md) example
-    2. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"}
+    2. Check the [NCCL tests](../../examples/clusters/nccl-tests/index.md) example
+    3. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"}

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/nebius.md

@@ -104,7 +104,7 @@ $ dstack apply -f .dstack.yml
 The new `nebius` backend supports CPU and GPU instances, [fleets](../../docs/concepts/fleets.md), 
 [distributed tasks](../../docs/concepts/tasks.md#distributed-tasks), and more. 
  
-> Support for [network volumes](../../docs/concepts/volumes.md#network-volumes) and accelerated cluster 
+> Support for [network volumes](../../docs/concepts/volumes.md#network) and accelerated cluster 
 interconnects is coming soon.
 
 !!! info "What's next?"

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/prometheus.md

@@ -5,7 +5,7 @@ description: "TBA"
 slug: prometheus
 image: https://dstack.ai/static-assets/static-assets/images/dstack-prometheus-v3.png
 categories:
-  - Monitoring
+  - Metrics
   - NVIDIA
 ---
 
@@ -46,7 +46,7 @@ Overall, `dstack` collects three groups of metrics:
 | **Runs**   | Run metrics include run counters for each user in each project.                                                                                                   |
 | **Jobs**   | A run consists of one or more jobs, each mapped to a container. Job metrics offer insights into execution time, cost, GPU model, NVIDIA DCGM telemetry, and more. |
 
-For a full list of available metrics and labels, check out [Monitoring](../../docs/guides/monitoring.md).
+For a full list of available metrics and labels, check out [Metrics](../../docs/guides/metrics.md).
 
 ??? info "NVIDIA"
     NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends,
@@ -60,7 +60,7 @@ For a full list of available metrics and labels, check out [Monitoring](../../do
     only accessible through the UI and the [`dstack metrics`](dstack-metrics.md) CLI.
 
 !!! info "What's next?"
-    1. See [Metrics](../../docs/guides/monitoring.md)
+    1. See [Metrics](../../docs/guides/metrics.md)
     1. Check [dev environments](../../docs/concepts/dev-environments.md),
        [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md),
        and [fleets](../../docs/concepts/fleets.md)

{dstack-0.19.9 → dstack-0.19.10}/docs/blog/posts/tpu-on-gcp.md

@@ -50,7 +50,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm-
 
 === "Optimum TPU"
 
-    <div editor-title="examples/deployment/optimum-tpu/service.dstack.yml"> 
+    <div editor-title="examples/inference/optimum-tpu/service.dstack.yml"> 
     
     ```yaml
     type: service
@@ -83,7 +83,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm-
     the official Docker image can be used instead of `dstackai/optimum-tpu:llama31`.
 
 === "vLLM"
-    <div editor-title="examples/deployment/vllm/service-tpu.dstack.yml"> 
+    <div editor-title="examples/inference/vllm/service-tpu.dstack.yml"> 
     
     ```yaml
     type: service

{dstack-0.19.9 → dstack-0.19.10}/docs/docs/concepts/fleets.md

@@ -63,25 +63,34 @@ Once the status of instances changes to `idle`, they can be used by dev environm
 
 To ensure instances are interconnected (e.g., for
 [distributed tasks](tasks.md#distributed-tasks)), set `placement` to `cluster`. 
-This ensures all instances are provisioned in the same backend and region with optimal inter-node connectivity
+This ensures all instances are provisioned with optimal inter-node connectivity.
 
 ??? info "AWS"
-    `dstack` automatically enables the Elastic Fabric Adapter for all
-    [EFA-capable instance types :material-arrow-top-right-thin:{ .external }](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-instance-types){:target="_blank"}.
-    If the `aws` backend config has `public_ips: false` set, `dstack` enables the maximum number of interfaces supported by the instance.
-    Otherwise, if instances have public IPs, only one EFA interface is enabled per instance due to AWS limitations.
+    When you create a cloud fleet with AWS, [Elastic Fabric Adapter networking :material-arrow-top-right-thin:{ .external }](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html){:target="_blank"} is automatically configured if it’s supported for the corresponding instance type.
+    Note, EFA requires the `public_ips` to be set to `false` in the `aws` backend configuration.
+    Otherwise, instances are only connected by the default VPC subnet.
+
+    Refer to the [EFA](../../blog/posts/efa.md) example for more details.
+
+??? info "GCP"
+    When you create a cloud fleet with GCP, for the A3 Mega and A3 High instance types, [GPUDirect-TCPXO and GPUDirect-TCPX :material-arrow-top-right-thin:{ .external }](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx-autopilot){:target="_blank"} networking is automatically configured.
+
+    !!! info "Backend configuration"    
+        Note, GPUDirect-TCPXO and GPUDirect-TCPX require `extra_vpcs` to be configured  in the `gcp` backend configuration.
+        Refer to the [A3 Mega](../../examples/clusters/a3mega/index.md) and 
+        [A3 High](../../examples/clusters/a3high/index.md) examples for more details.
 
 ??? info "Nebius"
-    `dstack` automatically creates an [InfiniBand cluster](https://docs.nebius.com/compute/clusters/gpu)
-    if all instances in the fleet support it.
+    When you create a cloud fleet with Nebius, [InfiniBand networking :material-arrow-top-right-thin:{ .external }](https://docs.nebius.com/compute/clusters/gpu){:target="_blank"} is automatically configured if it’s supported for the corresponding instance type.
     Otherwise, instances are only connected by the default VPC subnet.
 
-    An InfiniBand fabric for the cluster is selected automatically.
-    If you prefer to use some specific fabrics, configure them in the
+    An InfiniBand fabric for the cluster is selected automatically. If you prefer to use some specific fabrics, configure them in the
     [backend settings](../reference/server/config.yml.md#nebius).
 
-> The `cluster` placement is supported only for `aws`, `azure`, `gcp`, `nebius`, `oci`, and `vultr`
-> backends.
+The `cluster` placement is supported for `aws`, `azure`, `gcp`, `nebius`, `oci`, and `vultr`
+backends.
+
+> For more details on optimal inter-node connectivity, read the [Clusters](../guides/clusters.md) guide.
 
 #### Resources
 
@@ -254,26 +263,30 @@ Define a fleet configuration as a YAML file in your project directory. The file
 </div>
 
 ??? info "Requirements" 
-    1.&nbsp;Hosts should be pre-installed with Docker.
+    1.&nbsp;Hosts must be pre-installed with Docker.
 
     === "NVIDIA"
-        2.&nbsp;Hosts with NVIDIA GPUs should also be pre-installed with CUDA 12.1 and
+        2.&nbsp;Hosts with NVIDIA GPUs must also be pre-installed with CUDA 12.1 and
         [NVIDIA Container Toolkit :material-arrow-top-right-thin:{ .external }](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
 
     === "AMD"
-        2.&nbsp;Hosts with AMD GPUs should also be pre-installed with AMDGPU-DKMS kernel driver (e.g. via
+        2.&nbsp;Hosts with AMD GPUs must also be pre-installed with AMDGPU-DKMS kernel driver (e.g. via
         [native package manager :material-arrow-top-right-thin:{ .external }](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/native-install/index.html)
         or [AMDGPU installer :material-arrow-top-right-thin:{ .external }](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/amdgpu-install.html).)
 
     === "Intel Gaudi"
-        2.&nbsp;Hosts with Intel Gaudi accelerators should be pre-installed with [Gaudi software and drivers](https://docs.habana.ai/en/latest/Installation_Guide/Driver_Installation.html#driver-installation).
-        This should include the drivers, `hl-smi`, and Habana Container Runtime.
+        2.&nbsp;Hosts with Intel Gaudi accelerators must be pre-installed with [Gaudi software and drivers](https://docs.habana.ai/en/latest/Installation_Guide/Driver_Installation.html#driver-installation).
+        This must include the drivers, `hl-smi`, and Habana Container Runtime.
 
     === "Tenstorrent"
-        2.&nbsp;Hosts with Tenstorrent accelerators should be pre-installed with [Tenstorrent software](https://docs.tenstorrent.com/getting-started/README.html#software-installation).
-        This should include the drivers, `tt-smi`, and HugePages.
+        2.&nbsp;Hosts with Tenstorrent accelerators must be pre-installed with [Tenstorrent software](https://docs.tenstorrent.com/getting-started/README.html#software-installation).
+        This must include the drivers, `tt-smi`, and HugePages.
 
-    3.&nbsp;The user specified should have passwordless `sudo` access.
+    3.&nbsp;The user specified must have passwordless `sudo` access.
+
+    4.&nbsp;The SSH server must be running and configured with `AllowTcpForwarding yes` in `/etc/ssh/sshd_config`.
+
+    5.&nbsp;The firewall must allow SSH and should forbid any other connections from external networks. For `placement: cluster` fleets, it should also allow any communication between fleet nodes.
 
 To create or update the fleet, pass the fleet configuration to [`dstack apply`](../reference/cli/dstack/apply.md):
 
@@ -304,13 +317,14 @@ Once the status of instances changes to `idle`, they can be used by dev environm
 If the hosts are interconnected (i.e. share the same network), set `placement` to `cluster`. 
 This is required if you'd like to use the fleet for [distributed tasks](tasks.md#distributed-tasks).
 
-##### Network
-    
-By default, `dstack` automatically detects the network shared by the hosts. 
-However, it's possible to configure it explicitly via 
-the [`network`](../reference/dstack.yml/fleet.md#network) property.
+??? info "Network"  
+    By default, `dstack` automatically detects the network shared by the hosts. 
+    However, it's possible to configure it explicitly via 
+    the [`network`](../reference/dstack.yml/fleet.md#network) property.
+
+    [//]: # (TODO: Provide an example and more detail)
 
-[//]: # (TODO: Provide an example and more detail)
+> For more details on optimal inter-node connectivity, read the [Clusters](../guides/clusters.md) guide.
 
 #### Blocks { #ssh-blocks }
 
@@ -331,10 +345,10 @@ divided into, allowing multiple jobs to use these blocks concurrently.
       hosts:
         - hostname: 3.255.177.51
           blocks: 4
-        - hostaname: 3.255.177.52
+        - hostname: 3.255.177.52
           # As many as possible, according to numbers of GPUs and CPUs
           blocks: auto
-        - hostaname: 3.255.177.53
+        - hostname: 3.255.177.53
           # Do not sclice. This is the default value, may be omitted
           blocks: 1
     ```
@@ -463,5 +477,6 @@ Alternatively, you can delete a fleet by passing the fleet name  to `dstack flee
 To terminate and delete specific instances from a fleet, pass `-i INSTANCE_NUM`.
 
 !!! info "What's next?"
-    1. Read about [dev environments](dev-environments.md), [tasks](tasks.md), and
+    1. Check [dev environments](dev-environments.md), [tasks](tasks.md), and
     [services](services.md)
+    2. Read the [Clusters](../guides/clusters.md) guide

{dstack-0.19.9 → dstack-0.19.10}/docs/docs/concepts/gateways.md

@@ -71,7 +71,7 @@ To delete a gateway, pass the gateway configuration to [`dstack delete`](../refe
 <div class="termy">
 
 ```shell
-$ dstack delete -f examples/deployment/gateway.dstack.yml
+$ dstack delete -f examples/inference/gateway.dstack.yml
 ```
 
 </div>

{dstack-0.19.9 → dstack-0.19.10}/docs/docs/concepts/projects.md

@@ -49,11 +49,11 @@ and when using the CLI or API.
 
 ### Setting up the CLI
 
-To use the CLI with a specific project, run the `dstack config` command with the server address, user token, and project name.
+You can configure multiple projects on the client and set the default project using the [`dstack project`](../reference/cli/dstack/project.md) CLI command. 
 
 You can find the command on the project’s settings page:
 
-<img src="https://dstack.ai/static-assets/static-assets/images/dstack-projects-project-cli.png" width="750px" />
+<img src="https://dstack.ai/static-assets/static-assets/images/dstack-projects-project-cli-v2.png" width="750px" />
 
 ??? info "API"
     In addition to the UI, managing projects, users, and user permissions can also be done via the [REST API](../reference/api/rest/index.md).

{dstack-0.19.9 → dstack-0.19.10}/docs/docs/concepts/services.md

@@ -558,6 +558,6 @@ If one replica of a multi-replica service fails with retry enabled,
     1. Read about [dev environments](dev-environments.md), [tasks](tasks.md), and [repos](repos.md)
     2. Learn how to manage [fleets](fleets.md)
     3. See how to set up [gateways](gateways.md)
-    4. Check the [TGI :material-arrow-top-right-thin:{ .external }](../../examples/deployment/tgi/index.md){:target="_blank"},
-       [vLLM :material-arrow-top-right-thin:{ .external }](../../examples/deployment/vllm/index.md){:target="_blank"}, and 
-       [NIM :material-arrow-top-right-thin:{ .external }](../../examples/deployment/nim/index.md){:target="_blank"} examples
+    4. Check the [TGI :material-arrow-top-right-thin:{ .external }](../../examples/inference/tgi/index.md){:target="_blank"},
+       [vLLM :material-arrow-top-right-thin:{ .external }](../../examples/inference/vllm/index.md){:target="_blank"}, and 
+       [NIM :material-arrow-top-right-thin:{ .external }](../../examples/inference/nim/index.md){:target="_blank"} examples

{dstack-0.19.9 → dstack-0.19.10}/docs/docs/concepts/tasks.md

@@ -137,8 +137,7 @@ resources:
 
 Nodes can communicate using their private IP addresses.
 Use `DSTACK_MASTER_NODE_IP`, `DSTACK_NODES_IPS`, `DSTACK_NODE_RANK`, and other
-[System environment variables](#system-environment-variables)
-to discover IP addresses and other details.
+[System environment variables](#system-environment-variables) for inter-node communication.
 
 ??? info "Network interface"
     Distributed frameworks usually detect the correct network interface automatically,
@@ -427,6 +426,34 @@ retry:
 If one job of a multi-node task fails with retry enabled,
 `dstack` will stop all the jobs and resubmit the run.
 
+### Priority
+
+Be default, submitted runs are scheduled in the order they were submitted.
+When compute resources are limited, you may want to prioritize some runs over others.
+This can be done by specifying the [`priority`](../reference/dstack.yml/task.md) property in the run configuration:
+
+<div editor-title=".dstack.yml">
+
+```yaml
+type: task
+name: train
+
+python: "3.10"
+
+# Commands of the task
+commands:
+  - pip install -r fine-tuning/qlora/requirements.txt
+  - python fine-tuning/qlora/train.py
+
+priority: 50
+```
+
+</div>
+
+`dstack` tries to provision runs with higher priority first.
+Note that if a high priority run cannot be scheduled,
+it does not block other runs with lower priority from scheduling.
+
 --8<-- "docs/concepts/snippets/manage-fleets.ext"
 
 --8<-- "docs/concepts/snippets/manage-runs.ext"