dstack 0.20.22__tar.gz → 0.20.23__tar.gz

{dstack-0.20.22 → dstack-0.20.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.20.22
+Version: 0.20.23
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack

{dstack-0.20.22 → dstack-0.20.23}/mkdocs/assets/stylesheets/landing.css

@@ -17,98 +17,6 @@
     /*letter-spacing: -3px;*/
 }
 
-.tx-landing__hero_text h1.glitch {
-    position: relative;
-    animation: glitch-skew 10s infinite linear;
-}
-
-.tx-landing__hero_text h1.glitch::before,
-.tx-landing__hero_text h1.glitch::after {
-    content: attr(data-text);
-    position: absolute;
-    top: 0;
-    left: 0;
-    width: 100%;
-    height: 100%;
-    background: white;
-    overflow: hidden;
-    clip-path: inset(0 0 100% 0);
-}
-
-.tx-landing__hero_text h1.glitch::before {
-    text-shadow: -3px 0 rgba(0, 72, 255, 0.8);
-    animation: glitch-1 10s infinite linear;
-}
-
-.tx-landing__hero_text h1.glitch::after {
-    text-shadow: 3px 0 rgba(206, 0, 255, 0.8);
-    animation: glitch-2 10s infinite linear;
-}
-
-@keyframes glitch-1 {
-    0%, 9%, 15%, 54%, 60%, 100% {
-        clip-path: inset(0 0 100% 0);
-        transform: translate(0);
-    }
-    10%   { clip-path: inset(5% 0 88% 0);   transform: translate(6px, 0); }
-    10.5% { clip-path: inset(8% 0 85% 0);   transform: translate(-8px, 1px); }
-    11%   { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    11.5% { clip-path: inset(12% 0 80% 0);  transform: translate(9px, -1px); }
-    12%   { clip-path: inset(3% 0 90% 0);   transform: translate(-6px, 0); }
-    12.5% { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    13%   { clip-path: inset(7% 0 86% 0);   transform: translate(7px, 1px); }
-    13.5% { clip-path: inset(10% 0 83% 0);  transform: translate(-9px, 0); }
-    14%   { clip-path: inset(4% 0 89% 0);   transform: translate(5px, -1px); }
-
-    55%   { clip-path: inset(58% 0 34% 0);  transform: translate(-7px, 0); }
-    55.5% { clip-path: inset(62% 0 30% 0);  transform: translate(9px, 1px); }
-    56%   { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    56.5% { clip-path: inset(55% 0 38% 0);  transform: translate(-8px, -1px); }
-    57%   { clip-path: inset(60% 0 32% 0);  transform: translate(6px, 0); }
-    57.5% { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    58%   { clip-path: inset(63% 0 28% 0);  transform: translate(-7px, 1px); }
-    58.5% { clip-path: inset(56% 0 36% 0);  transform: translate(8px, 0); }
-    59%   { clip-path: inset(61% 0 31% 0);  transform: translate(-6px, -1px); }
-}
-
-@keyframes glitch-2 {
-    0%, 10.5%, 16.5%, 55.5%, 61.5%, 100% {
-        clip-path: inset(0 0 100% 0);
-        transform: translate(0);
-    }
-    11%   { clip-path: inset(30% 0 62% 0);  transform: translate(-7px, 1px); }
-    11.5% { clip-path: inset(35% 0 57% 0);  transform: translate(9px, 0); }
-    12%   { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    12.5% { clip-path: inset(33% 0 59% 0);  transform: translate(8px, 0); }
-    13%   { clip-path: inset(38% 0 54% 0);  transform: translate(-9px, 1px); }
-    13.5% { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    14%   { clip-path: inset(31% 0 61% 0);  transform: translate(6px, -1px); }
-    14.5% { clip-path: inset(36% 0 56% 0);  transform: translate(-8px, 0); }
-    15%   { clip-path: inset(34% 0 58% 0);  transform: translate(7px, 1px); }
-    15.5% { clip-path: inset(32% 0 60% 0);  transform: translate(-6px, 0); }
-
-    56%   { clip-path: inset(78% 0 14% 0);  transform: translate(7px, 0); }
-    56.5% { clip-path: inset(82% 0 10% 0);  transform: translate(-9px, -1px); }
-    57%   { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    57.5% { clip-path: inset(75% 0 17% 0);  transform: translate(8px, 1px); }
-    58%   { clip-path: inset(80% 0 12% 0);  transform: translate(-6px, 0); }
-    58.5% { clip-path: inset(0% 0 100% 0);  transform: translate(0); }
-    59%   { clip-path: inset(83% 0 9% 0);   transform: translate(-8px, 0); }
-    59.5% { clip-path: inset(77% 0 15% 0);  transform: translate(7px, -1px); }
-    60%   { clip-path: inset(81% 0 11% 0);  transform: translate(-6px, 1px); }
-    60.5% { clip-path: inset(79% 0 13% 0);  transform: translate(8px, 0); }
-}
-
-@keyframes glitch-skew {
-    0%, 9%, 16%, 54%, 62%, 100% {
-        transform: skew(0deg) scaleX(1);
-    }
-    11%   { transform: skew(0.5deg) scaleX(1.001); }
-    13%   { transform: skew(-0.4deg) scaleX(0.999); }
-    56%   { transform: skew(-0.5deg) scaleX(1.001); }
-    58%   { transform: skew(0.4deg) scaleX(0.999); }
-}
-
 .tx-landing__hero_text h1 strong {
     font-weight: 800;
 }
@@ -597,12 +505,6 @@
     grid-gap: 20px !important;
 }
 
-#typed {
-    background: -webkit-linear-gradient(45deg, #0048ff, #ce00ff);
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-}
-
 .providers.tx-landing__highlights_grid .feature-cell h3 {
     align-content: center;
     font-size: 1em;
@@ -1310,13 +1212,6 @@
     }
 }
 
-.typed-cursor {
-    background: linear-gradient(90deg, #4631C8 -1.29%, #CD4AE2 88.05%, #FFD43C 111.26%);
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-    padding: 0 7px;
-}
-
 .tx-landing__quotes_grid {
     grid-gap: 1.2rem;
     display: grid;
@@ -1389,10 +1284,6 @@
     }
 }
 
-.tx-landing__hero_text h1 {
-    font-family: 'Geist Pixel Square', var(--md-text-font-family);
-}
-
 .md-header__title {
     font-family: 'Geist Pixel Square', var(--md-text-font-family);
 }

{dstack-0.20.22 → dstack-0.20.23}/mkdocs/docs/guides/protips.md

@@ -499,6 +499,30 @@ The `offer` command allows you to filter and group offers with various [advanced
 
 `dstack` tracks essential metrics accessible via the CLI and UI. To access advanced metrics like DCGM, configure the server to export metrics to Prometheus. See [Metrics](../concepts/metrics.md) for details.
 
+## Pull progress
+
+When using a VM-based backend or an SSH instance, you can see the Docker image pull progress in the CLI while the job is in the `pulling` state.
+
+<div class="termy">
+
+```shell
+$ dstack apply -y
+
+Launching orange-yak-1...
+ NAME          BACKEND          STATUS                    SUBMITTED  
+ orange-yak-1  aws (eu-west-3)  pulling 1.19/3.44/4.76GB  3 mins ago
+```
+
+</div>
+
+The three slash-separated indicators represent:
+
+- Image data already extracted.
+- Image data already downloaded.
+- Total target image size.
+
+The exact total size is not always known initially, in which case a lower estimate is displayed with the `≥` sign. All values exclude any image layers already cached on the host prior to the pull.
+
 ## Service quotas
 
 If you're using your own AWS, GCP, Azure, or OCI accounts, before you can use GPUs or spot instances, you have to request the

{dstack-0.20.22 → dstack-0.20.23}/mkdocs/index.md

@@ -1,6 +1,6 @@
 ---
 template: home.html
-title: Agentic orchestration for AI infrastructure
+title: The orchestration stack for AI infrastructure
 hide:
    - navigation
    - toc

{dstack-0.20.22 → dstack-0.20.23}/mkdocs/overrides/home.html

@@ -14,21 +14,6 @@
         $crisp.push(["do", "chat:hide"]);
     })();
 </script>
-<script src="https://unpkg.com/typed.js@2.1.0/dist/typed.umd.js"></script>
-<script>
-    var typed = new Typed('#typed', {
-        strings: ['for AI infrastructure', 'for model training', 'for model inference',
-            // 'for sandboxes', 
-            'across GPU clouds', 'across Kubernetes', 'for bare-metal clusters',
-            'for AI researchers', 'for ML engineers',
-            'on NVIDIA GPUs', 'on AMD GPUs', 'on any GPU or CPU', 'without vendor lock-in',
-        ],
-        backDelay: 1800,
-        typeSpeed: 85,
-        backSpeed: 60,
-        loop: true,
-    });
-</script>
 <script>
     document.addEventListener('DOMContentLoaded', function() {
         var highlighters = {
@@ -288,12 +273,11 @@
     <div class="md-grid md-typeset">
         <div class="tx-landing__hero">
             <div class="tx-landing__hero_text">
-                <h1 class="glitch" data-text="Agentic orchestration" style="min-height: 2.4em;">Agentic orchestration<br><span id="typed" style="white-space: nowrap;"></span></h1>
+                <h1 style="min-height: 2.4em;">The orchestration stack<br>for AI infrastructure</h1>
 
                 <p>
-                    <span class="highlight">dstack</span> is an open-source control plane for agents and engineers to provision compute and run training, inference, and sandboxes across 
-                    <span class="nowrap-token"><span class="highlight">NVIDIA</span>,</span> <span class="nowrap-token"><span class="highlight">AMD</span>,</span> 
-                    <span class="nowrap-token"><span class="highlight">TPU</span>,</span> and <span class="highlight">Tenstorrent</span> GPUs—on clouds, Kubernetes, and bare-metal clusters.
+                    <span class="highlight">dstack</span> is a unified control plane for GPU provisioning and orchestration that works with any GPU cloud, Kubernetes, or on-prem clusters.
+                    It streamlines development, training, and inference, and is compatible with any hardware, open-source tools, and frameworks.
                 </p>
             </div>
 
@@ -327,20 +311,19 @@
         <div class="tx-landing__major_feature">
             <div class="section">
                 <div class="block margin right">
-                    <h2>A unified control plane for compute orchestration</h2>
+                    <h2>One control plane for AI compute</h2>
                     <p>
-                        Managing AI infrastructure requires fine-grained primitives for compute provisioning, 
-                        with native integration across GPU vendors, clouds, and open-source frameworks.
+                        Managing AI infrastructure requires first-class primitives for accelerator provisioning,
+                        workload scheduling, and observability across clouds, clusters, and open-source frameworks.
                     </p>
 
                     <p>
-                        <span class="highlight">dstack</span> is a unified control plane for provisioning 
-                        clusters and running training, inference, and sandboxes across clouds, Kubernetes, 
-                        and bare-metal clusters.
+                        <span class="highlight">dstack</span> unifies fleets, dev environments, tasks, services,
+                        volumes, and gateways in one control plane for AI workloads.
                     </p>
                     
                     <p>
-                        It’s built for containerized workloads and designed for both engineers and agents. 
+                        It’s built for containerized AI workloads with a simple CLI, UI, and API.
                         No <span class="highlight">Kubernetes</span> or <span class="highlight">Slurm</span> hassle required.
                     </p>
 
@@ -392,7 +375,7 @@ backends: [aws]</code></pre>
                 </div>
 
                 <div class="block margin left">
-                    <h2>Provision compute in any GPU cloud</h2>
+                    <h2>Provision GPU fleets across clouds</h2>
 
                     <div class="supported_clouds_inline">
                         <a href="/docs/concepts/backends#aws" target="_blank"><img width="33" src="/assets/images/aws-logo.svg"></a>
@@ -439,7 +422,7 @@ backends: [aws]</code></pre>
         <div class="tx-landing__major_feature">
             <div class="section">
                 <div class="block margin right">
-                    <h2>Bring your existing GPU clusters</h2>
+                    <h2>Bring your own clusters</h2>
                     
                     <p>
                         Have bare-metal servers or pre-provisioned VMs? Use <span class="highlight">SSH fleets</span> to connect them to <span class="highlight">dstack</span>.
@@ -540,9 +523,9 @@ resources:
                 </div>
 
                 <div class="block margin left">
-                    <h2>Run dev environments</h2>
+                    <h2>Launch GPU dev environments</h2>
                     <p>
-                        If you or your agent need a development environment with a GPU, let <span class="highlight">dstack</span> create you a <span class="highlight">dev environment</span>.
+                        If you need a remote development environment with a GPU, let <span class="highlight">dstack</span> create you a <span class="highlight">dev environment</span>.
                     </p>
 
                     <p>If you plan to work with it yourself, you can access it using your desktop IDE such as <span class="highlight">VS
@@ -564,7 +547,7 @@ resources:
         <div class="tx-landing__major_feature">
             <div class="section">
                 <div class="block margin right">
-                    <h2>Run training or batch jobs at any scale</h2>
+                    <h2>Run training and batch jobs at scale</h2>
 
                     <p>
                         Run training or batch workloads on a single GPU, or scale to multi-GPU and multi-node clusters using simple <span class="highlight">task</span> configurations.
@@ -665,7 +648,7 @@ resources:
                 </div>
 
                 <div class="block margin left">
-                    <h2>Run high-performance model inference</h2>
+                    <h2>Deploy production inference services</h2>
 
                     <p>
                         With <span class="highlight">dstack</span>, you can deploy models as secure,
@@ -912,17 +895,14 @@ resources:
 
                     <div class="cell">
                         <div class="photo">
-                            <img src="assets/images/quotes/cudopete.png"/>
+                            <img src="assets/images/quotes/jon.jpeg"/>
                         </div>
-                        <h3>Peter Hill</h3>
+                        <h3>Jon Stevens</h3>
 
-                        <h4>Co-Founder @CUDO Compute</h4>
+                        <h4>CEO @Hot Aisle</h4>
 
                         <p>
-                            <span class="highlight">dstack</span> simplifies infrastructure provisioning and AI
-                            development.
-                            If your team is on the lookout for an AI platform, I
-                            wholeheartedly recommend <span class="highlight">dstack</span>.
+                            <span class="highlight">dstack</span> 's advantages over <span class="highlight">Slurm</span> are clear: it's a modern, ground-up approach to running workloads at scale. If you're choosing an orchestration platform, <span class="highlight">dstack</span> is the place to start.
                         </p>
                     </div>
                 </div>

{dstack-0.20.22 → dstack-0.20.23}/mkdocs.yml

@@ -3,7 +3,7 @@ site_name: dstack
 site_url: https://dstack.ai
 site_author: dstack GmbH
 site_description: >-
-  dstack is an open-source control plane for GPU provisioning and orchestration across GPU clouds, Kubernetes, and on-prem clusters.
+  dstack is a unified control plane for GPU provisioning and orchestration that works with any GPU cloud, Kubernetes, or on-prem clusters.
 
 # Repository
 repo_url: https://github.com/dstackai/dstack

{dstack-0.20.22 → dstack-0.20.23}/runner/cmd/shim/main.go

@@ -27,6 +27,13 @@ import (
 // Version is a build-time variable. The value is overridden by ldflags.
 var Version string
 
+// https://everything.curl.dev/usingcurl/proxies/env.html
+// https://cs.opensource.google/go/x/net/+/657eb1317b5dd33038d683297c6be9cae05fa97d:http/httpproxy/proxy.go
+// We accept HTTP_PROXY in upper case without additional checks as it's unlikely that
+// the shim is running in the CGI context
+// The lower case form should be used as some applications ignore the upper case form, e.g., curl, apt
+const defaultPassEnv = "http_proxy,https_proxy,no_proxy,HTTP_PROXY,HTTPS_PROXY,NO_PROXY"
+
 func main() {
 	os.Exit(mainInner())
 }
@@ -147,6 +154,13 @@ func mainInner() int {
 				Sources:     cli.EnvVars("DSTACK_DCGM_ADDRESS"),
 			},
 			/* Docker Parameters */
+			&cli.StringFlag{
+				Name:        "pass-env",
+				Usage:       "Environment variables to pass on to the container, a comma-separated list of names",
+				Value:       defaultPassEnv,
+				Destination: &args.Docker.PassEnv,
+				Sources:     cli.EnvVars("DSTACK_DOCKER_PASS_ENV"),
+			},
 			&cli.BoolFlag{
 				Name:        "privileged",
 				Usage:       "Give extended privileges to the container",

{dstack-0.20.22 → dstack-0.20.23}/runner/internal/runner/ssh/sshd.go

@@ -23,10 +23,16 @@ type SshdManager interface {
 	AddAuthorizedKeys(context.Context, ...string) error
 }
 
-var hostKeys = [...]string{
-	"ssh_host_rsa_key",
-	"ssh_host_ecdsa_key",
-	"ssh_host_ed25519_key",
+// Host keys generated and configured for sshd. RSA is intentionally omitted:
+// ssh-keygen -A's RSA-3072 generation dominates runner startup (~1000ms vs
+// <50ms for ECDSA + Ed25519 combined). Both ECDSA and Ed25519 are
+// universally supported by modern SSH clients (OpenSSH >= 6.5).
+var hostKeys = [...]struct {
+	name    string
+	keyType string
+}{
+	{name: "ssh_host_ecdsa_key", keyType: "ecdsa"},
+	{name: "ssh_host_ed25519_key", keyType: "ed25519"},
 }
 
 // Implements SshdManager
@@ -163,14 +169,14 @@ func generateHostKeys(ctx context.Context, confDir string) error {
 		return err
 	}
 
-	// TODO: specify the full path if a custom OpenSSH build is used
-	cmd := exec.CommandContext(ctx, "ssh-keygen", "-A", "-f", tmpDir)
-	if err := cmd.Run(); err != nil {
-		return err
-	}
-
-	for _, key := range hostKeys {
-		if err := copyHostKey(keyDir, confDir, key); err != nil {
+	for _, k := range hostKeys {
+		// TODO: specify the full path if a custom OpenSSH build is used
+		cmd := exec.CommandContext(ctx, "ssh-keygen",
+			"-t", k.keyType, "-q", "-N", "", "-f", path.Join(keyDir, k.name))
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("generate %s host key: %w", k.keyType, err)
+		}
+		if err := copyHostKey(keyDir, confDir, k.name); err != nil {
 			return err
 		}
 	}
@@ -251,8 +257,8 @@ func createSshdConfig(ctx context.Context, confDir string, port int, logLevel st
 		"ClientAliveInterval 30",
 		"ClientAliveCountMax 4",
 	}
-	for _, hostKey := range hostKeys {
-		lines = append(lines, fmt.Sprintf("HostKey %s/%s", confDir, hostKey))
+	for _, k := range hostKeys {
+		lines = append(lines, fmt.Sprintf("HostKey %s/%s", confDir, k.name))
 	}
 	for _, line := range lines {
 		if _, err := fmt.Fprintln(file, line); err != nil {

{dstack-0.20.22 → dstack-0.20.23}/runner/internal/shim/docker.go

@@ -56,17 +56,26 @@ const (
 // https://github.com/moby/moby/blob/e77ff99ede5ee5952b3a9227863552ae6e5b6fb1/pkg/jsonmessage/jsonmessage.go#L144
 // All fields are optional.
 type PullMessage struct {
-	Id             string `json:"id"` // layer id
-	Status         string `json:"status"`
-	ProgressDetail struct {
-		Current uint64 `json:"current"` // bytes
-		Total   uint64 `json:"total"`   // bytes
-	} `json:"progressDetail"`
-	ErrorDetail struct {
+	Id             string         `json:"id"` // layer id
+	Status         string         `json:"status"`
+	ProgressDetail ProgressDetail `json:"progressDetail"`
+	ErrorDetail    struct {
 		Message string `json:"message"`
 	} `json:"errorDetail"`
 }
 
+type ProgressDetail struct {
+	Current uint64 `json:"current"`
+	Total   uint64 `json:"total"`
+	Units   string `json:"units"`
+}
+
+func (p *ProgressDetail) isUnitBytes() bool {
+	// > Units is the unit to print for progress. It defaults to "bytes" if empty
+	// https://github.com/moby/moby/blob/8151a55a776f5f83f68bcf0030c19031439ea357/api/types/jsonstream/progress.go#L9
+	return p.Units == "bytes" || p.Units == ""
+}
+
 type layerProgress struct {
 	Status          string
 	DownloadedBytes uint64
@@ -94,14 +103,18 @@ func (t *PullTracker) Update(msg PullMessage) {
 	case "Pulling fs layer", "Waiting", "Verifying Checksum", "Already exists":
 		// no bytes to update, just track status
 	case "Downloading":
-		layer.DownloadedBytes = msg.ProgressDetail.Current
-		layer.TotalBytes = msg.ProgressDetail.Total
+		if msg.ProgressDetail.isUnitBytes() {
+			layer.DownloadedBytes = msg.ProgressDetail.Current
+			layer.TotalBytes = msg.ProgressDetail.Total
+		}
 	case "Download complete":
 		layer.DownloadedBytes = layer.TotalBytes
 	case "Extracting":
-		layer.ExtractedBytes = msg.ProgressDetail.Current
-		layer.DownloadedBytes = msg.ProgressDetail.Total
-		layer.TotalBytes = msg.ProgressDetail.Total
+		if msg.ProgressDetail.isUnitBytes() {
+			layer.ExtractedBytes = msg.ProgressDetail.Current
+			layer.DownloadedBytes = msg.ProgressDetail.Total
+			layer.TotalBytes = msg.ProgressDetail.Total
+		}
 	case "Pull complete":
 		layer.ExtractedBytes = layer.TotalBytes
 		layer.DownloadedBytes = layer.TotalBytes
@@ -135,6 +148,7 @@ type DockerRunner struct {
 	client       *docker.Client
 	dockerParams DockerParameters
 	dockerInfo   dockersystem.Info
+	baseEnv      []string
 	gpus         []host.GpuInfo
 	gpuVendor    gpu.GpuVendor
 	gpuLock      *GpuLock
@@ -151,6 +165,15 @@ func NewDockerRunner(ctx context.Context, dockerParams DockerParameters) (*Docke
 		return nil, fmt.Errorf("get docker info: %w", err)
 	}
 
+	// Copy variables once rather than on a per-task basis
+	// We don't expect variables to change during the shim's lifetime
+	baseEnv := []string{}
+	for _, name := range dockerParams.DockerPassEnv() {
+		if value, ok := os.LookupEnv(name); ok {
+			baseEnv = append(baseEnv, fmt.Sprintf("%s=%s", name, value))
+		}
+	}
+
 	var gpuVendor gpu.GpuVendor
 	gpus := host.GetGpuInfo(ctx)
 	if len(gpus) > 0 {
@@ -167,6 +190,7 @@ func NewDockerRunner(ctx context.Context, dockerParams DockerParameters) (*Docke
 		client:       client,
 		dockerParams: dockerParams,
 		dockerInfo:   dockerInfo,
+		baseEnv:      baseEnv,
 		gpus:         gpus,
 		gpuVendor:    gpuVendor,
 		gpuLock:      gpuLock,
@@ -861,8 +885,9 @@ func (d *DockerRunner) createContainer(ctx context.Context, task *Task) error {
 
 	// Set the environment variables
 	envVars := []string{}
-	if d.dockerParams.DockerPJRTDevice() != "" {
-		envVars = append(envVars, fmt.Sprintf("PJRT_DEVICE=%s", d.dockerParams.DockerPJRTDevice()))
+	envVars = append(envVars, d.baseEnv...)
+	if pjrtDevice := d.dockerParams.DockerPJRTDevice(); pjrtDevice != "" {
+		envVars = append(envVars, fmt.Sprintf("PJRT_DEVICE=%s", pjrtDevice))
 	}
 
 	// Override /dev/shm with tmpfs mount with `exec` option (the default is `noexec`)
@@ -1235,6 +1260,16 @@ func getContainerLastLogs(ctx context.Context, client docker.APIClient, containe
 
 /* DockerParameters interface implementation for CLIArgs */
 
+func (c *CLIArgs) DockerPassEnv() []string {
+	names := []string{}
+	for _, name := range strings.Split(c.Docker.PassEnv, ",") {
+		if name = strings.TrimSpace(name); name != "" {
+			names = append(names, name)
+		}
+	}
+	return names
+}
+
 func (c *CLIArgs) DockerPrivileged() bool {
 	return c.Docker.Privileged
 }

{dstack-0.20.22 → dstack-0.20.23}/runner/internal/shim/docker_test.go

@@ -119,6 +119,10 @@ func (c *dockerParametersMock) DockerShellCommands(authorizedKeys []string, runn
 	return commands
 }
 
+func (c *dockerParametersMock) DockerPassEnv() []string {
+	return []string{}
+}
+
 func (c *dockerParametersMock) DockerPorts() []int {
 	return []int{}
 }
@@ -264,3 +268,31 @@ func TestPullTracker_MixedLayerStatuses(t *testing.T) {
 	assert.Equal(t, uint64(100+200), p.TotalBytes)
 	assert.False(t, p.IsTotalBytesFinal) // layer-waiting size unknown
 }
+
+func TestPullTracker_NonBytesExtractingUnit(t *testing.T) {
+	tracker := newPullTracker()
+	tracker.Update(PullMessage{Id: "3.11", Status: "Pulling from library/python"})
+	tracker.Update(PullMessage{Id: "aaa", Status: "Pulling fs layer"})
+	tracker.Update(pullMsg("aaa", "Downloading", 100, 200))
+	tracker.Update(pullMsg("aaa", "Downloading", 200, 200))
+	tracker.Update(PullMessage{Id: "aaa", Status: "Download complete"})
+	// Newer Docker daemons report extraction progress in seconds. Tested with 29.5.2
+	tracker.Update(PullMessage{Id: "aaa", Status: "Extracting", ProgressDetail: ProgressDetail{Current: 1, Units: "s"}})
+	tracker.Update(PullMessage{Id: "aaa", Status: "Extracting", ProgressDetail: ProgressDetail{Current: 2, Units: "s"}})
+
+	p := tracker.Progress()
+	require.NotNil(t, p)
+	assert.Equal(t, uint64(200), p.DownloadedBytes)
+	assert.Equal(t, uint64(0), p.ExtractedBytes) // reported in seconds, bytes unknown
+	assert.Equal(t, uint64(200), p.TotalBytes)
+	assert.True(t, p.IsTotalBytesFinal)
+
+	tracker.Update(PullMessage{Id: "aaa", Status: "Pull complete"})
+
+	p = tracker.Progress()
+	require.NotNil(t, p)
+	assert.Equal(t, uint64(200), p.DownloadedBytes)
+	assert.Equal(t, uint64(200), p.ExtractedBytes) // pull complete => extracted == total
+	assert.Equal(t, uint64(200), p.TotalBytes)
+	assert.True(t, p.IsTotalBytesFinal)
+}

{dstack-0.20.22 → dstack-0.20.23}/runner/internal/shim/models.go

@@ -5,6 +5,7 @@ import (
 )
 
 type DockerParameters interface {
+	DockerPassEnv() []string
 	DockerPrivileged() bool
 	DockerShellCommands(authorizedKeys []string, runnerHttpAddress string) []string
 	DockerMounts(string) ([]mount.Mount, error)
@@ -40,6 +41,7 @@ type CLIArgs struct {
 	}
 
 	Docker struct {
+		PassEnv    string
 		Privileged bool
 		PJRTDevice string
 	}

{dstack-0.20.22 → dstack-0.20.23}/src/dstack/_internal/cli/utils/run.py

@@ -339,6 +339,7 @@ def _format_pull_progress(progress: ImagePullProgress) -> str:
         def f(x: int) -> str:
             return f"{x / 2**20:.0f}"
 
+    # NOTE: The format is documented in protips.md. Keep in sync.
     total_sign = "≥" if not progress.is_total_bytes_final else ""
     return f"{f(progress.extracted_bytes)}/{f(progress.downloaded_bytes)}/{total_sign}{f(progress.total_bytes)}{unit}"
 

{dstack-0.20.22 → dstack-0.20.23}/src/dstack/_internal/core/backends/oci/resources.py

@@ -728,6 +728,12 @@ def create_pre_authenticated_request(
 def delete_bucket(
     namespace: str, bucket_name: str, client: oci.object_storage.ObjectStorageClient
 ) -> None:
+    in_progress_uploads: Iterable[oci.object_storage.models.MultipartUpload] = (
+        chain_paginated_responses(client.list_multipart_uploads, namespace, bucket_name)
+    )
+    for upload in in_progress_uploads:
+        client.abort_multipart_upload(namespace, bucket_name, upload.object, upload.upload_id)
+
     par_ids = {
         par.id
         for par in chain_paginated_responses(

{dstack-0.20.22 → dstack-0.20.23}/src/dstack/_internal/core/backends/verda/compute.py

@@ -78,19 +78,21 @@ class VerdaCompute(
     def _get_offers_with_availability(
         self, offers: List[InstanceOffer]
     ) -> List[InstanceOfferWithAvailability]:
-        raw_availabilities: List[Dict] = self.client.instances.get_availabilities()
-
         region_availabilities = {}
-        for location in raw_availabilities:
-            location_code = location["location_code"]
-            availabilities = location["availabilities"]
-            for name in availabilities:
-                key = (name, location_code)
-                region_availabilities[key] = InstanceAvailability.AVAILABLE
+        for is_spot in (False, True):
+            raw_availabilities: List[Dict] = self.client.instances.get_availabilities(
+                is_spot=is_spot
+            )
+            for location in raw_availabilities:
+                location_code = location["location_code"]
+                availabilities = location["availabilities"]
+                for name in availabilities:
+                    key = (name, location_code, is_spot)
+                    region_availabilities[key] = InstanceAvailability.AVAILABLE
 
         availability_offers = []
         for offer in offers:
-            key = (offer.instance.name, offer.region)
+            key = (offer.instance.name, offer.region, offer.instance.resources.spot)
             availability = region_availabilities.get(key, InstanceAvailability.NOT_AVAILABLE)
             availability_offers.append(offer.with_availability(availability=availability))