dstack 0.19.34__tar.gz → 0.19.35__tar.gz

{dstack-0.19.34 → dstack-0.19.35}/CONTRIBUTING.md

@@ -35,7 +35,7 @@ If you make a non-trivial change to `dstack`, we recommend you learn about `dsta
 * Bug fixes that address a clearly defined bug. Include steps to reproduce in the linked issue or the PR.
 * New features. Before submitting a feature PR, create an issue with a proposal to discuss it with the core team and other interested parties.
 * Minor fixes such as typos.
-* [Examples](examples/README.md).
+* [Examples](examples).
 
 ### Before pushing your changes
 

{dstack-0.19.34 → dstack-0.19.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.34
+Version: 0.19.35
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -73,7 +73,7 @@ Requires-Dist: grpcio>=1.50; extra == 'all'
 Requires-Dist: httpx; extra == 'all'
 Requires-Dist: jinja2; extra == 'all'
 Requires-Dist: kubernetes; extra == 'all'
-Requires-Dist: nebius<=0.2.72,>=0.2.40; (python_version >= '3.10') and extra == 'all'
+Requires-Dist: nebius<0.4,>=0.3.4; (python_version >= '3.10') and extra == 'all'
 Requires-Dist: oci>=2.150.0; extra == 'all'
 Requires-Dist: prometheus-client; extra == 'all'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'all'
@@ -259,7 +259,7 @@ Requires-Dist: fastapi; extra == 'nebius'
 Requires-Dist: grpcio>=1.50; extra == 'nebius'
 Requires-Dist: httpx; extra == 'nebius'
 Requires-Dist: jinja2; extra == 'nebius'
-Requires-Dist: nebius<=0.2.72,>=0.2.40; (python_version >= '3.10') and extra == 'nebius'
+Requires-Dist: nebius<0.4,>=0.3.4; (python_version >= '3.10') and extra == 'nebius'
 Requires-Dist: prometheus-client; extra == 'nebius'
 Requires-Dist: python-dxf==12.1.0; extra == 'nebius'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'nebius'

{dstack-0.19.34 → dstack-0.19.35}/docs/docs/guides/server-deployment.md

@@ -400,6 +400,28 @@ export DSTACK_DB_MAX_OVERFLOW=80
 You have to ensure your Postgres installation supports that many connections by
 configuring [`max_connections`](https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS) and/or using connection pooler.
 
+## Server upgrades
+
+When upgrading the `dstack` server, follow these guidelines to ensure a smooth transition and minimize downtime.
+
+### Before upgrading
+
+1. **Check the changelog**: Review the [release notes :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/releases){:target="_blank"} for breaking changes, new features, and migration notes.
+2. **Review backward compatibility**: Understand the [backward compatibility](#backward-compatibility) policy.
+3. **Back up your data**: Ensure you always create a backup before upgrading.
+
+### Best practices
+
+- **Test in staging**: Always test upgrades in a non-production environment first.
+- **Monitor logs**: Watch server logs during and after the upgrade for any errors or warnings.
+- **Keep backups**: Retain backups for at least a few days after a successful upgrade.
+
+### Troubleshooting
+
+**Deadlock when upgrading a multi-replica PostgreSQL deployment**
+
+If a deployment is stuck due to a deadlock when applying DB migrations, try scaling server replicas to 1 and retry the deployment multiple times. Some releases may not support rolling deployments, which is always noted in the release notes. If you think there is a bug, please [file an issue](https://github.com/dstackai/dstack/issues).
+
 ## FAQs
 
 ??? info "Can I run multiple replicas of dstack server?"

{dstack-0.19.34 → dstack-0.19.35}/pyproject.toml

@@ -84,6 +84,7 @@ include = [
     "src/dstack/_internal/server",
     "src/dstack/_internal/core/services",
     "src/dstack/_internal/core/backends/kubernetes",
+    "src/dstack/_internal/core/backends/runpod",
     "src/dstack/_internal/cli/services/configurators",
     "src/dstack/_internal/cli/commands",
 ]
@@ -189,8 +190,7 @@ oci = [
     "dstack[server]",
 ]
 nebius = [
-    # 0.2.73 breaks sdk backward compatibility: https://github.com/dstackai/dstack/issues/3171
-    "nebius>=0.2.40,<=0.2.72; python_version >= '3.10'",
+    "nebius>=0.3.4,<0.4; python_version >= '3.10'",
     "dstack[server]",
 ]
 all = [

{dstack-0.19.34 → dstack-0.19.35}/runner/cmd/runner/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/dstackai/dstack/runner/internal/log"
 	"github.com/dstackai/dstack/runner/internal/runner/api"
 	"github.com/sirupsen/logrus"
-	"github.com/ztrue/tracerr"
 )
 
 func main() {
@@ -21,17 +20,17 @@ func main() {
 
 func start(tempDir string, homeDir string, httpPort int, sshPort int, logLevel int, version string) error {
 	if err := os.MkdirAll(tempDir, 0o755); err != nil {
-		return tracerr.Errorf("Failed to create temp directory: %w", err)
+		return fmt.Errorf("create temp directory: %w", err)
 	}
 
 	defaultLogFile, err := log.CreateAppendFile(filepath.Join(tempDir, consts.RunnerDefaultLogFileName))
 	if err != nil {
-		return tracerr.Errorf("Failed to create default log file: %w", err)
+		return fmt.Errorf("create default log file: %w", err)
 	}
 	defer func() {
-		err = defaultLogFile.Close()
-		if err != nil {
-			tracerr.Print(err)
+		closeErr := defaultLogFile.Close()
+		if closeErr != nil {
+			log.Error(context.TODO(), "Failed to close default log file", "err", closeErr)
 		}
 	}()
 
@@ -40,12 +39,12 @@ func start(tempDir string, homeDir string, httpPort int, sshPort int, logLevel i
 
 	server, err := api.NewServer(tempDir, homeDir, fmt.Sprintf(":%d", httpPort), sshPort, version)
 	if err != nil {
-		return tracerr.Errorf("Failed to create server: %w", err)
+		return fmt.Errorf("create server: %w", err)
 	}
 
 	log.Trace(context.TODO(), "Starting API server", "port", httpPort)
 	if err := server.Run(); err != nil {
-		return tracerr.Errorf("Server failed: %w", err)
+		return fmt.Errorf("server failed: %w", err)
 	}
 
 	return nil

{dstack-0.19.34 → dstack-0.19.35}/runner/cmd/shim/main.go

@@ -252,9 +252,9 @@ func start(ctx context.Context, args shim.CLIArgs, serviceMode bool) (err error)
 	if serviceMode {
 		if err := shim.WriteHostInfo(shimHomeDir, dockerRunner.Resources(ctx)); err != nil {
 			if errors.Is(err, os.ErrExist) {
-				log.Error(ctx, "cannot write host info: file already exists")
+				log.Error(ctx, "write host info: file already exists")
 			} else {
-				return err
+				return fmt.Errorf("write host info: %w", err)
 			}
 		}
 	}

{dstack-0.19.34 → dstack-0.19.35}/runner/go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.10.0
 	github.com/urfave/cli/v2 v2.27.1
-	github.com/ztrue/tracerr v0.4.0
 	golang.org/x/crypto v0.22.0
 	golang.org/x/sys v0.26.0
 )

{dstack-0.19.34 → dstack-0.19.35}/runner/go.sum

@@ -196,8 +196,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04=
-github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8=
 go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k=

{dstack-0.19.34 → dstack-0.19.35}/runner/internal/api/common.go

@@ -113,11 +113,13 @@ func JSONResponseHandler(handler func(http.ResponseWriter, *http.Request) (inter
 			if errors.As(err, &apiErr) {
 				status = apiErr.Status
 				errMsg = apiErr.Error()
-				log.Warning(r.Context(), "API error", "err", errMsg, "status", status)
+				log.Warning(r.Context(), "API error", "err", errMsg, "method", r.Method, "endpoint", r.URL.Path, "status", status)
 			} else {
 				status = http.StatusInternalServerError
-				log.Error(r.Context(), "Unexpected API error", "err", err, "status", status)
+				log.Error(r.Context(), "Unexpected API error", "err", err, "method", r.Method, "endpoint", r.URL.Path, "status", status)
 			}
+		} else {
+			log.Trace(r.Context(), "", "method", r.Method, "endpoint", r.URL.Path, "status", status)
 		}
 
 		if status != 500 && body != nil {
@@ -127,7 +129,5 @@ func JSONResponseHandler(handler func(http.ResponseWriter, *http.Request) (inter
 		} else {
 			http.Error(w, errMsg, status)
 		}
-
-		log.Debug(r.Context(), "", "method", r.Method, "endpoint", r.URL.Path, "status", status)
 	}
 }

{dstack-0.19.34 → dstack-0.19.35}/runner/internal/common/gpu.go

@@ -16,6 +16,8 @@ const (
 )
 
 func GetGpuVendor() GpuVendor {
+	// FIXME: There might be errors other than os.ErrNotExist that are ignored silently.
+	// Propagate and log.
 	if _, err := os.Stat("/dev/kfd"); !errors.Is(err, os.ErrNotExist) {
 		return GpuVendorAmd
 	}
@@ -28,5 +30,11 @@ func GetGpuVendor() GpuVendor {
 	if _, err := os.Stat("/dev/tenstorrent"); !errors.Is(err, os.ErrNotExist) {
 		return GpuVendorTenstorrent
 	}
+	if _, err := os.Stat("/dev/dxg"); !errors.Is(err, os.ErrNotExist) {
+		// WSL2
+		if _, err := os.Stat("/usr/lib/wsl/lib/nvidia-smi"); !errors.Is(err, os.ErrNotExist) {
+			return GpuVendorNvidia
+		}
+	}
 	return GpuVendorNone
 }

{dstack-0.19.34 → dstack-0.19.35}/runner/internal/common/interpolator.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/dstackai/dstack/runner/internal/gerrors"
 	"github.com/dstackai/dstack/runner/internal/log"
 )
 
@@ -52,7 +51,7 @@ func (vi *VariablesInterpolator) Interpolate(ctx context.Context, s string) (str
 		sb.WriteString(s[start:opening])
 		closing := IndexWithOffset(s, PatternClosing, opening)
 		if closing == -1 {
-			return "", gerrors.Newf("no pattern closing: %s", s[opening:])
+			return "", fmt.Errorf("no pattern closing: %s", s[opening:])
 		}
 
 		name := strings.TrimSpace(s[opening+len(PatternOpening) : closing])