dstack 0.21.3__tar.gz → 0.21.4__tar.gz

{dstack-0.21.3 → dstack-0.21.4}/.github/workflows/build-artifacts.yml

@@ -163,33 +163,6 @@ jobs:
             runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
           retention-days: 1
 
-  gateway-build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up uv
-        uses: astral-sh/setup-uv@v5
-        with:
-          python-version: 3.11
-      - name: Build package
-        working-directory: gateway
-        run: |
-          echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py
-          # TODO: depend on a specific dstackai/dstack commit for staging builds?
-          if [[ "${{ inputs.staging }}" == "false" ]]; then
-            sed \
-              -i.old \
-              "s|@ https://github.com/dstackai/dstack/archive/refs/heads/master.tar.gz|== ${{ inputs.version }}|" \
-              pyproject.toml
-            diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1
-          fi
-          uv build
-      - uses: actions/upload-artifact@v4
-        with:
-          name: dstack-gateway
-          path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl
-          retention-days: 1
-
   python-build:
     needs: [code-lint, frontend-build]
     runs-on: ubuntu-latest

{dstack-0.21.3 → dstack-0.21.4}/.github/workflows/upload-pre-pypi-artifacts.yml

@@ -25,21 +25,6 @@ jobs:
           ignore-empty-workdir: true
       - name: Install AWS
         run: uv tool install awscli
-      - name: Download dstack-gateway
-        uses: actions/download-artifact@v4
-        with:
-          name: dstack-gateway
-          path: gateway
-      - name: Upload dstack-gateway to S3
-        working-directory: gateway
-        run: |
-          CHANNEL=release
-          if [ "${{ inputs.staging }}" = "true" ]; then
-            CHANNEL=stgn
-          fi
-          WHEEL=dstack_gateway-${{ inputs.version }}-py3-none-any.whl
-          aws s3 cp $WHEEL "s3://dstack-gateway-downloads/$CHANNEL/$WHEEL"
-          echo "${{ inputs.version }}" | aws s3 cp - "s3://dstack-gateway-downloads/$CHANNEL/latest-version"
       - name: Download dstack-runner
         uses: actions/download-artifact@v4
         with:

{dstack-0.21.3 → dstack-0.21.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dstack
-Version: 0.21.3
+Version: 0.21.4
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack

{dstack-0.21.3 → dstack-0.21.4}/contributing/AUTOSCALING.md

@@ -2,8 +2,8 @@
 
 `dstack` features auto-scaling for services published via the gateway. The general flow is:
 
-- STEP 1: `dstack-gateway` parses nginx `access.log` to collect per-second statistics about requests to the service and request times.
-- STEP 2: `dstack-gateway` aggregates statistics over several predefined windows.
+- STEP 1: The gateway parses nginx `access.log` to collect per-second statistics about requests to the service and request times.
+- STEP 2: The gateway aggregates statistics over several predefined windows.
 - STEP 3: The server keeps gateway connections alive in the scheduled `process_gateways_connections` task and continuously collects stats from active gateways. This is separate from `GatewayPipeline`, which handles gateway provisioning and deletion.
 - STEP 4: When `RunPipeline` processes a service run, it loads the latest collected gateway stats for that service.
 - STEP 5: The autoscaler (configured via `dstack.yml`) computes the desired replica count for each replica group.

{dstack-0.21.3 → dstack-0.21.4}/contributing/PROXY.md

@@ -137,7 +137,7 @@ Provisioning happens as follows:
 1. Launch a non-GPU instance (usually the smallest) with all ports exposed.
 2. Install Nginx, Certbot, and patch configs.
 3. Create blue-green virtual environments.
-4. Install the latest `dstack-gateway` package from the S3 bucket. `dstack-gateway` is a thin package that depends on the `dstack` package, which contains the actual gateway implementation.
+4. Install the `dstack[gateway]` Python package.
 5. Run the systemd service `dstack.gateway.service`.
 
 ### Update
@@ -152,56 +152,18 @@ The update process looks like this:
 
 ## Gateway development
 
-The gateway app needs to interact with Nginx and certbot, so running it locally can be challenging. One way to test your code is to upload your development branch to an existing gateway and run the gateway app from source.
+The gateway app needs to interact with Nginx and certbot, so running it locally can be challenging. You can test your development version by deploying it to a live gateway.
 
-1. Run `dstack server` with `DSTACK_SKIP_GATEWAY_UPDATE=1` environment variable. This will prevent `dstack` from updating and starting the standard gateway version on each server restart.
+For example, push your version to GitHub and run `dstack server` with the `DSTACK_GATEWAY_PACKAGE_URL` environment variable, which will instruct `dstack` to deploy that version to all gateways.
 
-1. Provision a gateway through `dstack`:
-
-   ```shell
-   dstack apply -f my-gateway.dstack.yml
-   ```
-
-1. Save the gateway key to a file:
-
-   ```shell
-   sqlite3 ~/.dstack/server/data/sqlite.db "SELECT ssh_private_key FROM gateway_computes WHERE deleted = 0 AND ip_address = '<gateway-ip-addr>'" > /tmp/gateway.key
-   chmod 600 /tmp/gateway.key
-   ```
-
-1. Deliver your code to the gateway. For example, clone it from a remote repo:
-
-   ```shell
-   ssh -i /tmp/gateway.key ubuntu@gateway.example "git clone https://github.com/dstackai/dstack.git ~/dstack-repo"
-   ```
-
-   Or push it from your machine:
-
-   ```shell
-   ssh -i /tmp/gateway.key ubuntu@gateway.example "git init ~/dstack-repo"
-   git remote add gateway ubuntu@gateway.example:~/dstack-repo
-   GIT_SSH_COMMAND='ssh -i /tmp/gateway.key' git push gateway branch_name
-   ```
-
-1. Connect to the gateway:
-
-   ```shell
-   ssh -i /tmp/gateway.key ubuntu@gateway.example
-   ```
-
-1. Prepare an environment with your development branch on the gateway:
-
-   ```shell
-   cd ~/dstack-repo
-   git checkout branch_name
-   curl -LsSf https://astral.sh/uv/install.sh | sh
-   source ~/.local/bin/env
-   uv sync --extra gateway
-   ```
+```shell
+DSTACK_GATEWAY_PACKAGE_URL=https://github.com/dstackai/dstack/archive/refs/heads/my_development_branch.zip dstack server
+```
 
-1. Stop the gateway service and start your development version from source:
+To access logs or troubleshoot, extract the gateway's private key from the database and connect to it via SSH. Logs are available in `journalctl`.
 
-   ```shell
-   sudo systemctl stop dstack.gateway.service
-   uv run uvicorn dstack._internal.proxy.gateway.main:app
-   ```
+```shell
+sqlite3 ~/.dstack/server/data/sqlite.db "SELECT ssh_private_key FROM gateway_computes WHERE deleted = 0 AND ip_address = '$GATEWAY_IP_ADDR'" > /tmp/gateway.key
+chmod 600 /tmp/gateway.key
+ssh -i /tmp/gateway.key ubuntu@$GATEWAY_IP_ADDR sudo journalctl -u dstack.gateway.service
+```

{dstack-0.21.3 → dstack-0.21.4}/contributing/RUNNER-AND-SHIM.md

@@ -56,6 +56,6 @@ All communication between the `dstack` server and `dstack-runner` happens via HT
 
 ## SSH tunnels
 
-`dstack` expects a running SSH server right next to the `dstack-runner`. It provides a secure channel for communication with the runner API and forwarding any ports without listening for `0.0.0.0`. The `dstack-gateway` also uses this SSH server for forwarding requests from public endpoints.
+`dstack` expects a running SSH server right next to the `dstack-runner`. It provides a secure channel for communication with the runner API and forwarding any ports without listening for `0.0.0.0`. The gateway also uses this SSH server for forwarding requests from public endpoints.
 
 `dstack-shim` must also be running next to the SSH server. The `dstack` server connects to this SSH server for interacting with both `dstack-shim` and `dstack-runner`. The CLI uses this SSH server as a jump host because the user wants to connect to the container.

{dstack-0.21.3 → dstack-0.21.4}/frontend/src/pages/Events/List/hooks/useFilters.ts

@@ -217,7 +217,8 @@ export const useFilters = ({
         targetTypes?.forEach((targetType) => {
             options.push({
                 propertyKey: filterKeys.INCLUDE_TARGET_TYPES,
-                value: targetType.label,
+                label: targetType.label,
+                value: targetType.value,
             });
         });
 

{dstack-0.21.3 → dstack-0.21.4}/mkdocs/docs/concepts/presets.md

@@ -362,7 +362,6 @@ At the same time, it's recommended to create presets using your own agent — ei
 ## Limitations
 
 * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime
-* Doesn't support PD disaggregation (coming soon)
 * The registry doesn't support public presets (coming soon)
 * Doesn't support ranges for `concurrency`
 

{dstack-0.21.3 → dstack-0.21.4}/mkdocs/docs/reference/env.md

@@ -173,6 +173,7 @@ slows down processing and may cause CPU spikes due to frequent SSH-connection es
       `https://dstack.example.com/{arch}/{version}/dstack-runner`.
      * `DSTACK_SHIM_DOWNLOAD_URL` – Overrides `dstack-shim` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders,
       see `DSTACK_RUNNER_DOWNLOAD_URL` for the details.
+     * `DSTACK_GATEWAY_PACKAGE_URL` – Overrides the URL the `dstack` package is installed from on gateway instances (used as `dstack[gateway] @ <URL>`).
      * `DSTACK_DEFAULT_CREDS_DISABLED` – Disables default credentials detection if set. Defaults to `None`.
 
 ## CLI

dstack-0.21.4/runner/internal/shim/authorized_keys.go

@@ -0,0 +1,273 @@
+package shim
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/dstackai/dstack/runner/internal/common/log"
+)
+
+// publicKeyMarker is appended to the comment field of every authorized_keys entry
+// added by the shim, mirroring the `# added by dstack` marker the server writes when
+// provisioning SSH fleets. sshd ignores everything after the key blob, so the marker
+// has no effect on authentication; it only records that the entry is ours.
+//
+// The marker is what makes Reconcile() safe: only the entries carrying it are rewritten,
+// so a key added by the user, or by the server at fleet provisioning time, is never
+// touched. Entries added by a shim version that did not write the marker are kept
+// forever; leaking a key we cannot prove we added beats revoking it.
+//
+// The marker must be matched as an exact suffix: a substring match on `# added by
+// dstack` would also claim the keys the server adds at fleet provisioning time.
+const publicKeyMarker = "# added by dstack-shim"
+
+const (
+	sshDirName             = ".ssh"
+	authorizedKeysFileName = "authorized_keys"
+	// Modes of the dir and the file when the shim creates them; an existing
+	// authorized_keys file keeps its own mode
+	sshDirMode             = 0o700
+	authorizedKeysFileMode = 0o600
+)
+
+func PublicKeyFingerprint(key string) (string, error) {
+	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(key))
+	if err != nil {
+		return "", fmt.Errorf("parse authorized key: %w", err)
+	}
+	keyFingerprint := ssh.FingerprintSHA256(pk)
+	return keyFingerprint, nil
+}
+
+// canonicalizePublicKey validates a public key received from the server and returns the
+// authorized_keys line to write for it, marked with publicKeyMarker.
+//
+// The line is rebuilt from the parsed key instead of reusing the original entry, so that
+// nothing unvalidated reaches the file.
+func canonicalizePublicKey(publicKey string) (string, error) {
+	key, comment, options, rest, err := ssh.ParseAuthorizedKey([]byte(publicKey))
+	if err != nil {
+		return "", fmt.Errorf("parse public key: %w", err)
+	}
+	// ParseAuthorizedKey stops at the end of the first key it finds
+	if len(bytes.TrimSpace(rest)) > 0 {
+		return "", errors.New("more than one key in a single entry")
+	}
+	// Options are a feature of the authorized_keys format, not of the on-disk public key
+	// format, therefore an entry carrying them is not a public key
+	if len(options) > 0 {
+		return "", errors.New("unexpected authorized_keys options")
+	}
+	// MarshalAuthorizedKey returns a "<type> <base64>\n" line
+	keyLine := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key)))
+	// The comment cannot span lines, but may contain other whitespace, e.g., \t or \r
+	commentFields := strings.Fields(comment)
+	commentFields = append(commentFields, publicKeyMarker)
+	return keyLine + " " + strings.Join(commentFields, " "), nil
+}
+
+// isShimEntry reports whether the authorized_keys line has been added by the shim, that
+// is, whether it carries publicKeyMarker. canonicalizePublicKey() always puts the marker
+// last, therefore matching it as a suffix cannot claim an entry that merely mentions it.
+func isShimEntry(line string) bool {
+	return strings.HasSuffix(strings.TrimRight(line, " \t\r"), " "+publicKeyMarker)
+}
+
+type AuthorizedKeys struct {
+	user   string
+	lookup func(username string) (*user.User, error)
+}
+
+// Reconcile makes the shim-owned part of the user's authorized_keys file exactly the
+// given set of keys: the entries carrying publicKeyMarker are replaced with the entries
+// for publicKeys. Everything else -- keys added by the user, keys added by the server
+// when provisioning an SSH fleet, comments, blank lines -- is kept verbatim and in order.
+//
+// The keys of all the tasks that still need them are passed on every call, so that a task
+// releasing a key shared with another task does not revoke it, see dstackai/dstack#4174.
+// Duplicates collapse into a single entry, which is what makes the file depend on the set
+// of keys in use and not on the number of tasks using them.
+// A key that is also present as an entry the shim does not own is still written as an
+// entry of its own: the two have different owners, and sharing one would let a manual
+// edit revoke the access of a running task.
+//
+// Invalid keys are skipped, so that one bad key does not keep the rest out of the file.
+func (ak AuthorizedKeys) Reconcile(ctx context.Context, publicKeys []string) error {
+	usr, err := ak.lookup(ak.user)
+	if err != nil {
+		return fmt.Errorf("lookup user %s: %w", ak.user, err)
+	}
+	path := authorizedKeysPath(usr.HomeDir)
+
+	lines, mode, err := readAuthorizedKeys(path)
+	if err != nil {
+		return err
+	}
+	kept := make([]string, 0, len(lines)+len(publicKeys))
+	for _, line := range lines {
+		if !isShimEntry(line) {
+			kept = append(kept, line)
+		}
+	}
+	kept = append(kept, shimEntries(ctx, publicKeys)...)
+
+	return writeAuthorizedKeys(path, kept, mode, usr)
+}
+
+// shimEntries returns the marked authorized_keys lines to write for the keys, skipping
+// the invalid ones and collapsing the duplicates
+func shimEntries(ctx context.Context, publicKeys []string) []string {
+	lines := make([]string, 0, len(publicKeys))
+	seen := make(map[string]struct{}, len(publicKeys))
+	for _, publicKey := range publicKeys {
+		line, err := canonicalizePublicKey(publicKey)
+		if err != nil {
+			// Whitespace is collapsed to keep the entry on a single log line
+			log.Error(
+				ctx, "skipping invalid public key",
+				"key", strings.Join(strings.Fields(publicKey), " "), "err", err,
+			)
+			continue
+		}
+		// Matched by fingerprint and not by line, so that the same key submitted with
+		// different comments does not yield two entries
+		fingerprint, err := PublicKeyFingerprint(line)
+		if err != nil {
+			// canonicalizePublicKey() has already parsed the key, so this cannot happen
+			log.Error(ctx, "failed to fingerprint canonicalized key", "err", err)
+			continue
+		}
+		if _, ok := seen[fingerprint]; ok {
+			continue
+		}
+		seen[fingerprint] = struct{}{}
+		lines = append(lines, line)
+	}
+	return lines
+}
+
+// readAuthorizedKeys returns the lines of the file and its mode. A missing file is
+// reported as an empty one with the mode to create it with: sshd treats it the same way,
+// and the shim creates it as the server does when provisioning an SSH fleet.
+func readAuthorizedKeys(path string) ([]string, os.FileMode, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, authorizedKeysFileMode, nil
+		}
+		return nil, 0, fmt.Errorf("open authorized keys: %w", err)
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return nil, 0, fmt.Errorf("stat authorized keys: %w", err)
+	}
+	var lines []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, 0, fmt.Errorf("scan authorized keys: %w", err)
+	}
+	return lines, info.Mode().Perm(), nil
+}
+
+// writeAuthorizedKeys replaces the file with the given lines, creating it and its dir if
+// they do not exist. The content is written to a temporary file in the same dir and
+// renamed over the old one, so that a failed or interrupted write cannot leave the user
+// with a partial file, that is, without some of their keys.
+func writeAuthorizedKeys(path string, lines []string, mode os.FileMode, usr *user.User) (err error) {
+	uid, gid, err := userIDs(usr)
+	if err != nil {
+		return err
+	}
+	dir := filepath.Dir(path)
+	if _, statErr := os.Stat(dir); errors.Is(statErr, os.ErrNotExist) {
+		if err := os.MkdirAll(dir, sshDirMode); err != nil {
+			return fmt.Errorf("create %s: %w", dir, err)
+		}
+		// The shim normally runs as root, therefore a dir it creates must be given
+		// to the user. An existing dir is left alone, as it is not ours to fix
+		if err := os.Chown(dir, uid, gid); err != nil {
+			return fmt.Errorf("chown %s: %w", dir, err)
+		}
+	}
+
+	var content []byte
+	if len(lines) > 0 {
+		// The last line is terminated as well, so that appending to the file by hand
+		// cannot join a new entry to the last one
+		content = []byte(strings.Join(lines, "\n") + "\n")
+	}
+	tempPath := path + ".tmp"
+	defer func() {
+		if err != nil {
+			// sshd does not read the temporary file, so leaving it behind would only
+			// clutter the dir with a half-written copy of the user's keys
+			if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
+				err = errors.Join(err, fmt.Errorf("remove %s: %w", tempPath, removeErr))
+			}
+		}
+	}()
+	if err := writeFileSync(tempPath, content, mode); err != nil {
+		return fmt.Errorf("write authorized keys: %w", err)
+	}
+	// The mode passed to writeFileSync() only applies to a file it creates, and is
+	// masked by the umask, therefore it is set explicitly
+	if err := os.Chmod(tempPath, mode); err != nil {
+		return fmt.Errorf("chmod %s: %w", tempPath, err)
+	}
+	if err := os.Chown(tempPath, uid, gid); err != nil {
+		return fmt.Errorf("chown %s: %w", tempPath, err)
+	}
+	if err := os.Rename(tempPath, path); err != nil {
+		return fmt.Errorf("rename %s: %w", tempPath, err)
+	}
+	return nil
+}
+
+// userIDs returns the numeric ids of the user. os/user reports them as strings, since
+// not all platforms have numeric ids, but Linux, the only platform the shim runs on, does
+func userIDs(usr *user.User) (int, int, error) {
+	uid, err := strconv.Atoi(usr.Uid)
+	if err != nil {
+		return 0, 0, fmt.Errorf("parse uid %q of user %s: %w", usr.Uid, usr.Username, err)
+	}
+	gid, err := strconv.Atoi(usr.Gid)
+	if err != nil {
+		return 0, 0, fmt.Errorf("parse gid %q of user %s: %w", usr.Gid, usr.Username, err)
+	}
+	return uid, gid, nil
+}
+
+func (ak AuthorizedKeys) GetHomeDirectory() (string, error) {
+	usr, err := ak.lookup(ak.user)
+	if err != nil {
+		return "", err
+	}
+	return usr.HomeDir, nil
+}
+
+func (ak AuthorizedKeys) GetAuthorizedKeysPath() (string, error) {
+	homeDir, err := ak.GetHomeDirectory()
+	if err != nil {
+		return "", err
+	}
+	return authorizedKeysPath(homeDir), nil
+}
+
+func authorizedKeysPath(homeDir string) string {
+	return filepath.Join(homeDir, sshDirName, authorizedKeysFileName)
+}