dstack 0.18.39__py3-none-any.whl → 0.18.40__py3-none-any.whl

dstack/_internal/cli/services/configurators/fleet.py

@@ -353,7 +353,7 @@ def _print_plan_header(plan: FleetPlan):
                 f"${plan.max_offer_price:g} max[/]"
             )
         console.print()
-    else:
+    elif fleet_type == "cloud":
         console.print(NO_OFFERS_WARNING)
 
 

dstack/_internal/cli/services/configurators/gateway.py

@@ -22,6 +22,7 @@ from dstack._internal.core.models.gateways import (
     GatewayStatus,
 )
 from dstack._internal.core.models.repos.base import Repo
+from dstack._internal.core.services.diff import diff_models
 from dstack._internal.utils.common import local_time
 from dstack.api._public import Client
 
@@ -56,7 +57,15 @@ class GatewayConfigurator(BaseApplyConfigurator):
             confirm_message += "Create the gateway?"
         else:
             action_message += f"Found gateway [code]{plan.spec.configuration.name}[/]."
-            if plan.current_resource.configuration == plan.spec.configuration:
+            diff = diff_models(
+                plan.spec.configuration,
+                plan.current_resource.configuration,
+            )
+            changed_fields = list(diff.keys())
+            if (
+                plan.current_resource.configuration == plan.spec.configuration
+                or changed_fields == ["default"]
+            ):
                 if command_args.yes and not command_args.force:
                     # --force is required only with --yes,
                     # otherwise we may ask for force apply interactively.

dstack/_internal/cli/utils/common.py

@@ -25,8 +25,7 @@ LIVE_TABLE_REFRESH_RATE_PER_SEC = 1
 LIVE_TABLE_PROVISION_INTERVAL_SECS = 2
 NO_OFFERS_WARNING = (
     "[warning]"
-    "Could not find instance offers that match your requirements."
-    " Please check the requirements table above or visit the troubleshooting guide:"
+    "No matching instance offers available. Possible reasons:"
     " https://dstack.ai/docs/guides/troubleshooting/#no-offers"
     "[/]\n"
 )

dstack/_internal/core/models/configurations.py

@@ -21,6 +21,7 @@ from dstack._internal.core.models.volumes import MountPoint, VolumeConfiguration
 CommandsList = List[str]
 ValidPort = conint(gt=0, le=65536)
 SERVICE_HTTPS_DEFAULT = True
+STRIP_PREFIX_DEFAULT = True
 
 
 class RunConfigurationType(str, Enum):
@@ -130,6 +131,16 @@ class BaseRunConfiguration(CoreModel):
             description="Use image with NVIDIA CUDA Compiler (NVCC) included. Mutually exclusive with `image`"
         ),
     ]
+    single_branch: Annotated[
+        Optional[bool],
+        Field(
+            description=(
+                "Whether to clone and track only the current branch or all remote branches."
+                " Relevant only when using remote Git repos."
+                " Defaults to `false` for dev environments and to `true` for tasks and services"
+            )
+        ),
+    ] = None
     env: Annotated[
         Env,
         Field(description="The mapping or the list of environment variables"),
@@ -236,6 +247,16 @@ class ServiceConfigurationParams(CoreModel):
             ),
         ),
     ] = None
+    strip_prefix: Annotated[
+        bool,
+        Field(
+            description=(
+                "Strip the `/proxy/services/<project name>/<run name>/` path prefix"
+                " when forwarding requests to the service. Only takes effect"
+                " when running the service without a gateway"
+            )
+        ),
+    ] = STRIP_PREFIX_DEFAULT
     model: Annotated[
         Optional[Union[AnyModel, str]],
         Field(

dstack/_internal/core/models/runs.py

@@ -184,6 +184,7 @@ class JobSpec(CoreModel):
     home_dir: Optional[str]
     image_name: str
     privileged: bool = False
+    single_branch: Optional[bool] = None
     max_duration: Optional[int]
     stop_duration: Optional[int] = None
     registry_auth: Optional[RegistryAuth]

dstack/_internal/core/models/volumes.py

@@ -136,6 +136,15 @@ class VolumeMountPoint(CoreModel):
 class InstanceMountPoint(CoreModel):
     instance_path: Annotated[str, Field(description="The absolute path on the instance (host)")]
     path: Annotated[str, Field(description="The absolute path in the container")]
+    optional: Annotated[
+        bool,
+        Field(
+            description=(
+                "Allow running without this volume"
+                " in backends that do not support instance volumes"
+            ),
+        ),
+    ] = False
 
     _validate_instance_path = validator("instance_path", allow_reuse=True)(
         _validate_mount_point_path

dstack/_internal/core/services/logs.py

@@ -42,11 +42,14 @@ class URLReplacer:
         qs = {k: v[0] for k, v in urllib.parse.parse_qs(url.query).items()}
         if app_spec and app_spec.url_query_params is not None:
             qs.update({k.encode(): v.encode() for k, v in app_spec.url_query_params.items()})
+        path = url.path
+        if not path.startswith(self.path_prefix.removesuffix(b"/")):
+            path = concat_url_path(self.path_prefix, path)
 
         url = url._replace(
             scheme=("https" if self.secure else "http").encode(),
             netloc=(self.hostname if omit_port else f"{self.hostname}:{local_port}").encode(),
-            path=concat_url_path(self.path_prefix, url.path),
+            path=path,
             query=urllib.parse.urlencode(qs).encode(),
         )
         return url.geturl()

dstack/_internal/core/services/ssh/attach.py

@@ -56,6 +56,7 @@ class SSHAttach:
         ssh_port: int,
         container_ssh_port: int,
         user: str,
+        container_user: str,
         id_rsa_path: PathLike,
         ports_lock: PortsLock,
         run_name: str,
@@ -74,7 +75,7 @@ class SSHAttach:
         self.control_sock_path = FilePath(control_sock_path)
         self.identity_file = FilePath(id_rsa_path)
         self.tunnel = SSHTunnel(
-            destination=run_name,
+            destination=f"root@{run_name}",
             identity=self.identity_file,
             forwarded_sockets=ports_to_forwarded_sockets(
                 ports=self.ports,
@@ -91,7 +92,7 @@ class SSHAttach:
             self.host_config = {
                 "HostName": hostname,
                 "Port": ssh_port,
-                "User": user,
+                "User": user if dockerized else container_user,
                 "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
@@ -111,7 +112,7 @@ class SSHAttach:
             self.container_config = {
                 "HostName": "localhost",
                 "Port": container_ssh_port,
-                "User": "root",  # TODO(#1535): support non-root images properly
+                "User": container_user,
                 "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
@@ -122,7 +123,7 @@ class SSHAttach:
             self.container_config = {
                 "HostName": hostname,
                 "Port": ssh_port,
-                "User": user,
+                "User": container_user,
                 "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
@@ -136,7 +137,7 @@ class SSHAttach:
             self.host_config = {
                 "HostName": hostname,
                 "Port": container_ssh_port,
-                "User": "root",  # TODO(#1535): support non-root images properly
+                "User": container_user,
                 "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",

dstack/_internal/proxy/lib/models.py

@@ -32,6 +32,7 @@ class Service(ImmutableModel):
     https: Optional[bool]  # only used on gateways
     auth: bool
     client_max_body_size: int  # only enforced on gateways
+    strip_prefix: bool = True  # only used in-server
     replicas: tuple[Replica, ...]
 
     @property

dstack/_internal/proxy/lib/testing/common.py

@@ -29,6 +29,7 @@ def make_service(
     domain: Optional[str] = None,
     https: Optional[bool] = None,
     auth: bool = False,
+    strip_prefix: bool = True,
 ) -> Service:
     return Service(
         project_name=project_name,
@@ -37,6 +38,7 @@ def make_service(
         https=https,
         auth=auth,
         client_max_body_size=2**20,
+        strip_prefix=strip_prefix,
         replicas=(
             Replica(
                 id="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",

dstack/_internal/server/app.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Awaitable, Callable, List
 
 import sentry_sdk
-from fastapi import FastAPI, Request, status
+from fastapi import FastAPI, Request, Response, status
 from fastapi.datastructures import URL
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
@@ -215,10 +215,14 @@ def register_routes(app: FastAPI, ui: bool = True):
     @app.middleware("http")
     async def log_request(request: Request, call_next):
         start_time = time.time()
-        response = await call_next(request)
+        response: Response = await call_next(request)
         process_time = time.time() - start_time
         logger.debug(
-            "Processed request %s %s in %s", request.method, request.url, f"{process_time:0.6f}s"
+            "Processed request %s %s in %s. Status: %s",
+            request.method,
+            request.url,
+            f"{process_time:0.6f}s",
+            response.status_code,
         )
         return response
 

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -64,7 +64,7 @@ from dstack._internal.server.services.pools import (
 )
 from dstack._internal.server.services.runs import (
     check_can_attach_run_volumes,
-    check_run_spec_has_instance_mounts,
+    check_run_spec_requires_instance_mounts,
     get_offer_volumes,
     get_run_volume_models,
     get_run_volumes,
@@ -418,7 +418,7 @@ async def _run_job_on_new_instance(
         master_job_provisioning_data=master_job_provisioning_data,
         volumes=volumes,
         privileged=job.job_spec.privileged,
-        instance_mounts=check_run_spec_has_instance_mounts(run.run_spec),
+        instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec),
     )
     # Limit number of offers tried to prevent long-running processing
     # in case all offers fail.

dstack/_internal/server/schemas/runner.py

@@ -61,6 +61,7 @@ class SubmitBody(CoreModel):
                 "entrypoint",
                 "env",
                 "gateway",
+                "single_branch",
                 "max_duration",
                 "working_dir",
             }

dstack/_internal/server/services/fleets.py

@@ -239,7 +239,6 @@ async def get_plan(
     user: UserModel,
     spec: FleetSpec,
 ) -> FleetPlan:
-    # TODO: refactor offers logic into a separate module to avoid depending on runs
     current_fleet: Optional[Fleet] = None
     current_fleet_id: Optional[uuid.UUID] = None
     if spec.configuration.name is not None:
@@ -259,6 +258,7 @@ async def get_plan(
             requirements=_get_fleet_requirements(spec),
         )
         offers = [offer for _, offer in offers_with_backends]
+    _remove_fleet_spec_sensitive_info(spec)
     plan = FleetPlan(
         project_name=project.name,
         user=user.name,

dstack/_internal/server/services/jobs/configurators/base.py

@@ -63,6 +63,10 @@ class JobConfigurator(ABC):
     def _shell_commands(self) -> List[str]:
         pass
 
+    @abstractmethod
+    def _default_single_branch(self) -> bool:
+        pass
+
     @abstractmethod
     def _default_max_duration(self) -> Optional[int]:
         pass
@@ -104,6 +108,7 @@ class JobConfigurator(ABC):
             image_name=self._image_name(),
             user=await self._user(),
             privileged=self._privileged(),
+            single_branch=self._single_branch(),
             max_duration=self._max_duration(),
             stop_duration=self._stop_duration(),
             registry_auth=self._registry_auth(),
@@ -172,6 +177,11 @@ class JobConfigurator(ABC):
     def _privileged(self) -> bool:
         return self.run_spec.configuration.privileged
 
+    def _single_branch(self) -> bool:
+        if self.run_spec.configuration.single_branch is None:
+            return self._default_single_branch()
+        return self.run_spec.configuration.single_branch
+
     def _max_duration(self) -> Optional[int]:
         if self.run_spec.merged_profile.max_duration in [None, True]:
             return self._default_max_duration()

dstack/_internal/server/services/jobs/configurators/dev.py

@@ -40,6 +40,9 @@ class DevEnvironmentJobConfigurator(JobConfigurator):
         commands += ["tail -f /dev/null"]  # idle
         return commands
 
+    def _default_single_branch(self) -> bool:
+        return False
+
     def _default_max_duration(self) -> Optional[int]:
         return DEFAULT_MAX_DURATION_SECONDS
 

dstack/_internal/server/services/jobs/configurators/service.py

@@ -11,6 +11,9 @@ class ServiceJobConfigurator(JobConfigurator):
     def _shell_commands(self) -> List[str]:
         return self.run_spec.configuration.commands
 
+    def _default_single_branch(self) -> bool:
+        return True
+
     def _default_max_duration(self) -> Optional[int]:
         return None
 

dstack/_internal/server/services/jobs/configurators/task.py

@@ -25,6 +25,9 @@ class TaskJobConfigurator(JobConfigurator):
     def _shell_commands(self) -> List[str]:
         return self.run_spec.configuration.commands
 
+    def _default_single_branch(self) -> bool:
+        return True
+
     def _default_max_duration(self) -> Optional[int]:
         return DEFAULT_MAX_DURATION_SECONDS
 

dstack/_internal/server/services/proxy/repo.py

@@ -98,6 +98,7 @@ class ServerProxyRepo(BaseProxyRepo):
             https=None,
             auth=run_spec.configuration.auth,
             client_max_body_size=DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE,
+            strip_prefix=run_spec.configuration.strip_prefix,
             replicas=tuple(replicas),
         )
 

dstack/_internal/server/services/proxy/services/service_proxy.py

@@ -12,6 +12,7 @@ from dstack._internal.proxy.lib.services.service_connection import (
     ServiceConnectionPool,
     get_service_replica_client,
 )
+from dstack._internal.utils.common import concat_url_path
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -37,6 +38,9 @@ async def proxy(
 
     client = await get_service_replica_client(service, repo, service_conn_pool)
 
+    if not service.strip_prefix:
+        path = concat_url_path(request.scope.get("root_path", "/"), request.url.path)
+
     try:
         upstream_request = await build_upstream_request(request, path, client)
     except ClientDisconnect:

dstack/_internal/server/services/runs.py

@@ -330,7 +330,7 @@ async def get_plan(
             multinode=jobs[0].job_spec.jobs_per_replica > 1,
             volumes=volumes,
             privileged=jobs[0].job_spec.privileged,
-            instance_mounts=check_run_spec_has_instance_mounts(run_spec),
+            instance_mounts=check_run_spec_requires_instance_mounts(run_spec),
         )
 
     job_plans = []
@@ -897,9 +897,10 @@ def get_offer_mount_point_volume(
     raise ServerClientError("Failed to find an eligible volume for the mount point")
 
 
-def check_run_spec_has_instance_mounts(run_spec: RunSpec) -> bool:
+def check_run_spec_requires_instance_mounts(run_spec: RunSpec) -> bool:
     return any(
-        is_core_model_instance(mp, InstanceMountPoint) for mp in run_spec.configuration.volumes
+        is_core_model_instance(mp, InstanceMountPoint) and not mp.optional
+        for mp in run_spec.configuration.volumes
     )
 
 
@@ -967,7 +968,7 @@ def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
 _UPDATABLE_SPEC_FIELDS = ["repo_code_hash", "configuration"]
 # Most service fields can be updated via replica redeployment.
 # TODO: Allow updating other fields when a rolling deployment is supported.
-_UPDATABLE_CONFIGURATION_FIELDS = ["replicas", "scaling"]
+_UPDATABLE_CONFIGURATION_FIELDS = ["replicas", "scaling", "strip_prefix"]
 
 
 def _can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec) -> bool:

dstack/api/_public/runs.py

@@ -324,11 +324,19 @@ class Run(ABC):
             if runtime_data is not None and runtime_data.ports is not None:
                 container_ssh_port = runtime_data.ports.get(container_ssh_port, container_ssh_port)
 
+            # TODO: get login name from runner in case it's not specified in the run configuration
+            # (i.e. the default image user is used, and it is not root)
+            if job.job_spec.user is not None and job.job_spec.user.username is not None:
+                container_user = job.job_spec.user.username
+            else:
+                container_user = "root"
+
             self._ssh_attach = SSHAttach(
                 hostname=provisioning_data.hostname,
                 ssh_port=provisioning_data.ssh_port,
                 container_ssh_port=container_ssh_port,
                 user=provisioning_data.username,
+                container_user=container_user,
                 id_rsa_path=ssh_identity_file,
                 ports_lock=self._ports_lock,
                 run_name=name,

dstack/api/server/__init__.py

@@ -133,9 +133,16 @@ class APIClient:
         else:
             raise ClientError(f"Failed to connect to dstack server {self._base_url}")
 
+        if 400 <= resp.status_code < 600:
+            logger.debug(
+                "Error requesting %s. Status: %s. Headers: %s. Body: %s",
+                resp.request.url,
+                resp.status_code,
+                resp.headers,
+                resp.content,
+            )
+
         if raise_for_status:
-            if resp.status_code == 500:
-                raise ClientError("Unexpected dstack server error")
             if resp.status_code == 400:  # raise ServerClientError
                 detail: List[Dict] = resp.json()["detail"]
                 if len(detail) == 1 and detail[0]["code"] in _server_client_errors:
@@ -145,7 +152,16 @@ class APIClient:
             if resp.status_code == 422:
                 formatted_error = pprint.pformat(resp.json())
                 raise ClientError(f"Server validation error: \n{formatted_error}")
-            resp.raise_for_status()
+            if resp.status_code == 403:
+                raise ClientError(
+                    f"Access to {resp.request.url} is denied. Please check your access token"
+                )
+            if 400 <= resp.status_code < 600:
+                raise ClientError(
+                    f"Unexpected error: status code {resp.status_code}"
+                    f" when requesting {resp.request.url}."
+                    " Check server logs or run with DSTACK_CLI_LOG_LEVEL=DEBUG to see more details"
+                )
         return resp
 
 

dstack/api/server/_runs.py

@@ -1,9 +1,14 @@
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 from uuid import UUID
 
 from pydantic import parse_obj_as
 
+from dstack._internal.core.models.common import is_core_model_instance
+from dstack._internal.core.models.configurations import (
+    STRIP_PREFIX_DEFAULT,
+    ServiceConfiguration,
+)
 from dstack._internal.core.models.pools import Instance
 from dstack._internal.core.models.profiles import Profile
 from dstack._internal.core.models.runs import (
@@ -14,6 +19,7 @@ from dstack._internal.core.models.runs import (
     RunPlan,
     RunSpec,
 )
+from dstack._internal.core.models.volumes import InstanceMountPoint
 from dstack._internal.server.schemas.runs import (
     ApplyRunPlanRequest,
     CreateInstanceRequest,
@@ -117,34 +123,48 @@ class RunsAPIClient(APIClientGroup):
 
 def _get_run_spec_excludes(run_spec: RunSpec) -> Optional[dict]:
     spec_excludes: dict[str, set[str]] = {}
-    configuration_excludes: set[str] = set()
+    configuration_excludes: dict[str, Any] = {}
     profile_excludes: set[str] = set()
     configuration = run_spec.configuration
     profile = run_spec.profile
 
     # client >= 0.18.18 / server <= 0.18.17 compatibility tweak
     if not configuration.privileged:
-        configuration_excludes.add("privileged")
+        configuration_excludes["privileged"] = True
     # client >= 0.18.23 / server <= 0.18.22 compatibility tweak
     if configuration.type == "service" and configuration.gateway is None:
-        configuration_excludes.add("gateway")
+        configuration_excludes["gateway"] = True
     # client >= 0.18.30 / server <= 0.18.29 compatibility tweak
     if run_spec.configuration.user is None:
-        configuration_excludes.add("user")
+        configuration_excludes["user"] = True
     # client >= 0.18.30 / server <= 0.18.29 compatibility tweak
     if configuration.reservation is None:
-        configuration_excludes.add("reservation")
+        configuration_excludes["reservation"] = True
     if profile is not None and profile.reservation is None:
         profile_excludes.add("reservation")
     if configuration.idle_duration is None:
-        configuration_excludes.add("idle_duration")
+        configuration_excludes["idle_duration"] = True
     if profile is not None and profile.idle_duration is None:
         profile_excludes.add("idle_duration")
     # client >= 0.18.38 / server <= 0.18.37 compatibility tweak
     if configuration.stop_duration is None:
-        configuration_excludes.add("stop_duration")
+        configuration_excludes["stop_duration"] = True
     if profile is not None and profile.stop_duration is None:
         profile_excludes.add("stop_duration")
+    # client >= 0.18.40 / server <= 0.18.39 compatibility tweak
+    if (
+        is_core_model_instance(configuration, ServiceConfiguration)
+        and configuration.strip_prefix == STRIP_PREFIX_DEFAULT
+    ):
+        configuration_excludes["strip_prefix"] = True
+    if configuration.single_branch is None:
+        configuration_excludes["single_branch"] = True
+    if all(
+        not is_core_model_instance(v, InstanceMountPoint) or not v.optional
+        for v in configuration.volumes
+    ):
+        configuration_excludes["volumes"] = {"__all__": {"optional"}}
+
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes
     if profile_excludes:

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.18.39"
+__version__ = "0.18.40"
 __is_release__ = True
 base_image = "0.6"

{dstack-0.18.39.dist-info → dstack-0.18.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.18.39
+Version: 0.18.40
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov
@@ -303,17 +303,16 @@ for AI workloads both in the cloud and on-prem, speeding up the development, tra
 
 #### Accelerators
 
-`dstack` supports `NVIDIA GPU`, `AMD GPU`, and `Google Cloud TPU` out of the box.
+`dstack` supports `NVIDIA`, `AMD`, `Google TPU`, and `Intel Gaudi` accelerators out of the box.
 
 ## Major news ✨
 
-- [2025/01] [dstack 0.18.35: Vultr backend](https://github.com/dstackai/dstack/releases/tag/0.18.35)
-- [2024/12] [dstack 0.18.33: TPU v6e support](https://github.com/dstackai/dstack/releases/tag/0.18.33)
+- [2025/01] [dstack 0.18.38: Intel Gaudi](https://github.com/dstackai/dstack/releases/tag/0.18.38)
+- [2025/01] [dstack 0.18.35: Vultr](https://github.com/dstackai/dstack/releases/tag/0.18.35)
+- [2024/12] [dstack 0.18.32: TPU v6e](https://github.com/dstackai/dstack/releases/tag/0.18.32)
 - [2024/12] [dstack 0.18.30: AWS Capacity Reservations and Capacity Blocks](https://github.com/dstackai/dstack/releases/tag/0.18.30)
-- [2024/11] [dstack 0.18.23: Gateway is optional](https://github.com/dstackai/dstack/releases/tag/0.18.23)
 - [2024/10] [dstack 0.18.21: Instance volumes](https://github.com/dstackai/dstack/releases/tag/0.18.21)
-- [2024/10] [dstack 0.18.18: Hardware metrics](https://github.com/dstackai/dstack/releases/tag/0.18.18)
-- [2024/10] [dstack 0.18.17: AMD support with SSH fleets, AWS EFA](https://github.com/dstackai/dstack/releases/tag/0.18.17)
+- [2024/10] [dstack 0.18.18: Hardware metrics monitoring](https://github.com/dstackai/dstack/releases/tag/0.18.18)
 
 ## Installation
 

{dstack-0.18.39.dist-info → dstack-0.18.40.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=7qGndtEXTRftCO2WIy-ZZmNA1UtbKIp-_gFrfNOHU3I,65
+dstack/version.py,sha256=v4U-Zn6q3of9xvVoHDDcM25yWZKnFyXN_K1y22za1rw,65
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
@@ -27,12 +27,12 @@ dstack/_internal/cli/services/profile.py,sha256=Fl052TeMCbWO1Q6HiaPVD9CdfClzZAkk
 dstack/_internal/cli/services/repos.py,sha256=n2INRJtTYV5y60-Yp3F-V5kJkqwZI_fdGXPnxvZhkNw,3321
 dstack/_internal/cli/services/configurators/__init__.py,sha256=z94VPBFqybP8Zpwy3CzYxmpPAqYBOvRRLpXoz2H4GKI,2697
 dstack/_internal/cli/services/configurators/base.py,sha256=YsyNmHTv1wVbvaAjQi_qNQAb4oK9T_Wgu6RKg-uyzvY,3379
-dstack/_internal/cli/services/configurators/fleet.py,sha256=YbZh6x1UuvECh7ZMmxvQQfGzBUi0Kxn3stsVutKWLKo,14259
-dstack/_internal/cli/services/configurators/gateway.py,sha256=0pwAPFYoANioeh6RJiq5IggMc4ywAVfEz3hX91FoPaY,8061
+dstack/_internal/cli/services/configurators/fleet.py,sha256=iqNJSoji7IiVb3Kb4KA6XTzI5k2xUYF7DK9aE7iverk,14281
+dstack/_internal/cli/services/configurators/gateway.py,sha256=BlLRBC-rl9a0xpO3c1N4k5yinvVQB5YU238g-8NMINY,8389
 dstack/_internal/cli/services/configurators/run.py,sha256=GLGhCcrOzp96Yc6RiISg-DZEM43WT7bhsZmD8z0oAVA,23271
 dstack/_internal/cli/services/configurators/volume.py,sha256=UAKfx-J5fUdnnR2RBvbRJy0YKzXOoub4f4ywMsZwk38,7983
 dstack/_internal/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/cli/utils/common.py,sha256=xuHxRG6pl9Amt8PRbpu0wav6P5ac2wtLUMBj0URxRSc,1852
+dstack/_internal/cli/utils/common.py,sha256=rfmzqrsgR3rXW3wj0vxDdvrhUUg2aIy4A6E9MZbd55g,1763
 dstack/_internal/cli/utils/fleet.py,sha256=tumFG4qgrKjmb0Iik64tWcPrenaOjpeXC6AZSezQ3_A,2991
 dstack/_internal/cli/utils/gateway.py,sha256=jMytH6u3x8hctMhm9bcmXLJxSgTXmpW8M9abATQrw3c,1474
 dstack/_internal/cli/utils/rich.py,sha256=Gx1MJU929kMKsbdo9qF7XHARNta2426Ssb-xMLVhwbQ,5710
@@ -114,7 +114,7 @@ dstack/_internal/core/backends/vultr/config.py,sha256=TqjT2yhW9rAiDpJ96gBFTM638F
 dstack/_internal/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/models/common.py,sha256=XNfEP_lHAs398oDfSVn8kCX79GIWvl0hDs05Rz1CBic,2579
 dstack/_internal/core/models/config.py,sha256=JJ7rT7dztzTWCY5TkoyxXxTvG5D4IFYhGe7EzwkLOWQ,581
-dstack/_internal/core/models/configurations.py,sha256=0ZCQ7atvMnIKvpK1FFmw7IXox3WvAgyfqAtaJmWCLns,13030
+dstack/_internal/core/models/configurations.py,sha256=BrxbkQ5xhCw-pT7DsEdJeLa92uEWQM3LHPmSiYOohHE,13806
 dstack/_internal/core/models/envs.py,sha256=ls2ahIhEb3LYLX_auH3F46uTRp2zdlMWKEpjmU_V3S4,5017
 dstack/_internal/core/models/fleets.py,sha256=tSAFkBugchMhvLRk8c0loRpteTYnbA0sDhUI8q_EcXI,8969
 dstack/_internal/core/models/gateways.py,sha256=n9lPq6X0n2a7fF-CseiBVDDwasJsU3VQumTetMsfZcc,3465
@@ -126,13 +126,13 @@ dstack/_internal/core/models/pools.py,sha256=mZSZ_9CKTIhnoXfkQLEOjJg_EDZW7kXXigC
 dstack/_internal/core/models/profiles.py,sha256=CRvVf2bph56HOz7OmpCwreOR2yCHdfVgovyjKxYBsiM,8011
 dstack/_internal/core/models/projects.py,sha256=hY3rhLWkuDRsavAdNvQaK6IhnTr7yBHTmNwy9k7zjVE,643
 dstack/_internal/core/models/resources.py,sha256=AzBqGEZLHmg-HpjQtcb_W4pJQRxRtDyeuiHO4berdBA,12408
-dstack/_internal/core/models/runs.py,sha256=fHpvKW_4LfhhtbzvTHwVXRye6gzQikclmEudKPGQEHk,16930
+dstack/_internal/core/models/runs.py,sha256=CqyQ_wMne7G9XHLm2BRH84J0tgG3Ny77No1A8hTCmgQ,16971
 dstack/_internal/core/models/secrets.py,sha256=OlnBI8ESBnpjSqB0-Vr3z8JcqB2Ydfiwo8IJUuM5jAc,219
 dstack/_internal/core/models/server.py,sha256=Hkc1v2s3KOiwslsWVmhUOAzcSeREoG-HD1SzSX9WUGg,152
 dstack/_internal/core/models/services.py,sha256=2Hpi7j0Q1shaf_0wd0C0044AJAmuYi-D3qx3PH849oI,3076
 dstack/_internal/core/models/unix.py,sha256=KxnSQELnkAjjuUgYcQKVkf-UAbYREBD8WCWDvHfOkuA,1915
 dstack/_internal/core/models/users.py,sha256=o_rd0GAmd6jufypVUs9P12NRri3rgAPDt-KxnqNNsGw,703
-dstack/_internal/core/models/volumes.py,sha256=piArLbgpUyejRJW2vtluqmqXhLmyj9rhzsCfPPGAZ50,4975
+dstack/_internal/core/models/volumes.py,sha256=MagqSb9jB2WnQlP-xVYkkPYHtKh9r54ajVEUYNF9iKc,5215
 dstack/_internal/core/models/backends/__init__.py,sha256=iPMAv4j7gpHc0cjt3SxzQGb-sywms8LUXt7IjtKNPnM,5589
 dstack/_internal/core/models/backends/aws.py,sha256=H0RB7pTVb7vjW_-MDpBBwPLbAoKDRtYFHe_n4ppXz6w,2463
 dstack/_internal/core/models/backends/azure.py,sha256=u5R8vLVS85X27P17IZ1L-EcXmxg08sC0E1EFxkVG2YA,1996
@@ -157,12 +157,12 @@ dstack/_internal/core/models/repos/virtual.py,sha256=1p6k2vBfWI6UT91os3lZfV5wl8b
 dstack/_internal/core/services/__init__.py,sha256=Utrh2zbYsP6IBGWMGROaHUefgo_TvLloN3logcdhuC4,282
 dstack/_internal/core/services/api_client.py,sha256=HTQ0fcZciUh-nmfV09hUt__Z4N2zq_R12xAiYXD0Jy0,650
 dstack/_internal/core/services/diff.py,sha256=lw7vTOYnGtRWlEulgVlHt1KjLdC8VJpfBcqMRuFZzSQ,532
-dstack/_internal/core/services/logs.py,sha256=63s5OKyrZEs-0iLBKRFSn5VldelyWY75gD4RHj_Ejlo,2053
+dstack/_internal/core/services/logs.py,sha256=7_eJdH4MD-3rVb4A6rIJfjj_p4jzUOCmjRVlPD-UDsg,2166
 dstack/_internal/core/services/profiles.py,sha256=SJGT6LG8JmrfsqlyXOkPZd61BDwZ4Q1ngcmDVgq826M,2160
 dstack/_internal/core/services/repos.py,sha256=YgTmLX8U1Y5cz2dG9yThK1-Geq02iik_C6DGZ0y6uEY,5311
 dstack/_internal/core/services/configs/__init__.py,sha256=2jwu1w0vEDNcKGdeI15SS4EtpLSZzRpG8xZslHgTJmY,5484
 dstack/_internal/core/services/ssh/__init__.py,sha256=UhgC3Lv3CPSGqSPEQZIKOfLKUlCFnaB0uqPQhfKCFt0,878
-dstack/_internal/core/services/ssh/attach.py,sha256=4KNiHnM5vsJ8SNWcKGRqhZSKygxDoGLfMBkIAICLHEE,7488
+dstack/_internal/core/services/ssh/attach.py,sha256=A-A8H9lwdIFrxk6FOtHaCZ02fNXo73yz4c-QN1_BQ98,7489
 dstack/_internal/core/services/ssh/client.py,sha256=FUg74jNOagYR3OKuWMBwNgJWIDwHfYrIMSER8xn92D8,4102
 dstack/_internal/core/services/ssh/ports.py,sha256=yWgcACqTNwfLMG24U28LYXlZuEb1V7cSiYBQBqVN5yU,2884
 dstack/_internal/core/services/ssh/tunnel.py,sha256=lHNK1hEcqMP_T_Y6cM_OLHVIRCEA4hud-sM5Jmp4svM,10296
@@ -201,7 +201,7 @@ dstack/_internal/proxy/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 dstack/_internal/proxy/lib/auth.py,sha256=bI85UFTupPxlcAJZqpOuqS64I4MeIyzp_gHxM3916V4,183
 dstack/_internal/proxy/lib/deps.py,sha256=fFq1mJQK_XcqCNgORTowhc0jCblRiGIK1TeRDC2wkL8,3538
 dstack/_internal/proxy/lib/errors.py,sha256=sUO1dDMkr0oKghB0yUdoHLhB66tX--FkHWrJ5jzPB4w,437
-dstack/_internal/proxy/lib/models.py,sha256=MC8ATwPdgXi6l2SCQjuR8WFp-yuuhCn0OD8DU9QXZsg,2625
+dstack/_internal/proxy/lib/models.py,sha256=duixvPYU6lnusp3tYjxU675Zo3idXfOiAPQ1ZRnv1h8,2678
 dstack/_internal/proxy/lib/repo.py,sha256=zkWZ9XZzQHfCa-eifec7H7UYnJZLgeRuiQls7Rc2pkA,781
 dstack/_internal/proxy/lib/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/routers/model_proxy.py,sha256=57GFRpVRXcVY-347HnUSUr4w4RsxsjLuuZiJs8DwDpM,3895
@@ -217,9 +217,9 @@ dstack/_internal/proxy/lib/services/model_proxy/clients/openai.py,sha256=K8cMgGQ
 dstack/_internal/proxy/lib/services/model_proxy/clients/tgi.py,sha256=7TsqrlUVo2PYdEIbmmGNTz0ktf_T8l4Zpw4w4HYAYTs,7845
 dstack/_internal/proxy/lib/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/testing/auth.py,sha256=-firWTnis9Eogoi58BURv1S-te4Hf9x1Q_aYLZMDll4,465
-dstack/_internal/proxy/lib/testing/common.py,sha256=peH9vQF0g4wsJCcF_FasKOXmSlCFVSEuwOBpA0CHANc,1452
+dstack/_internal/proxy/lib/testing/common.py,sha256=5e2VYboMqjBnUxkvidaWLoQ-uaBGh_hnURb_VJc38q4,1518
 dstack/_internal/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/app.py,sha256=Bz875CPuYuewbqoo7slo9b1UckLWz1TTvBJtu50o9qI,11214
+dstack/_internal/server/app.py,sha256=pPto7JaBEkwvPF2FR8sqDzwIyMovI_85xk6WXfnnCYU,11317
 dstack/_internal/server/db.py,sha256=WjuqmjG3QAZmSMCeUaJ_ynbowlHuNAvYCZO649cTPHc,3210
 dstack/_internal/server/deps.py,sha256=31e8SU_ogPJWHIDLkgl7cuC_5V91xbJoLyAj17VanfM,670
 dstack/_internal/server/main.py,sha256=kztKhCYNoHSDyJJQScWfZXE0naNleJOCQULW6dd8SGw,109
@@ -234,7 +234,7 @@ dstack/_internal/server/background/tasks/process_metrics.py,sha256=99Uu41B-XS7QZ
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=DZctiMF2TXKbYeygoz--ZiC9MXVL1sQJgq0oqRIc7hg,3858
 dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=9vRGefwyjBagmOyot5tjBi6tXP9gwycdrLGqs_-Iu14,27634
 dstack/_internal/server/background/tasks/process_runs.py,sha256=LPe0RvDmNGSnIT2RNEYh5TceFx69z_JPYxwpU8mP9kQ,17254
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=F-w8njVMXF_fmKZayzptChCn-Unn4dTq7WDMAZ0sFv4,24354
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=CkLdkt3D59RK3UkzLqvHkXZty2Zpf-Ksk3mmVQskHgg,24364
 dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=Y_ISp_lY0S0H-Qn7s5t-jVBYiBPoS7080LM5HbP1pRY,3745
 dstack/_internal/server/background/tasks/process_volumes.py,sha256=UoMtYjX6MbSoMXOOuBhA0FVIlQMT_wZWsRNeYVV3cSI,4703
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -319,7 +319,7 @@ dstack/_internal/server/schemas/logs.py,sha256=JGt39fBEFRjHhlGT1jIC6kwQhujxPO8ue
 dstack/_internal/server/schemas/pools.py,sha256=gPUUXLVwwud_nyOc5cXfSqz-6ladlnBb4Uzu4SJgCg4,734
 dstack/_internal/server/schemas/projects.py,sha256=-YIaAYxMgOMKtw-5xPTmmBBwa_2tjJ3jqVPYEnOeTF8,437
 dstack/_internal/server/schemas/repos.py,sha256=dRmT4CEs1lXN_kB8ZKtL1aZGLuBy-zGfPxLXTn4FwBY,2027
-dstack/_internal/server/schemas/runner.py,sha256=Nq4vQHsOiry2iIET2KvBYVCJkQlalW6QqjqxWNXW2Vw,4159
+dstack/_internal/server/schemas/runner.py,sha256=Avqfu4iorAta_0BTWXfk_ydz4V0Eu0_6ayo3DHzH8HQ,4192
 dstack/_internal/server/schemas/runs.py,sha256=Tp-_q_XijeuH-CR14y5xM1-B4TfTDSzWouVCbsg3NsU,1774
 dstack/_internal/server/schemas/secrets.py,sha256=mfqLSM7PqxVQ-GIWB6RfPRUOvSvvaRv-JxXAYxZ6dyY,373
 dstack/_internal/server/schemas/users.py,sha256=FuDqwRVe3mOmv497vOZKjI0a_d4Wt2g4ZiCJcyfHEKA,495
@@ -329,7 +329,7 @@ dstack/_internal/server/security/permissions.py,sha256=K756I8fnBLBrq4PZwz1ttt74H
 dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/config.py,sha256=rXGihEDrfdDJtDsfeYlbHQW8eGej56VDYTV6BJ_aWjg,29241
 dstack/_internal/server/services/docker.py,sha256=3EcYPiVsrNBGDQYOb60QJ241mzTT6lJROYQXIwt-8dk,5351
-dstack/_internal/server/services/fleets.py,sha256=XmZWulyVLxjWRDhVVTqh9Z12TPdGZR6B8RBZdg0P__s,28038
+dstack/_internal/server/services/fleets.py,sha256=SOU6m-EVbRdpfTkT7I0UUyVXQJTF4_zpcSNWEIb1lQo,27998
 dstack/_internal/server/services/locking.py,sha256=UV5kc-BgROBuNDpBrp8jkcf-fHPOpHLc8JufL-8mbN8,2453
 dstack/_internal/server/services/logging.py,sha256=Nu1628kW2hqB__N0Eyr07wGWjVWxfyJnczonTJ72kSM,417
 dstack/_internal/server/services/logs.py,sha256=53pymPDaM9-xXHFzCyEHdvM49JPTpsLI2aPTSP5zaPo,20090
@@ -340,7 +340,7 @@ dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBi
 dstack/_internal/server/services/pools.py,sha256=nkYsYb47lBnPswl3GleNvm30OIhRw4fPH7Z9PJpmhyo,25154
 dstack/_internal/server/services/projects.py,sha256=HWOnFOC6LmvbjDzRKADv7tXKIDWthbO11zrsHU7vkew,14709
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
-dstack/_internal/server/services/runs.py,sha256=PzDBJGI_RJ_KKNgZ0SKE0R3XfY9Oj1nZrJ8_qfj7b4I,40530
+dstack/_internal/server/services/runs.py,sha256=W8y7oOzLG_b2qsQar7UG0482NUhcWIGCnqFgrUmiGr4,40584
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
 dstack/_internal/server/services/users.py,sha256=L-exfxHdhj3TKX-gSjezHrYK6tnrt5qsQs-zZng1tUI,7123
 dstack/_internal/server/services/volumes.py,sha256=GMHUlSjeiIc9LSWKes-ZPRKgyvWD6y1VT8w6mH-URtg,12635
@@ -372,21 +372,21 @@ dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBW
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
 dstack/_internal/server/services/jobs/__init__.py,sha256=cgY4Tp66zIxIxsW_oj8_JUpEnUXuTutwhewzx2off2Y,19593
 dstack/_internal/server/services/jobs/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/jobs/configurators/base.py,sha256=gfcUY5dpJu3LfRsD683TNvfBgb8VrDSEfJStCSZsPdM,8347
-dstack/_internal/server/services/jobs/configurators/dev.py,sha256=V9e1M2Fyb4j4PzjASrkORtyPJN6Gh-YSClQBW2ZJnjA,1950
-dstack/_internal/server/services/jobs/configurators/service.py,sha256=2LnaFWq0vLieX6amjTPh2Upu4GKvKlz_THUrb8Nlo9k,862
-dstack/_internal/server/services/jobs/configurators/task.py,sha256=zq_J3Pksn2fBWgvexOu7EJln480zVTwINVpRo9LgMwM,1435
+dstack/_internal/server/services/jobs/configurators/base.py,sha256=uqT5bQWjf8IcYrD85_-WYEYoFyMYYn-FAc-sqKE-zQA,8683
+dstack/_internal/server/services/jobs/configurators/dev.py,sha256=2EEtfOpNXP5d9b3ZErE3dA3Co8aAAv1fbx9wAjRZpGY,2018
+dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YFSMuGqjOwYTLMV4FuIM5lCMDFjS0l0CoLI,929
+dstack/_internal/server/services/jobs/configurators/task.py,sha256=ojCgdMyGak8u00tdu-th6U6w0pGRENV1mPtajwzeA_s,1502
 dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
 dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
 dstack/_internal/server/services/proxy/__init__.py,sha256=aklmvGaGXISztQft-nH8R98WRU6x_L0Fx3RwVDhwt3g,85
 dstack/_internal/server/services/proxy/auth.py,sha256=AVhDCnk9KvxJ7Jsd8Xmotl_29V4lNuBw8mQNBhx90Ws,485
 dstack/_internal/server/services/proxy/deps.py,sha256=u4dHHKBlTgC8Fnnl3onErgE1HGKFwyZTwvq0uT6IdQ4,846
-dstack/_internal/server/services/proxy/repo.py,sha256=MRJ7n3PM1lgGGZeQc1-Va5eWiGCafSMgPm3FLpBlE5I,6326
+dstack/_internal/server/services/proxy/repo.py,sha256=JaEgvTsWREoI4aK16qdiRWWlJEV8JdyRnjRK90uylqM,6388
 dstack/_internal/server/services/proxy/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/proxy/routers/service_proxy.py,sha256=8rsuenv9LUlFJghV8Ly4vewGmc4pDdVTyuU0xiNLuVQ,1581
 dstack/_internal/server/services/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=uhu6GoUR6bqSBC-uBjn1gc8j-cDznTOAylxJU8MKucA,4701
+dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=4JrSxHqhBYqU1oENii89Db-bzkFWExYrOy-0mNEhWBs,4879
 dstack/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/runner/client.py,sha256=G8LTHOfZTqGpkSiv1ceJyckN-o4Xk3Zf4pDt_Da3acI,14805
 dstack/_internal/server/services/runner/ssh.py,sha256=RlfnCS3d6rx06qYUXXbCY7aAjRdxe6Q6A5VbaRwzEbk,3933
@@ -524,9 +524,9 @@ dstack/api/_public/backends.py,sha256=IBeotauJzlte379LzRM6ByZCl_98QemGZsm3p_WX98
 dstack/api/_public/pools.py,sha256=wlTai0px6aQNeKyL-6nOil6DCj-GdozqqDGmnTg4ubw,1033
 dstack/api/_public/repos.py,sha256=LYWy1W3Z-2y_D82Jln6YzU2F_asPULGJq1GM9HAAMIc,6173
 dstack/api/_public/resources.py,sha256=lTQM2nb7_QB2RCwmpQlBkYkpiN_aWjgF78wI7Op_RlQ,2947
-dstack/api/_public/runs.py,sha256=qPGu95Yd4cy6MqkThXURpxHMT_mhK1d6H0WfGg_2-44,24292
+dstack/api/_public/runs.py,sha256=3527ziltD9UNTDPMkaHHUXXQog2VLFXKQ0zgmRsnCb4,24718
 dstack/api/huggingface/__init__.py,sha256=oIrEij3wttLZ1yrywEGvCMd6zswMQrX5pPjrqdSi0UA,2201
-dstack/api/server/__init__.py,sha256=l1SWc9szAYtcBhXugkReq3NZQUl04XIrIHtASBEDJE0,5142
+dstack/api/server/__init__.py,sha256=1Sw1bB8NItIFWlL1bdeUN1Tuwu3uaYOJWFrg1c4VULY,5801
 dstack/api/server/_backends.py,sha256=mMePYgFMHAUma3Gycx54VsaxAL_IFhBRcI4H4eWWMg0,1908
 dstack/api/server/_fleets.py,sha256=aB8eJtiQDKjhBxVkkYhQCaiF0Q5hWeJOXuTFs7SwnBg,3870
 dstack/api/server/_gateways.py,sha256=_OcFOOxUdh4YIA15ZNQQT6mfj0cSKP2jJ_bQXxugS_w,2568
@@ -536,7 +536,7 @@ dstack/api/server/_metrics.py,sha256=OPb8sLhI_U605sHOPrELgy0_6cNFLJVfpvr-qkEukRM
 dstack/api/server/_pools.py,sha256=PSs8R4FAKZk-jtD4GmATceMyipnasOy2XEg-8lmiN28,2715
 dstack/api/server/_projects.py,sha256=g6kNSU6jer8u7Kaut1I0Ft4wRMLBBCQShJf3fOB63hQ,1440
 dstack/api/server/_repos.py,sha256=gf9fAQw3rzqfPGq-uT8I2Ju_Zn6G4aWTKis8gb8Polc,1885
-dstack/api/server/_runs.py,sha256=YF9ezqBKC4mBR8Nf79ypHx8yT-ur4GehoQ68EZuEpwQ,6100
+dstack/api/server/_runs.py,sha256=C8vE8uLRjiftRo_wKrr2wpcd_416qy2CsgUNXS3u_yY,6946
 dstack/api/server/_secrets.py,sha256=VqLfrIcmBJtPxNDRkXTG44H5SWoY788YJapScUukvdY,1576
 dstack/api/server/_users.py,sha256=XzhgGKc5Tsr0-xkz3T6rGyWZ1tO7aYNhLux2eE7dAoY,1738
 dstack/api/server/_volumes.py,sha256=E5VlZY44DRkaRUT0swrftYDoEgicDoVIBzzpYkiJhkw,1356
@@ -578,7 +578,7 @@ tests/_internal/core/models/test_volumes.py,sha256=V47bMK5UI1-1wrTl_rq-WClAF_OjC
 tests/_internal/core/models/repos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/core/models/repos/test_remote.py,sha256=6PQcuk9pVfQFG97Vpj_blkK2Veq8tvP6oRg4MUC9mX4,3041
 tests/_internal/core/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/core/services/test_logs.py,sha256=uM4en0RtLbVm_OOVgYU1nsE2_tY03zGzv4civY9TNZE,5290
+tests/_internal/core/services/test_logs.py,sha256=EI4qmFTPWEP-SXYMcXddSlrAqz03PTdx6VRvYl3Jinw,5732
 tests/_internal/core/services/ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/core/services/ssh/test_client.py,sha256=xR8O8DiCtPVRxBILiQLS921XjdxzE3CMXHAQt635_HU,2537
 tests/_internal/core/services/ssh/test_tunnel.py,sha256=icgac2ni8SMgh49uwtPll_vxJu0lLN_py1oHXAo4yqU,7317
@@ -610,7 +610,7 @@ tests/_internal/server/background/tasks/test_process_metrics.py,sha256=k6L0biz7E
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=shHqZvS8QoNwQe8J29-aHk2X2HqX-gsxcQiZevU8yuY,1528
 tests/_internal/server/background/tasks/test_process_running_jobs.py,sha256=EzdAYVUowb9cGb_bdvKG11jTHMuL9D48brXVD4V6GFY,18812
 tests/_internal/server/background/tasks/test_process_runs.py,sha256=Ks3K1O-BMyYSbMr-1L76IEzZ9aLQaVjxF6go-Nj-HxY,13668
-tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=sWXMhti6ws4CBXHOhHypt6IbslNYkO9E0Ai689u3jNE,19767
+tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=K4vdsvTWQfcRWVVAN_yRA8FBqkntIvwnZAjZCObWPsk,22633
 tests/_internal/server/background/tasks/test_process_submitted_volumes.py,sha256=njfRDdDzVtDL8rech_004Wf5z5PpoQVWFufKh1yUyYY,2171
 tests/_internal/server/background/tasks/test_process_terminating_jobs.py,sha256=hatdc4ANgQvJiVLfR9jE3URQjzqFD9FI6ZG9ES_vJOo,9541
 tests/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -623,7 +623,7 @@ tests/_internal/server/routers/test_metrics.py,sha256=NSdb_x_1Nwsqgrf6KXBf257qd2
 tests/_internal/server/routers/test_pools.py,sha256=pqPfIengCzI74D8ojYZbswEtGYhBXFtAlCvzg2RQr6c,25027
 tests/_internal/server/routers/test_projects.py,sha256=Pa_YL9lYhYt06TFmuZFEf9pxc-K3fM9L29ULf4gCELw,23950
 tests/_internal/server/routers/test_repos.py,sha256=4O4mCDlAPh8xJU_XWtwLhFWRkoTxXvZC1N2DLPyz2AI,17225
-tests/_internal/server/routers/test_runs.py,sha256=6BDz8MJLkeOxIiGmhZ0bGsNAqzPfeaFsTIp3RISrOTY,70375
+tests/_internal/server/routers/test_runs.py,sha256=tjY_Q3HVpKky45lCq2TeT_hEhsCqXyCHZnofXy-NYPM,70541
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
 tests/_internal/server/routers/test_users.py,sha256=5QSLvfn9SroGsZoBGmSTEaaqJcdEO8EUUy_YuvLL8ss,12787
 tests/_internal/server/routers/test_volumes.py,sha256=4nv7ba9VkjbYFyYyQmO0coyav_Z9x5t_ayoiyKmvXVw,15947
@@ -641,9 +641,9 @@ tests/_internal/server/services/encryption/keys/__init__.py,sha256=47DEQpj8HBSa-
 tests/_internal/server/services/encryption/keys/test_aes.py,sha256=BtDb5ZeXKKNkAg7KOLXKjpQivMIWpmZe5zaJg-JHFo0,601
 tests/_internal/server/services/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/services/proxy/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/server/services/proxy/routers/test_service_proxy.py,sha256=L-5eubZeixAWsN0qu0v82Y10jUIEvXOmot4GibEcSj8,8263
+tests/_internal/server/services/proxy/routers/test_service_proxy.py,sha256=cGyBwfSgT268CRhkgpTEIM3O1U9z8tSNTYjHQrn3e0E,9843
 tests/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/server/services/runner/test_client.py,sha256=9Z18Cd5haCBRfNafA1-HMl9Mus0yeNmougHJVM_XeUY,16114
+tests/_internal/server/services/runner/test_client.py,sha256=LRJzmyMOGtURc-S4uICfhHjO2K0fxvCa--yXflnoTDM,16212
 tests/_internal/server/services/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/services/services/test_autoscalers.py,sha256=YMi4W5gKFOpEdutc2Fli1L1DIUxCJLHE7ji-CJoiVac,3986
 tests/_internal/server/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -658,9 +658,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.18.39.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.18.39.dist-info/METADATA,sha256=CYXwEOvMQeQdMnOMUUWcVf-8g2zMRwMnqzY1SFee_ic,17813
-dstack-0.18.39.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.18.39.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.18.39.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.18.39.dist-info/RECORD,,
+dstack-0.18.40.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.18.40.dist-info/METADATA,sha256=yWuhZjhn3TFZLBfOUVx5ZANLFkIDGbPWufBypQko9Kk,17690
+dstack-0.18.40.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.18.40.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.18.40.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.18.40.dist-info/RECORD,,

tests/_internal/core/services/test_logs.py

@@ -1,3 +1,5 @@
+import pytest
+
 from dstack._internal.core.models.runs import AppSpec
 from dstack._internal.core.services.logs import URLReplacer
 
@@ -126,7 +128,18 @@ class TestServiceURLReplacer:
         )
         assert replacer(b"http://0.0.0.0:8000/qwerty") == b"https://secure.host.com/qwerty"
 
-    def test_in_server_proxy(self):
+    @pytest.mark.parametrize(
+        ("in_path", "out_path"),
+        [
+            ("", "/proxy/services/main/service/"),
+            ("/", "/proxy/services/main/service/"),
+            ("/a/b/c", "/proxy/services/main/service/a/b/c"),
+            ("/proxy/services/main/service", "/proxy/services/main/service"),
+            ("/proxy/services/main/service/", "/proxy/services/main/service/"),
+            ("/proxy/services/main/service/a/b/c", "/proxy/services/main/service/a/b/c"),
+        ],
+    )
+    def test_adds_prefix_unless_already_present(self, in_path: str, out_path: str) -> None:
         replacer = URLReplacer(
             ports={8888: 3000},
             app_specs=[],
@@ -135,9 +148,6 @@ class TestServiceURLReplacer:
             path_prefix="/proxy/services/main/service/",
         )
         assert (
-            replacer(b"http://0.0.0.0:8888") == b"http://0.0.0.0:3000/proxy/services/main/service/"
-        )
-        assert (
-            replacer(b"http://0.0.0.0:8888/qwerty")
-            == b"http://0.0.0.0:3000/proxy/services/main/service/qwerty"
+            replacer(f"http://0.0.0.0:8888{in_path}".encode())
+            == f"http://0.0.0.0:3000{out_path}".encode()
         )

tests/_internal/server/background/tasks/test_process_submitted_jobs.py

@@ -302,6 +302,78 @@ class TestProcessSubmittedJobs:
         await session.refresh(pool)
         assert not pool.instances
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+    async def test_provisions_job_with_optional_instance_volume_not_attached(
+        self,
+        test_db,
+        session: AsyncSession,
+    ):
+        project = await create_project(session=session)
+        user = await create_user(session=session)
+        pool = await create_pool(session=session, project=project)
+        repo = await create_repo(
+            session=session,
+            project_id=project.id,
+        )
+        run_spec = get_run_spec(run_name="test-run", repo_id=repo.name)
+        run_spec.configuration.volumes = [
+            InstanceMountPoint(instance_path="/root/.cache", path="/cache", optional=True)
+        ]
+        run = await create_run(
+            session=session,
+            project=project,
+            repo=repo,
+            user=user,
+            run_name="test-run",
+            run_spec=run_spec,
+        )
+        job = await create_job(
+            session=session,
+            run=run,
+            instance_assigned=True,
+        )
+        offer = InstanceOfferWithAvailability(
+            backend=BackendType.RUNPOD,
+            instance=InstanceType(
+                name="instance",
+                resources=Resources(cpus=1, memory_mib=512, spot=False, gpus=[]),
+            ),
+            region="us",
+            price=1.0,
+            availability=InstanceAvailability.AVAILABLE,
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            backend_mock = Mock()
+            m.return_value = [backend_mock]
+            backend_mock.TYPE = BackendType.RUNPOD
+            backend_mock.compute.return_value.get_offers_cached.return_value = [offer]
+            backend_mock.compute.return_value.run_job.return_value = JobProvisioningData(
+                backend=offer.backend,
+                instance_type=offer.instance,
+                instance_id="instance_id",
+                hostname="1.1.1.1",
+                internal_ip=None,
+                region=offer.region,
+                price=offer.price,
+                username="ubuntu",
+                ssh_port=22,
+                ssh_proxy=None,
+                dockerized=False,
+                backend_data=None,
+            )
+            await process_submitted_jobs()
+
+        await session.refresh(job)
+        assert job is not None
+        assert job.status == JobStatus.PROVISIONING
+
+        await session.refresh(pool)
+        instance_offer = InstanceOfferWithAvailability.parse_raw(pool.instances[0].offer)
+        assert offer == instance_offer
+        pool_job_provisioning_data = pool.instances[0].job_provisioning_data
+        assert pool_job_provisioning_data == job.job_provisioning_data
+
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_fails_job_when_no_capacity(self, test_db, session: AsyncSession):
@@ -412,7 +484,8 @@ class TestProcessSubmittedJobs:
         run_spec = get_run_spec(run_name="test-run", repo_id=repo.name)
         run_spec.configuration.volumes = [
             VolumeMountPoint(name=volume.name, path="/volume"),
-            InstanceMountPoint(instance_path="/root/.cache", path="/cache"),
+            InstanceMountPoint(instance_path="/root/.data", path="/data"),
+            InstanceMountPoint(instance_path="/root/.cache", path="/cache", optional=True),
         ]
         run = await create_run(
             session=session,

tests/_internal/server/routers/test_runs.py

@@ -110,6 +110,7 @@ def get_dev_env_run_plan_dict(
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
+                "single_branch": None,
                 "max_duration": "off",
                 "stop_duration": None,
                 "max_price": None,
@@ -180,6 +181,7 @@ def get_dev_env_run_plan_dict(
                     "replica_num": 0,
                     "job_num": 0,
                     "jobs_per_replica": 1,
+                    "single_branch": False,
                     "max_duration": None,
                     "stop_duration": 300,
                     "registry_auth": None,
@@ -261,6 +263,7 @@ def get_dev_env_run_dict(
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
+                "single_branch": None,
                 "max_duration": "off",
                 "stop_duration": None,
                 "max_price": None,
@@ -331,6 +334,7 @@ def get_dev_env_run_dict(
                     "replica_num": 0,
                     "job_num": 0,
                     "jobs_per_replica": 1,
+                    "single_branch": False,
                     "max_duration": None,
                     "stop_duration": 300,
                     "registry_auth": None,

tests/_internal/server/services/proxy/routers/test_service_proxy.py

@@ -4,6 +4,7 @@ from unittest.mock import patch
 import httpx
 import pytest
 from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
 
 from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
 from dstack._internal.proxy.lib.auth import BaseProxyAuthProvider
@@ -25,6 +26,8 @@ ProxyTestRepo = GatewayProxyRepo
 
 @pytest.fixture
 def mock_replica_client_httpbin(httpbin) -> Generator[None, None, None]:
+    """Mocks deployed services. Replaces them with httpbin"""
+
     with patch(
         "dstack._internal.proxy.lib.services.service_connection.ServiceConnectionPool.get_or_add"
     ) as add_connection_mock:
@@ -34,6 +37,20 @@ def mock_replica_client_httpbin(httpbin) -> Generator[None, None, None]:
         yield
 
 
+@pytest.fixture
+def mock_replica_client_path_reporter() -> Generator[None, None, None]:
+    """Mocks deployed services. Replaces them with an app that returns the requested path"""
+
+    app = FastAPI()
+    app.get("{path:path}")(lambda path: PlainTextResponse(path))
+    client = ServiceClient(base_url="http://test/", transport=httpx.ASGITransport(app))
+    with patch(
+        "dstack._internal.proxy.lib.services.service_connection.ServiceConnectionPool.get_or_add"
+    ) as add_connection_mock:
+        add_connection_mock.return_value.client.return_value = client
+        yield
+
+
 def make_app(
     repo: BaseProxyRepo, auth: BaseProxyAuthProvider = ProxyTestAuthProvider()
 ) -> FastAPI:
@@ -200,3 +217,25 @@ async def test_auth(mock_replica_client_httpbin, token: Optional[str], status: i
     url = "http://test-host/proxy/services/test-proj/httpbin/"
     resp = await client.get(url, headers=headers)
     assert resp.status_code == status
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("strip", "downstream_path", "upstream_path"),
+    [
+        (True, "/proxy/services/my-proj/my-run/", "/"),
+        (True, "/proxy/services/my-proj/my-run/a/b", "/a/b"),
+        (False, "/proxy/services/my-proj/my-run/", "/proxy/services/my-proj/my-run/"),
+        (False, "/proxy/services/my-proj/my-run/a/b", "/proxy/services/my-proj/my-run/a/b"),
+    ],
+)
+async def test_strip_prefix(
+    mock_replica_client_path_reporter, strip: bool, downstream_path: str, upstream_path: str
+) -> None:
+    repo = ProxyTestRepo()
+    await repo.set_project(make_project("my-proj"))
+    await repo.set_service(make_service("my-proj", "my-run", strip_prefix=strip))
+    _, client = make_app_client(repo)
+    resp = await client.get(f"http://test-host{downstream_path}")
+    assert resp.status_code == 200
+    assert resp.text == upstream_path

tests/_internal/server/services/runner/test_client.py

@@ -175,7 +175,9 @@ class TestShimClientV1(BaseShimClientTest):
                     "device_name": "/dev/sdv",
                 }
             ],
-            "instance_mounts": [{"instance_path": "/mnt/nfs/home", "path": "/home"}],
+            "instance_mounts": [
+                {"instance_path": "/mnt/nfs/home", "path": "/home", "optional": False}
+            ],
         }
         self.assert_request(adapter, 0, "POST", "/api/submit", expected_request)
 
@@ -341,7 +343,9 @@ class TestShimClientV2(BaseShimClientTest):
                 }
             ],
             "volume_mounts": [{"name": "vol", "path": "/vol"}],
-            "instance_mounts": [{"instance_path": "/mnt/nfs/home", "path": "/home"}],
+            "instance_mounts": [
+                {"instance_path": "/mnt/nfs/home", "path": "/home", "optional": False}
+            ],
             "host_ssh_user": "dstack",
             "host_ssh_keys": ["host_key"],
             "container_ssh_keys": ["project_key", "user_key"],