dstack 0.18.38__py3-none-any.whl → 0.18.40rc1__py3-none-any.whl

dstack/api/_public/runs.py

@@ -324,11 +324,19 @@ class Run(ABC):
             if runtime_data is not None and runtime_data.ports is not None:
                 container_ssh_port = runtime_data.ports.get(container_ssh_port, container_ssh_port)
 
+            # TODO: get login name from runner in case it's not specified in the run configuration
+            # (i.e. the default image user is used, and it is not root)
+            if job.job_spec.user is not None and job.job_spec.user.username is not None:
+                container_user = job.job_spec.user.username
+            else:
+                container_user = "root"
+
             self._ssh_attach = SSHAttach(
                 hostname=provisioning_data.hostname,
                 ssh_port=provisioning_data.ssh_port,
                 container_ssh_port=container_ssh_port,
                 user=provisioning_data.username,
+                container_user=container_user,
                 id_rsa_path=ssh_identity_file,
                 ports_lock=self._ports_lock,
                 run_name=name,

dstack/api/server/__init__.py

@@ -133,9 +133,16 @@ class APIClient:
         else:
             raise ClientError(f"Failed to connect to dstack server {self._base_url}")
 
+        if 400 <= resp.status_code < 600:
+            logger.debug(
+                "Error requesting %s. Status: %s. Headers: %s. Body: %s",
+                resp.request.url,
+                resp.status_code,
+                resp.headers,
+                resp.content,
+            )
+
         if raise_for_status:
-            if resp.status_code == 500:
-                raise ClientError("Unexpected dstack server error")
             if resp.status_code == 400:  # raise ServerClientError
                 detail: List[Dict] = resp.json()["detail"]
                 if len(detail) == 1 and detail[0]["code"] in _server_client_errors:
@@ -145,7 +152,16 @@ class APIClient:
             if resp.status_code == 422:
                 formatted_error = pprint.pformat(resp.json())
                 raise ClientError(f"Server validation error: \n{formatted_error}")
-            resp.raise_for_status()
+            if resp.status_code == 403:
+                raise ClientError(
+                    f"Access to {resp.request.url} is denied. Please check your access token"
+                )
+            if 400 <= resp.status_code < 600:
+                raise ClientError(
+                    f"Unexpected error: status code {resp.status_code}"
+                    f" when requesting {resp.request.url}."
+                    " Check server logs or run with DSTACK_CLI_LOG_LEVEL=DEBUG to see more details"
+                )
         return resp
 
 

dstack/api/server/_fleets.py

@@ -73,12 +73,17 @@ def _get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[_ExcludeDict]:
         ):
             configuration_excludes["ssh_config"] = {"hosts": {"__all__": {"internal_ip"}}}
     # client >= 0.18.30 / server <= 0.18.29 compatibility tweak
-    if not fleet_spec.configuration.reservation:
+    if fleet_spec.configuration.reservation is None:
         configuration_excludes["reservation"] = True
+    if fleet_spec.profile is not None and fleet_spec.profile.reservation is None:
         profile_excludes.add("reservation")
     if fleet_spec.configuration.idle_duration is None:
         configuration_excludes["idle_duration"] = True
+    if fleet_spec.profile is not None and fleet_spec.profile.idle_duration is None:
         profile_excludes.add("idle_duration")
+    # client >= 0.18.38 / server <= 0.18.37 compatibility tweak
+    if fleet_spec.profile is not None and fleet_spec.profile.stop_duration is None:
+        profile_excludes.add("stop_duration")
 
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes

dstack/api/server/_runs.py

@@ -1,9 +1,14 @@
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 from uuid import UUID
 
 from pydantic import parse_obj_as
 
+from dstack._internal.core.models.common import is_core_model_instance
+from dstack._internal.core.models.configurations import (
+    STRIP_PREFIX_DEFAULT,
+    ServiceConfiguration,
+)
 from dstack._internal.core.models.pools import Instance
 from dstack._internal.core.models.profiles import Profile
 from dstack._internal.core.models.runs import (
@@ -14,6 +19,7 @@ from dstack._internal.core.models.runs import (
     RunPlan,
     RunSpec,
 )
+from dstack._internal.core.models.volumes import InstanceMountPoint
 from dstack._internal.server.schemas.runs import (
     ApplyRunPlanRequest,
     CreateInstanceRequest,
@@ -117,34 +123,48 @@ class RunsAPIClient(APIClientGroup):
 
 def _get_run_spec_excludes(run_spec: RunSpec) -> Optional[dict]:
     spec_excludes: dict[str, set[str]] = {}
-    configuration_excludes: set[str] = set()
+    configuration_excludes: dict[str, Any] = {}
     profile_excludes: set[str] = set()
     configuration = run_spec.configuration
     profile = run_spec.profile
 
     # client >= 0.18.18 / server <= 0.18.17 compatibility tweak
     if not configuration.privileged:
-        configuration_excludes.add("privileged")
+        configuration_excludes["privileged"] = True
     # client >= 0.18.23 / server <= 0.18.22 compatibility tweak
     if configuration.type == "service" and configuration.gateway is None:
-        configuration_excludes.add("gateway")
+        configuration_excludes["gateway"] = True
     # client >= 0.18.30 / server <= 0.18.29 compatibility tweak
     if run_spec.configuration.user is None:
-        configuration_excludes.add("user")
+        configuration_excludes["user"] = True
     # client >= 0.18.30 / server <= 0.18.29 compatibility tweak
     if configuration.reservation is None:
-        configuration_excludes.add("reservation")
+        configuration_excludes["reservation"] = True
     if profile is not None and profile.reservation is None:
         profile_excludes.add("reservation")
     if configuration.idle_duration is None:
-        configuration_excludes.add("idle_duration")
+        configuration_excludes["idle_duration"] = True
     if profile is not None and profile.idle_duration is None:
         profile_excludes.add("idle_duration")
     # client >= 0.18.38 / server <= 0.18.37 compatibility tweak
     if configuration.stop_duration is None:
-        configuration_excludes.add("stop_duration")
+        configuration_excludes["stop_duration"] = True
     if profile is not None and profile.stop_duration is None:
         profile_excludes.add("stop_duration")
+    # client >= 0.18.40 / server <= 0.18.39 compatibility tweak
+    if (
+        is_core_model_instance(configuration, ServiceConfiguration)
+        and configuration.strip_prefix == STRIP_PREFIX_DEFAULT
+    ):
+        configuration_excludes["strip_prefix"] = True
+    if configuration.single_branch is None:
+        configuration_excludes["single_branch"] = True
+    if all(
+        not is_core_model_instance(v, InstanceMountPoint) or not v.optional
+        for v in configuration.volumes
+    ):
+        configuration_excludes["volumes"] = {"__all__": {"optional"}}
+
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes
     if profile_excludes:

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.18.38"
+__version__ = "0.18.40rc1"
 __is_release__ = True
 base_image = "0.6"

{dstack-0.18.38.dist-info → dstack-0.18.40rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.18.38
+Version: 0.18.40rc1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov
@@ -303,17 +303,16 @@ for AI workloads both in the cloud and on-prem, speeding up the development, tra
 
 #### Accelerators
 
-`dstack` supports `NVIDIA GPU`, `AMD GPU`, and `Google Cloud TPU` out of the box.
+`dstack` supports `NVIDIA`, `AMD`, `Google TPU`, and `Intel Gaudi` accelerators out of the box.
 
 ## Major news ✨
 
-- [2025/01] [dstack 0.18.35: Vultr backend](https://github.com/dstackai/dstack/releases/tag/0.18.35)
-- [2024/12] [dstack 0.18.33: TPU v6e support](https://github.com/dstackai/dstack/releases/tag/0.18.33)
+- [2025/01] [dstack 0.18.38: Intel Gaudi](https://github.com/dstackai/dstack/releases/tag/0.18.38)
+- [2025/01] [dstack 0.18.35: Vultr](https://github.com/dstackai/dstack/releases/tag/0.18.35)
+- [2024/12] [dstack 0.18.32: TPU v6e](https://github.com/dstackai/dstack/releases/tag/0.18.32)
 - [2024/12] [dstack 0.18.30: AWS Capacity Reservations and Capacity Blocks](https://github.com/dstackai/dstack/releases/tag/0.18.30)
-- [2024/11] [dstack 0.18.23: Gateway is optional](https://github.com/dstackai/dstack/releases/tag/0.18.23)
 - [2024/10] [dstack 0.18.21: Instance volumes](https://github.com/dstackai/dstack/releases/tag/0.18.21)
-- [2024/10] [dstack 0.18.18: Hardware metrics](https://github.com/dstackai/dstack/releases/tag/0.18.18)
-- [2024/10] [dstack 0.18.17: AMD support with SSH fleets, AWS EFA](https://github.com/dstackai/dstack/releases/tag/0.18.17)
+- [2024/10] [dstack 0.18.18: Hardware metrics monitoring](https://github.com/dstackai/dstack/releases/tag/0.18.18)
 
 ## Installation
 

{dstack-0.18.38.dist-info → dstack-0.18.40rc1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=OvARIDo4OSI1Z2KsgU_k_BTxGxoEuwwebH_g2G8Xqjk,65
+dstack/version.py,sha256=C2zHMrldaVFflhC9dNqTI0mPTOgTGyqywHXOnApNTEE,68
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
@@ -27,12 +27,12 @@ dstack/_internal/cli/services/profile.py,sha256=Fl052TeMCbWO1Q6HiaPVD9CdfClzZAkk
 dstack/_internal/cli/services/repos.py,sha256=n2INRJtTYV5y60-Yp3F-V5kJkqwZI_fdGXPnxvZhkNw,3321
 dstack/_internal/cli/services/configurators/__init__.py,sha256=z94VPBFqybP8Zpwy3CzYxmpPAqYBOvRRLpXoz2H4GKI,2697
 dstack/_internal/cli/services/configurators/base.py,sha256=YsyNmHTv1wVbvaAjQi_qNQAb4oK9T_Wgu6RKg-uyzvY,3379
-dstack/_internal/cli/services/configurators/fleet.py,sha256=YbZh6x1UuvECh7ZMmxvQQfGzBUi0Kxn3stsVutKWLKo,14259
-dstack/_internal/cli/services/configurators/gateway.py,sha256=0pwAPFYoANioeh6RJiq5IggMc4ywAVfEz3hX91FoPaY,8061
+dstack/_internal/cli/services/configurators/fleet.py,sha256=iqNJSoji7IiVb3Kb4KA6XTzI5k2xUYF7DK9aE7iverk,14281
+dstack/_internal/cli/services/configurators/gateway.py,sha256=BlLRBC-rl9a0xpO3c1N4k5yinvVQB5YU238g-8NMINY,8389
 dstack/_internal/cli/services/configurators/run.py,sha256=GLGhCcrOzp96Yc6RiISg-DZEM43WT7bhsZmD8z0oAVA,23271
 dstack/_internal/cli/services/configurators/volume.py,sha256=UAKfx-J5fUdnnR2RBvbRJy0YKzXOoub4f4ywMsZwk38,7983
 dstack/_internal/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/cli/utils/common.py,sha256=xuHxRG6pl9Amt8PRbpu0wav6P5ac2wtLUMBj0URxRSc,1852
+dstack/_internal/cli/utils/common.py,sha256=rfmzqrsgR3rXW3wj0vxDdvrhUUg2aIy4A6E9MZbd55g,1763
 dstack/_internal/cli/utils/fleet.py,sha256=tumFG4qgrKjmb0Iik64tWcPrenaOjpeXC6AZSezQ3_A,2991
 dstack/_internal/cli/utils/gateway.py,sha256=jMytH6u3x8hctMhm9bcmXLJxSgTXmpW8M9abATQrw3c,1474
 dstack/_internal/cli/utils/rich.py,sha256=Gx1MJU929kMKsbdo9qF7XHARNta2426Ssb-xMLVhwbQ,5710
@@ -114,7 +114,7 @@ dstack/_internal/core/backends/vultr/config.py,sha256=TqjT2yhW9rAiDpJ96gBFTM638F
 dstack/_internal/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/models/common.py,sha256=XNfEP_lHAs398oDfSVn8kCX79GIWvl0hDs05Rz1CBic,2579
 dstack/_internal/core/models/config.py,sha256=JJ7rT7dztzTWCY5TkoyxXxTvG5D4IFYhGe7EzwkLOWQ,581
-dstack/_internal/core/models/configurations.py,sha256=0ZCQ7atvMnIKvpK1FFmw7IXox3WvAgyfqAtaJmWCLns,13030
+dstack/_internal/core/models/configurations.py,sha256=BrxbkQ5xhCw-pT7DsEdJeLa92uEWQM3LHPmSiYOohHE,13806
 dstack/_internal/core/models/envs.py,sha256=ls2ahIhEb3LYLX_auH3F46uTRp2zdlMWKEpjmU_V3S4,5017
 dstack/_internal/core/models/fleets.py,sha256=tSAFkBugchMhvLRk8c0loRpteTYnbA0sDhUI8q_EcXI,8969
 dstack/_internal/core/models/gateways.py,sha256=n9lPq6X0n2a7fF-CseiBVDDwasJsU3VQumTetMsfZcc,3465
@@ -126,13 +126,13 @@ dstack/_internal/core/models/pools.py,sha256=mZSZ_9CKTIhnoXfkQLEOjJg_EDZW7kXXigC
 dstack/_internal/core/models/profiles.py,sha256=CRvVf2bph56HOz7OmpCwreOR2yCHdfVgovyjKxYBsiM,8011
 dstack/_internal/core/models/projects.py,sha256=hY3rhLWkuDRsavAdNvQaK6IhnTr7yBHTmNwy9k7zjVE,643
 dstack/_internal/core/models/resources.py,sha256=AzBqGEZLHmg-HpjQtcb_W4pJQRxRtDyeuiHO4berdBA,12408
-dstack/_internal/core/models/runs.py,sha256=fHpvKW_4LfhhtbzvTHwVXRye6gzQikclmEudKPGQEHk,16930
+dstack/_internal/core/models/runs.py,sha256=CqyQ_wMne7G9XHLm2BRH84J0tgG3Ny77No1A8hTCmgQ,16971
 dstack/_internal/core/models/secrets.py,sha256=OlnBI8ESBnpjSqB0-Vr3z8JcqB2Ydfiwo8IJUuM5jAc,219
 dstack/_internal/core/models/server.py,sha256=Hkc1v2s3KOiwslsWVmhUOAzcSeREoG-HD1SzSX9WUGg,152
 dstack/_internal/core/models/services.py,sha256=2Hpi7j0Q1shaf_0wd0C0044AJAmuYi-D3qx3PH849oI,3076
 dstack/_internal/core/models/unix.py,sha256=KxnSQELnkAjjuUgYcQKVkf-UAbYREBD8WCWDvHfOkuA,1915
 dstack/_internal/core/models/users.py,sha256=o_rd0GAmd6jufypVUs9P12NRri3rgAPDt-KxnqNNsGw,703
-dstack/_internal/core/models/volumes.py,sha256=piArLbgpUyejRJW2vtluqmqXhLmyj9rhzsCfPPGAZ50,4975
+dstack/_internal/core/models/volumes.py,sha256=MagqSb9jB2WnQlP-xVYkkPYHtKh9r54ajVEUYNF9iKc,5215
 dstack/_internal/core/models/backends/__init__.py,sha256=iPMAv4j7gpHc0cjt3SxzQGb-sywms8LUXt7IjtKNPnM,5589
 dstack/_internal/core/models/backends/aws.py,sha256=H0RB7pTVb7vjW_-MDpBBwPLbAoKDRtYFHe_n4ppXz6w,2463
 dstack/_internal/core/models/backends/azure.py,sha256=u5R8vLVS85X27P17IZ1L-EcXmxg08sC0E1EFxkVG2YA,1996
@@ -157,12 +157,12 @@ dstack/_internal/core/models/repos/virtual.py,sha256=1p6k2vBfWI6UT91os3lZfV5wl8b
 dstack/_internal/core/services/__init__.py,sha256=Utrh2zbYsP6IBGWMGROaHUefgo_TvLloN3logcdhuC4,282
 dstack/_internal/core/services/api_client.py,sha256=HTQ0fcZciUh-nmfV09hUt__Z4N2zq_R12xAiYXD0Jy0,650
 dstack/_internal/core/services/diff.py,sha256=lw7vTOYnGtRWlEulgVlHt1KjLdC8VJpfBcqMRuFZzSQ,532
-dstack/_internal/core/services/logs.py,sha256=63s5OKyrZEs-0iLBKRFSn5VldelyWY75gD4RHj_Ejlo,2053
+dstack/_internal/core/services/logs.py,sha256=7_eJdH4MD-3rVb4A6rIJfjj_p4jzUOCmjRVlPD-UDsg,2166
 dstack/_internal/core/services/profiles.py,sha256=SJGT6LG8JmrfsqlyXOkPZd61BDwZ4Q1ngcmDVgq826M,2160
 dstack/_internal/core/services/repos.py,sha256=YgTmLX8U1Y5cz2dG9yThK1-Geq02iik_C6DGZ0y6uEY,5311
 dstack/_internal/core/services/configs/__init__.py,sha256=2jwu1w0vEDNcKGdeI15SS4EtpLSZzRpG8xZslHgTJmY,5484
 dstack/_internal/core/services/ssh/__init__.py,sha256=UhgC3Lv3CPSGqSPEQZIKOfLKUlCFnaB0uqPQhfKCFt0,878
-dstack/_internal/core/services/ssh/attach.py,sha256=4KNiHnM5vsJ8SNWcKGRqhZSKygxDoGLfMBkIAICLHEE,7488
+dstack/_internal/core/services/ssh/attach.py,sha256=A-A8H9lwdIFrxk6FOtHaCZ02fNXo73yz4c-QN1_BQ98,7489
 dstack/_internal/core/services/ssh/client.py,sha256=FUg74jNOagYR3OKuWMBwNgJWIDwHfYrIMSER8xn92D8,4102
 dstack/_internal/core/services/ssh/ports.py,sha256=yWgcACqTNwfLMG24U28LYXlZuEb1V7cSiYBQBqVN5yU,2884
 dstack/_internal/core/services/ssh/tunnel.py,sha256=lHNK1hEcqMP_T_Y6cM_OLHVIRCEA4hud-sM5Jmp4svM,10296
@@ -201,7 +201,7 @@ dstack/_internal/proxy/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 dstack/_internal/proxy/lib/auth.py,sha256=bI85UFTupPxlcAJZqpOuqS64I4MeIyzp_gHxM3916V4,183
 dstack/_internal/proxy/lib/deps.py,sha256=fFq1mJQK_XcqCNgORTowhc0jCblRiGIK1TeRDC2wkL8,3538
 dstack/_internal/proxy/lib/errors.py,sha256=sUO1dDMkr0oKghB0yUdoHLhB66tX--FkHWrJ5jzPB4w,437
-dstack/_internal/proxy/lib/models.py,sha256=MC8ATwPdgXi6l2SCQjuR8WFp-yuuhCn0OD8DU9QXZsg,2625
+dstack/_internal/proxy/lib/models.py,sha256=duixvPYU6lnusp3tYjxU675Zo3idXfOiAPQ1ZRnv1h8,2678
 dstack/_internal/proxy/lib/repo.py,sha256=zkWZ9XZzQHfCa-eifec7H7UYnJZLgeRuiQls7Rc2pkA,781
 dstack/_internal/proxy/lib/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/routers/model_proxy.py,sha256=57GFRpVRXcVY-347HnUSUr4w4RsxsjLuuZiJs8DwDpM,3895
@@ -217,9 +217,9 @@ dstack/_internal/proxy/lib/services/model_proxy/clients/openai.py,sha256=K8cMgGQ
 dstack/_internal/proxy/lib/services/model_proxy/clients/tgi.py,sha256=7TsqrlUVo2PYdEIbmmGNTz0ktf_T8l4Zpw4w4HYAYTs,7845
 dstack/_internal/proxy/lib/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/testing/auth.py,sha256=-firWTnis9Eogoi58BURv1S-te4Hf9x1Q_aYLZMDll4,465
-dstack/_internal/proxy/lib/testing/common.py,sha256=peH9vQF0g4wsJCcF_FasKOXmSlCFVSEuwOBpA0CHANc,1452
+dstack/_internal/proxy/lib/testing/common.py,sha256=5e2VYboMqjBnUxkvidaWLoQ-uaBGh_hnURb_VJc38q4,1518
 dstack/_internal/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/app.py,sha256=Bz875CPuYuewbqoo7slo9b1UckLWz1TTvBJtu50o9qI,11214
+dstack/_internal/server/app.py,sha256=pPto7JaBEkwvPF2FR8sqDzwIyMovI_85xk6WXfnnCYU,11317
 dstack/_internal/server/db.py,sha256=WjuqmjG3QAZmSMCeUaJ_ynbowlHuNAvYCZO649cTPHc,3210
 dstack/_internal/server/deps.py,sha256=31e8SU_ogPJWHIDLkgl7cuC_5V91xbJoLyAj17VanfM,670
 dstack/_internal/server/main.py,sha256=kztKhCYNoHSDyJJQScWfZXE0naNleJOCQULW6dd8SGw,109
@@ -234,7 +234,7 @@ dstack/_internal/server/background/tasks/process_metrics.py,sha256=99Uu41B-XS7QZ
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=DZctiMF2TXKbYeygoz--ZiC9MXVL1sQJgq0oqRIc7hg,3858
 dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=9vRGefwyjBagmOyot5tjBi6tXP9gwycdrLGqs_-Iu14,27634
 dstack/_internal/server/background/tasks/process_runs.py,sha256=LPe0RvDmNGSnIT2RNEYh5TceFx69z_JPYxwpU8mP9kQ,17254
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=F-w8njVMXF_fmKZayzptChCn-Unn4dTq7WDMAZ0sFv4,24354
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=CkLdkt3D59RK3UkzLqvHkXZty2Zpf-Ksk3mmVQskHgg,24364
 dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=Y_ISp_lY0S0H-Qn7s5t-jVBYiBPoS7080LM5HbP1pRY,3745
 dstack/_internal/server/background/tasks/process_volumes.py,sha256=UoMtYjX6MbSoMXOOuBhA0FVIlQMT_wZWsRNeYVV3cSI,4703
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -319,7 +319,7 @@ dstack/_internal/server/schemas/logs.py,sha256=JGt39fBEFRjHhlGT1jIC6kwQhujxPO8ue
 dstack/_internal/server/schemas/pools.py,sha256=gPUUXLVwwud_nyOc5cXfSqz-6ladlnBb4Uzu4SJgCg4,734
 dstack/_internal/server/schemas/projects.py,sha256=-YIaAYxMgOMKtw-5xPTmmBBwa_2tjJ3jqVPYEnOeTF8,437
 dstack/_internal/server/schemas/repos.py,sha256=dRmT4CEs1lXN_kB8ZKtL1aZGLuBy-zGfPxLXTn4FwBY,2027
-dstack/_internal/server/schemas/runner.py,sha256=Nq4vQHsOiry2iIET2KvBYVCJkQlalW6QqjqxWNXW2Vw,4159
+dstack/_internal/server/schemas/runner.py,sha256=Avqfu4iorAta_0BTWXfk_ydz4V0Eu0_6ayo3DHzH8HQ,4192
 dstack/_internal/server/schemas/runs.py,sha256=Tp-_q_XijeuH-CR14y5xM1-B4TfTDSzWouVCbsg3NsU,1774
 dstack/_internal/server/schemas/secrets.py,sha256=mfqLSM7PqxVQ-GIWB6RfPRUOvSvvaRv-JxXAYxZ6dyY,373
 dstack/_internal/server/schemas/users.py,sha256=FuDqwRVe3mOmv497vOZKjI0a_d4Wt2g4ZiCJcyfHEKA,495
@@ -329,7 +329,7 @@ dstack/_internal/server/security/permissions.py,sha256=K756I8fnBLBrq4PZwz1ttt74H
 dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/config.py,sha256=rXGihEDrfdDJtDsfeYlbHQW8eGej56VDYTV6BJ_aWjg,29241
 dstack/_internal/server/services/docker.py,sha256=3EcYPiVsrNBGDQYOb60QJ241mzTT6lJROYQXIwt-8dk,5351
-dstack/_internal/server/services/fleets.py,sha256=XmZWulyVLxjWRDhVVTqh9Z12TPdGZR6B8RBZdg0P__s,28038
+dstack/_internal/server/services/fleets.py,sha256=SOU6m-EVbRdpfTkT7I0UUyVXQJTF4_zpcSNWEIb1lQo,27998
 dstack/_internal/server/services/locking.py,sha256=UV5kc-BgROBuNDpBrp8jkcf-fHPOpHLc8JufL-8mbN8,2453
 dstack/_internal/server/services/logging.py,sha256=Nu1628kW2hqB__N0Eyr07wGWjVWxfyJnczonTJ72kSM,417
 dstack/_internal/server/services/logs.py,sha256=53pymPDaM9-xXHFzCyEHdvM49JPTpsLI2aPTSP5zaPo,20090
@@ -340,7 +340,7 @@ dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBi
 dstack/_internal/server/services/pools.py,sha256=nkYsYb47lBnPswl3GleNvm30OIhRw4fPH7Z9PJpmhyo,25154
 dstack/_internal/server/services/projects.py,sha256=HWOnFOC6LmvbjDzRKADv7tXKIDWthbO11zrsHU7vkew,14709
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
-dstack/_internal/server/services/runs.py,sha256=PzDBJGI_RJ_KKNgZ0SKE0R3XfY9Oj1nZrJ8_qfj7b4I,40530
+dstack/_internal/server/services/runs.py,sha256=W8y7oOzLG_b2qsQar7UG0482NUhcWIGCnqFgrUmiGr4,40584
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
 dstack/_internal/server/services/users.py,sha256=L-exfxHdhj3TKX-gSjezHrYK6tnrt5qsQs-zZng1tUI,7123
 dstack/_internal/server/services/volumes.py,sha256=GMHUlSjeiIc9LSWKes-ZPRKgyvWD6y1VT8w6mH-URtg,12635
@@ -372,21 +372,21 @@ dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBW
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
 dstack/_internal/server/services/jobs/__init__.py,sha256=cgY4Tp66zIxIxsW_oj8_JUpEnUXuTutwhewzx2off2Y,19593
 dstack/_internal/server/services/jobs/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/jobs/configurators/base.py,sha256=gfcUY5dpJu3LfRsD683TNvfBgb8VrDSEfJStCSZsPdM,8347
-dstack/_internal/server/services/jobs/configurators/dev.py,sha256=V9e1M2Fyb4j4PzjASrkORtyPJN6Gh-YSClQBW2ZJnjA,1950
-dstack/_internal/server/services/jobs/configurators/service.py,sha256=2LnaFWq0vLieX6amjTPh2Upu4GKvKlz_THUrb8Nlo9k,862
-dstack/_internal/server/services/jobs/configurators/task.py,sha256=zq_J3Pksn2fBWgvexOu7EJln480zVTwINVpRo9LgMwM,1435
+dstack/_internal/server/services/jobs/configurators/base.py,sha256=uqT5bQWjf8IcYrD85_-WYEYoFyMYYn-FAc-sqKE-zQA,8683
+dstack/_internal/server/services/jobs/configurators/dev.py,sha256=2EEtfOpNXP5d9b3ZErE3dA3Co8aAAv1fbx9wAjRZpGY,2018
+dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YFSMuGqjOwYTLMV4FuIM5lCMDFjS0l0CoLI,929
+dstack/_internal/server/services/jobs/configurators/task.py,sha256=ojCgdMyGak8u00tdu-th6U6w0pGRENV1mPtajwzeA_s,1502
 dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
 dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
 dstack/_internal/server/services/proxy/__init__.py,sha256=aklmvGaGXISztQft-nH8R98WRU6x_L0Fx3RwVDhwt3g,85
 dstack/_internal/server/services/proxy/auth.py,sha256=AVhDCnk9KvxJ7Jsd8Xmotl_29V4lNuBw8mQNBhx90Ws,485
 dstack/_internal/server/services/proxy/deps.py,sha256=u4dHHKBlTgC8Fnnl3onErgE1HGKFwyZTwvq0uT6IdQ4,846
-dstack/_internal/server/services/proxy/repo.py,sha256=MRJ7n3PM1lgGGZeQc1-Va5eWiGCafSMgPm3FLpBlE5I,6326
+dstack/_internal/server/services/proxy/repo.py,sha256=JaEgvTsWREoI4aK16qdiRWWlJEV8JdyRnjRK90uylqM,6388
 dstack/_internal/server/services/proxy/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/proxy/routers/service_proxy.py,sha256=8rsuenv9LUlFJghV8Ly4vewGmc4pDdVTyuU0xiNLuVQ,1581
 dstack/_internal/server/services/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=uhu6GoUR6bqSBC-uBjn1gc8j-cDznTOAylxJU8MKucA,4701
+dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=4JrSxHqhBYqU1oENii89Db-bzkFWExYrOy-0mNEhWBs,4879
 dstack/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/runner/client.py,sha256=G8LTHOfZTqGpkSiv1ceJyckN-o4Xk3Zf4pDt_Da3acI,14805
 dstack/_internal/server/services/runner/ssh.py,sha256=RlfnCS3d6rx06qYUXXbCY7aAjRdxe6Q6A5VbaRwzEbk,3933
@@ -418,9 +418,9 @@ dstack/_internal/server/statics/e467d7d60aae81ab198b.svg,sha256=_XHc9mfQZgGkcy4h
 dstack/_internal/server/statics/eb9b344b73818fe2b71a.png,sha256=2H14eOCQRyZhFGJ1Kn2LH1j70kTF1Qop4vH-tiKqyPI,85
 dstack/_internal/server/statics/f517dd626eb964120de0.png,sha256=4QQuNa8SqmcZ67HK6739OHCyjnAJseU1bkcn454KRQs,159
 dstack/_internal/server/statics/f958aecddee5d8e3222c.png,sha256=8CoZkVNgRfOAe62X1dU-AZDvwh_nESKaQblEmaX2Xrs,87
-dstack/_internal/server/statics/index.html,sha256=DeJDAA8wDI-XHzlMwikiKVJcUgwPrv6rJMcfV2hu79Q,10468
-dstack/_internal/server/statics/main-623e02815ab7a6b56019.js,sha256=S--x6OArTSgcLr54yvVoLs8l7_M4vpehrjHqb8-fjhA,6204312
-dstack/_internal/server/statics/main-623e02815ab7a6b56019.js.map,sha256=ln4mxOwqSdoMcZ97UnrZmJsjZR2gFaMFLMzh_K08X1M,8137728
+dstack/_internal/server/statics/index.html,sha256=mIwMaWADHVWQ1EJYM8C1ecX5Y1MR4wkKV998jb_yBsc,10468
+dstack/_internal/server/statics/main-11ec5e4a00ea6ec833e3.js,sha256=DmQ1fNhJdIj8GNLZTQjQlwAxUoL3mTpsOAZS5ggB8Is,6204633
+dstack/_internal/server/statics/main-11ec5e4a00ea6ec833e3.js.map,sha256=lmvijj1GItZUgF5A9DSDIdujoSYIOZo9eHx4ICUMuD0,8138945
 dstack/_internal/server/statics/main-fc56d1f4af8e57522a1c.css,sha256=qFBof-8aUjytItFl5d-7u1syFn6jI7Kq_3V4oJqTS98,1296001
 dstack/_internal/server/statics/manifest.json,sha256=430w2BoWVmYYVr14lDvUxx-ROPt3VjigzeMqfLeiSCM,340
 dstack/_internal/server/statics/robots.txt,sha256=kNJLw79pisHhc3OVAimMzKcq3x9WT6sF9IS4xI0crdI,67
@@ -524,11 +524,11 @@ dstack/api/_public/backends.py,sha256=IBeotauJzlte379LzRM6ByZCl_98QemGZsm3p_WX98
 dstack/api/_public/pools.py,sha256=wlTai0px6aQNeKyL-6nOil6DCj-GdozqqDGmnTg4ubw,1033
 dstack/api/_public/repos.py,sha256=LYWy1W3Z-2y_D82Jln6YzU2F_asPULGJq1GM9HAAMIc,6173
 dstack/api/_public/resources.py,sha256=lTQM2nb7_QB2RCwmpQlBkYkpiN_aWjgF78wI7Op_RlQ,2947
-dstack/api/_public/runs.py,sha256=qPGu95Yd4cy6MqkThXURpxHMT_mhK1d6H0WfGg_2-44,24292
+dstack/api/_public/runs.py,sha256=3527ziltD9UNTDPMkaHHUXXQog2VLFXKQ0zgmRsnCb4,24718
 dstack/api/huggingface/__init__.py,sha256=oIrEij3wttLZ1yrywEGvCMd6zswMQrX5pPjrqdSi0UA,2201
-dstack/api/server/__init__.py,sha256=l1SWc9szAYtcBhXugkReq3NZQUl04XIrIHtASBEDJE0,5142
+dstack/api/server/__init__.py,sha256=1Sw1bB8NItIFWlL1bdeUN1Tuwu3uaYOJWFrg1c4VULY,5801
 dstack/api/server/_backends.py,sha256=mMePYgFMHAUma3Gycx54VsaxAL_IFhBRcI4H4eWWMg0,1908
-dstack/api/server/_fleets.py,sha256=4XZF10r-pOBF052WwBOkLGASFeeOJ7yY8NB6VKxDc2M,3506
+dstack/api/server/_fleets.py,sha256=aB8eJtiQDKjhBxVkkYhQCaiF0Q5hWeJOXuTFs7SwnBg,3870
 dstack/api/server/_gateways.py,sha256=_OcFOOxUdh4YIA15ZNQQT6mfj0cSKP2jJ_bQXxugS_w,2568
 dstack/api/server/_group.py,sha256=f9a_YvwZ7s1vDTQhOWHLwm3kO84C5R_zXToD4cxAnvY,432
 dstack/api/server/_logs.py,sha256=ng8QvFAIaoVOVChTK6Wuu5BeM6y7gAdx30KEYRsn9xA,500
@@ -536,7 +536,7 @@ dstack/api/server/_metrics.py,sha256=OPb8sLhI_U605sHOPrELgy0_6cNFLJVfpvr-qkEukRM
 dstack/api/server/_pools.py,sha256=PSs8R4FAKZk-jtD4GmATceMyipnasOy2XEg-8lmiN28,2715
 dstack/api/server/_projects.py,sha256=g6kNSU6jer8u7Kaut1I0Ft4wRMLBBCQShJf3fOB63hQ,1440
 dstack/api/server/_repos.py,sha256=gf9fAQw3rzqfPGq-uT8I2Ju_Zn6G4aWTKis8gb8Polc,1885
-dstack/api/server/_runs.py,sha256=YF9ezqBKC4mBR8Nf79ypHx8yT-ur4GehoQ68EZuEpwQ,6100
+dstack/api/server/_runs.py,sha256=C8vE8uLRjiftRo_wKrr2wpcd_416qy2CsgUNXS3u_yY,6946
 dstack/api/server/_secrets.py,sha256=VqLfrIcmBJtPxNDRkXTG44H5SWoY788YJapScUukvdY,1576
 dstack/api/server/_users.py,sha256=XzhgGKc5Tsr0-xkz3T6rGyWZ1tO7aYNhLux2eE7dAoY,1738
 dstack/api/server/_volumes.py,sha256=E5VlZY44DRkaRUT0swrftYDoEgicDoVIBzzpYkiJhkw,1356
@@ -578,7 +578,7 @@ tests/_internal/core/models/test_volumes.py,sha256=V47bMK5UI1-1wrTl_rq-WClAF_OjC
 tests/_internal/core/models/repos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/core/models/repos/test_remote.py,sha256=6PQcuk9pVfQFG97Vpj_blkK2Veq8tvP6oRg4MUC9mX4,3041
 tests/_internal/core/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/core/services/test_logs.py,sha256=uM4en0RtLbVm_OOVgYU1nsE2_tY03zGzv4civY9TNZE,5290
+tests/_internal/core/services/test_logs.py,sha256=EI4qmFTPWEP-SXYMcXddSlrAqz03PTdx6VRvYl3Jinw,5732
 tests/_internal/core/services/ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/core/services/ssh/test_client.py,sha256=xR8O8DiCtPVRxBILiQLS921XjdxzE3CMXHAQt635_HU,2537
 tests/_internal/core/services/ssh/test_tunnel.py,sha256=icgac2ni8SMgh49uwtPll_vxJu0lLN_py1oHXAo4yqU,7317
@@ -610,7 +610,7 @@ tests/_internal/server/background/tasks/test_process_metrics.py,sha256=k6L0biz7E
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=shHqZvS8QoNwQe8J29-aHk2X2HqX-gsxcQiZevU8yuY,1528
 tests/_internal/server/background/tasks/test_process_running_jobs.py,sha256=EzdAYVUowb9cGb_bdvKG11jTHMuL9D48brXVD4V6GFY,18812
 tests/_internal/server/background/tasks/test_process_runs.py,sha256=Ks3K1O-BMyYSbMr-1L76IEzZ9aLQaVjxF6go-Nj-HxY,13668
-tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=sWXMhti6ws4CBXHOhHypt6IbslNYkO9E0Ai689u3jNE,19767
+tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=K4vdsvTWQfcRWVVAN_yRA8FBqkntIvwnZAjZCObWPsk,22633
 tests/_internal/server/background/tasks/test_process_submitted_volumes.py,sha256=njfRDdDzVtDL8rech_004Wf5z5PpoQVWFufKh1yUyYY,2171
 tests/_internal/server/background/tasks/test_process_terminating_jobs.py,sha256=hatdc4ANgQvJiVLfR9jE3URQjzqFD9FI6ZG9ES_vJOo,9541
 tests/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -623,7 +623,7 @@ tests/_internal/server/routers/test_metrics.py,sha256=NSdb_x_1Nwsqgrf6KXBf257qd2
 tests/_internal/server/routers/test_pools.py,sha256=pqPfIengCzI74D8ojYZbswEtGYhBXFtAlCvzg2RQr6c,25027
 tests/_internal/server/routers/test_projects.py,sha256=Pa_YL9lYhYt06TFmuZFEf9pxc-K3fM9L29ULf4gCELw,23950
 tests/_internal/server/routers/test_repos.py,sha256=4O4mCDlAPh8xJU_XWtwLhFWRkoTxXvZC1N2DLPyz2AI,17225
-tests/_internal/server/routers/test_runs.py,sha256=6BDz8MJLkeOxIiGmhZ0bGsNAqzPfeaFsTIp3RISrOTY,70375
+tests/_internal/server/routers/test_runs.py,sha256=tjY_Q3HVpKky45lCq2TeT_hEhsCqXyCHZnofXy-NYPM,70541
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
 tests/_internal/server/routers/test_users.py,sha256=5QSLvfn9SroGsZoBGmSTEaaqJcdEO8EUUy_YuvLL8ss,12787
 tests/_internal/server/routers/test_volumes.py,sha256=4nv7ba9VkjbYFyYyQmO0coyav_Z9x5t_ayoiyKmvXVw,15947
@@ -641,9 +641,9 @@ tests/_internal/server/services/encryption/keys/__init__.py,sha256=47DEQpj8HBSa-
 tests/_internal/server/services/encryption/keys/test_aes.py,sha256=BtDb5ZeXKKNkAg7KOLXKjpQivMIWpmZe5zaJg-JHFo0,601
 tests/_internal/server/services/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/services/proxy/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/server/services/proxy/routers/test_service_proxy.py,sha256=L-5eubZeixAWsN0qu0v82Y10jUIEvXOmot4GibEcSj8,8263
+tests/_internal/server/services/proxy/routers/test_service_proxy.py,sha256=cGyBwfSgT268CRhkgpTEIM3O1U9z8tSNTYjHQrn3e0E,9843
 tests/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/server/services/runner/test_client.py,sha256=9Z18Cd5haCBRfNafA1-HMl9Mus0yeNmougHJVM_XeUY,16114
+tests/_internal/server/services/runner/test_client.py,sha256=LRJzmyMOGtURc-S4uICfhHjO2K0fxvCa--yXflnoTDM,16212
 tests/_internal/server/services/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/services/services/test_autoscalers.py,sha256=YMi4W5gKFOpEdutc2Fli1L1DIUxCJLHE7ji-CJoiVac,3986
 tests/_internal/server/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -658,9 +658,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.18.38.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.18.38.dist-info/METADATA,sha256=SWmdtNqytsZoKHDkIbR29cUGY_ujgydOXo5ETgkrjuY,17813
-dstack-0.18.38.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.18.38.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.18.38.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.18.38.dist-info/RECORD,,
+dstack-0.18.40rc1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.18.40rc1.dist-info/METADATA,sha256=a1rkEX-5wyY5dIeivhnTmdkNxg5sFPChQ3RgA-3AwJY,17693
+dstack-0.18.40rc1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.18.40rc1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.18.40rc1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.18.40rc1.dist-info/RECORD,,

tests/_internal/core/services/test_logs.py

@@ -1,3 +1,5 @@
+import pytest
+
 from dstack._internal.core.models.runs import AppSpec
 from dstack._internal.core.services.logs import URLReplacer
 
@@ -126,7 +128,18 @@ class TestServiceURLReplacer:
         )
         assert replacer(b"http://0.0.0.0:8000/qwerty") == b"https://secure.host.com/qwerty"
 
-    def test_in_server_proxy(self):
+    @pytest.mark.parametrize(
+        ("in_path", "out_path"),
+        [
+            ("", "/proxy/services/main/service/"),
+            ("/", "/proxy/services/main/service/"),
+            ("/a/b/c", "/proxy/services/main/service/a/b/c"),
+            ("/proxy/services/main/service", "/proxy/services/main/service"),
+            ("/proxy/services/main/service/", "/proxy/services/main/service/"),
+            ("/proxy/services/main/service/a/b/c", "/proxy/services/main/service/a/b/c"),
+        ],
+    )
+    def test_adds_prefix_unless_already_present(self, in_path: str, out_path: str) -> None:
         replacer = URLReplacer(
             ports={8888: 3000},
             app_specs=[],
@@ -135,9 +148,6 @@ class TestServiceURLReplacer:
             path_prefix="/proxy/services/main/service/",
         )
         assert (
-            replacer(b"http://0.0.0.0:8888") == b"http://0.0.0.0:3000/proxy/services/main/service/"
-        )
-        assert (
-            replacer(b"http://0.0.0.0:8888/qwerty")
-            == b"http://0.0.0.0:3000/proxy/services/main/service/qwerty"
+            replacer(f"http://0.0.0.0:8888{in_path}".encode())
+            == f"http://0.0.0.0:3000{out_path}".encode()
         )

tests/_internal/server/background/tasks/test_process_submitted_jobs.py

@@ -302,6 +302,78 @@ class TestProcessSubmittedJobs:
         await session.refresh(pool)
         assert not pool.instances
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+    async def test_provisions_job_with_optional_instance_volume_not_attached(
+        self,
+        test_db,
+        session: AsyncSession,
+    ):
+        project = await create_project(session=session)
+        user = await create_user(session=session)
+        pool = await create_pool(session=session, project=project)
+        repo = await create_repo(
+            session=session,
+            project_id=project.id,
+        )
+        run_spec = get_run_spec(run_name="test-run", repo_id=repo.name)
+        run_spec.configuration.volumes = [
+            InstanceMountPoint(instance_path="/root/.cache", path="/cache", optional=True)
+        ]
+        run = await create_run(
+            session=session,
+            project=project,
+            repo=repo,
+            user=user,
+            run_name="test-run",
+            run_spec=run_spec,
+        )
+        job = await create_job(
+            session=session,
+            run=run,
+            instance_assigned=True,
+        )
+        offer = InstanceOfferWithAvailability(
+            backend=BackendType.RUNPOD,
+            instance=InstanceType(
+                name="instance",
+                resources=Resources(cpus=1, memory_mib=512, spot=False, gpus=[]),
+            ),
+            region="us",
+            price=1.0,
+            availability=InstanceAvailability.AVAILABLE,
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            backend_mock = Mock()
+            m.return_value = [backend_mock]
+            backend_mock.TYPE = BackendType.RUNPOD
+            backend_mock.compute.return_value.get_offers_cached.return_value = [offer]
+            backend_mock.compute.return_value.run_job.return_value = JobProvisioningData(
+                backend=offer.backend,
+                instance_type=offer.instance,
+                instance_id="instance_id",
+                hostname="1.1.1.1",
+                internal_ip=None,
+                region=offer.region,
+                price=offer.price,
+                username="ubuntu",
+                ssh_port=22,
+                ssh_proxy=None,
+                dockerized=False,
+                backend_data=None,
+            )
+            await process_submitted_jobs()
+
+        await session.refresh(job)
+        assert job is not None
+        assert job.status == JobStatus.PROVISIONING
+
+        await session.refresh(pool)
+        instance_offer = InstanceOfferWithAvailability.parse_raw(pool.instances[0].offer)
+        assert offer == instance_offer
+        pool_job_provisioning_data = pool.instances[0].job_provisioning_data
+        assert pool_job_provisioning_data == job.job_provisioning_data
+
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_fails_job_when_no_capacity(self, test_db, session: AsyncSession):
@@ -412,7 +484,8 @@ class TestProcessSubmittedJobs:
         run_spec = get_run_spec(run_name="test-run", repo_id=repo.name)
         run_spec.configuration.volumes = [
             VolumeMountPoint(name=volume.name, path="/volume"),
-            InstanceMountPoint(instance_path="/root/.cache", path="/cache"),
+            InstanceMountPoint(instance_path="/root/.data", path="/data"),
+            InstanceMountPoint(instance_path="/root/.cache", path="/cache", optional=True),
         ]
         run = await create_run(
             session=session,

tests/_internal/server/routers/test_runs.py

@@ -110,6 +110,7 @@ def get_dev_env_run_plan_dict(
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
+                "single_branch": None,
                 "max_duration": "off",
                 "stop_duration": None,
                 "max_price": None,
@@ -180,6 +181,7 @@ def get_dev_env_run_plan_dict(
                     "replica_num": 0,
                     "job_num": 0,
                     "jobs_per_replica": 1,
+                    "single_branch": False,
                     "max_duration": None,
                     "stop_duration": 300,
                     "registry_auth": None,
@@ -261,6 +263,7 @@ def get_dev_env_run_dict(
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
+                "single_branch": None,
                 "max_duration": "off",
                 "stop_duration": None,
                 "max_price": None,
@@ -331,6 +334,7 @@ def get_dev_env_run_dict(
                     "replica_num": 0,
                     "job_num": 0,
                     "jobs_per_replica": 1,
+                    "single_branch": False,
                     "max_duration": None,
                     "stop_duration": 300,
                     "registry_auth": None,