dstack 0.19.10__py3-none-any.whl → 0.19.11rc1__py3-none-any.whl

dstack/_internal/cli/commands/metrics.py

@@ -39,8 +39,6 @@ class MetricsCommand(APIBaseCommand):
         run = self.api.runs.get(run_name=args.run_name)
         if run is None:
             raise CLIError(f"Run {args.run_name} not found")
-        if run.status.is_finished():
-            raise CLIError(f"Run {args.run_name} is finished")
         metrics = _get_run_jobs_metrics(api=self.api, run=run)
 
         if not args.watch:
@@ -55,8 +53,6 @@ class MetricsCommand(APIBaseCommand):
                     run = self.api.runs.get(run_name=args.run_name)
                     if run is None:
                         raise CLIError(f"Run {args.run_name} not found")
-                    if run.status.is_finished():
-                        raise CLIError(f"Run {args.run_name} is finished")
                     metrics = _get_run_jobs_metrics(api=self.api, run=run)
         except KeyboardInterrupt:
             pass
@@ -78,11 +74,12 @@ def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
 def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
     table = Table(box=None)
     table.add_column("NAME", style="bold", no_wrap=True)
+    table.add_column("STATUS")
     table.add_column("CPU")
     table.add_column("MEMORY")
     table.add_column("GPU")
 
-    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
+    run_row: Dict[Union[str, int], Any] = {"NAME": run.name, "STATUS": run.status.value}
     if len(run._run.jobs) != 1:
         add_row_from_dict(table, run_row)
 
@@ -101,9 +98,9 @@ def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
             cpu_usage = f"{cpu_usage:.0f}%"
         memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
         if memory_usage is not None:
-            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
+            memory_usage = _format_memory(memory_usage, 2)
             if resources is not None:
-                memory_usage += f"/{resources.memory_mib}MB"
+                memory_usage += f"/{_format_memory(resources.memory_mib * 1024 * 1024, 2)}"
         gpu_metrics = ""
         gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
         if gpus_detected_num is not None:
@@ -113,13 +110,16 @@ def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
                 if gpu_memory_usage is not None:
                     if i != 0:
                         gpu_metrics += "\n"
-                    gpu_metrics += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
+                    gpu_metrics += f"gpu={i} mem={_format_memory(gpu_memory_usage, 2)}"
                     if resources is not None:
-                        gpu_metrics += f"/{resources.gpus[i].memory_mib}MB"
-                    gpu_metrics += f" {gpu_util_percent}% Util"
+                        gpu_metrics += (
+                            f"/{_format_memory(resources.gpus[i].memory_mib * 1024 * 1024, 2)}"
+                        )
+                    gpu_metrics += f" util={gpu_util_percent}%"
 
         job_row: Dict[Union[str, int], Any] = {
             "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
+            "STATUS": job.job_submissions[-1].status.value,
             "CPU": cpu_usage or "-",
             "MEMORY": memory_usage or "-",
             "GPU": gpu_metrics or "-",
@@ -136,3 +136,18 @@ def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
         if metric.name == name:
             return metric.values[-1]
     return None
+
+
+def _format_memory(memory_bytes: int, decimal_places: int) -> str:
+    """See test_format_memory in tests/_internal/cli/commands/test_metrics.py for examples."""
+    memory_mb = memory_bytes / 1024 / 1024
+    if memory_mb >= 1024:
+        value = memory_mb / 1024
+        unit = "GB"
+    else:
+        value = memory_mb
+        unit = "MB"
+
+    if decimal_places == 0:
+        return f"{round(value)}{unit}"
+    return f"{value:.{decimal_places}f}".rstrip("0").rstrip(".") + unit

dstack/_internal/core/backends/azure/compute.py

@@ -391,11 +391,8 @@ class VMImageVariant(enum.Enum):
 
 
 _SUPPORTED_VM_SERIES_PATTERNS = [
-    # TODO: Support newer CPU series (Dsv6, Esv6).
-    # They are NVMe-only and require marking the VM image as NVMe.
-    r"D(\d+)s_v3",  # Dsv3-series (general purpose)
-    r"E(\d+)i?s_v4",  # Esv4-series (memory optimized)
-    r"E(\d+)-(\d+)s_v4",  # Esv4-series (constrained vCPU)
+    r"D(\d+)s_v6",  # Dsv6-series (general purpose)
+    r"E(\d+)i?s_v6",  # Esv6-series (memory optimized)
     r"F(\d+)s_v2",  # Fsv2-series (compute optimized)
     r"NC(\d+)s_v3",  # NCv3-series [V100 16GB]
     r"NC(\d+)as_T4_v3",  # NCasT4_v3-series [T4]
@@ -404,6 +401,11 @@ _SUPPORTED_VM_SERIES_PATTERNS = [
     r"NC(\d+)ads_A100_v4",  # NC A100 v4-series [A100 80GB]
     r"ND(\d+)asr_v4",  # ND A100 v4-series [8xA100 40GB]
     r"ND(\d+)amsr_A100_v4",  # NDm A100 v4-series [8xA100 80GB]
+    # Deprecated series
+    # TODO: Remove after several releases
+    r"D(\d+)s_v3",  # Dsv3-series (general purpose)
+    r"E(\d+)i?s_v4",  # Esv4-series (memory optimized)
+    r"E(\d+)-(\d+)s_v4",  # Esv4-series (constrained vCPU)
 ]
 _SUPPORTED_VM_SERIES_PATTERN = (
     "^Standard_(" + "|".join(f"({s})" for s in _SUPPORTED_VM_SERIES_PATTERNS) + ")$"

dstack/_internal/core/backends/base/compute.py

@@ -19,6 +19,7 @@ from dstack._internal.core.consts import (
     DSTACK_RUNNER_SSH_PORT,
     DSTACK_SHIM_HTTP_PORT,
 )
+from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR
 from dstack._internal.core.models.gateways import (
     GatewayComputeConfiguration,
     GatewayProvisioningData,
@@ -754,7 +755,7 @@ def get_docker_commands(
             f" --ssh-port {DSTACK_RUNNER_SSH_PORT}"
             " --temp-dir /tmp/runner"
             " --home-dir /root"
-            " --working-dir /workflow"
+            f" --working-dir {DEFAULT_REPO_DIR}"
         ),
     ]
 

dstack/_internal/core/models/configurations.py

@@ -26,6 +26,7 @@ STRIP_PREFIX_DEFAULT = True
 RUN_PRIOTIRY_MIN = 0
 RUN_PRIOTIRY_MAX = 100
 RUN_PRIORITY_DEFAULT = 0
+DEFAULT_REPO_DIR = "/workflow"
 
 
 class RunConfigurationType(str, Enum):
@@ -181,7 +182,7 @@ class BaseRunConfiguration(CoreModel):
         Field(
             description=(
                 "The path to the working directory inside the container."
-                " It's specified relative to the repository directory (`/workflow`) and should be inside it."
+                f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it."
                 ' Defaults to `"."` '
             )
         ),

dstack/_internal/core/models/runs.py

@@ -8,6 +8,7 @@ from typing_extensions import Annotated
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import ApplyAction, CoreModel, NetworkMode, RegistryAuth
 from dstack._internal.core.models.configurations import (
+    DEFAULT_REPO_DIR,
     AnyRunConfiguration,
     RunConfiguration,
 )
@@ -338,7 +339,7 @@ class RunSpec(CoreModel):
         Field(
             description=(
                 "The path to the working directory inside the container."
-                " It's specified relative to the repository directory (`/workflow`) and should be inside it."
+                f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it."
                 ' Defaults to `"."`.'
             )
         ),

dstack/_internal/proxy/gateway/resources/nginx/00-log-format.conf

@@ -1 +1,11 @@
-log_format dstack_stat '$time_iso8601 $host $status $request_time';
+log_format dstack_stat '$time_iso8601 $host $status $request_time $dstack_replica_hit';
+
+
+# A hack to avoid this Nginx reload error when no services are registered:
+#     nginx: [emerg] unknown "dstack_replica_hit" variable
+server {
+    listen unix:/tmp/dstack-dummy-nginx.sock;
+    server_name placeholder.local;
+    deny all;
+    set $dstack_replica_hit 0;
+}

dstack/_internal/proxy/gateway/resources/nginx/service.jinja2

@@ -14,6 +14,7 @@ upstream {{ domain }}.upstream {
 server {
     server_name {{ domain }};
     limit_req_status 429;
+    set $dstack_replica_hit 0;
     access_log {{ access_log_path }} dstack_stat;
     client_max_body_size {{ client_max_body_size }};
 
@@ -23,11 +24,7 @@ server {
         auth_request /_dstack_auth;
         {% endif %}
 
-        {% if replicas %}
         try_files /nonexistent @$http_upgrade;
-        {% else %}
-        return 503;
-        {% endif %}
 
         {% if location.limit_req %}
         limit_req zone={{ location.limit_req.zone }}{% if location.limit_req.burst %} burst={{ location.limit_req.burst }} nodelay{% endif %};
@@ -35,8 +32,9 @@ server {
     }
     {% endfor %}
 
-    {% if replicas %}
     location @websocket {
+        set $dstack_replica_hit 1;
+        {% if replicas %}
         proxy_pass http://{{ domain }}.upstream;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header Host $host;
@@ -44,19 +42,27 @@ server {
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "Upgrade";
         proxy_read_timeout 300s;
+        {% else %}
+        return 503;
+        {% endif %}
     }
     location @ {
+        set $dstack_replica_hit 1;
+        {% if replicas %}
         proxy_pass http://{{ domain }}.upstream;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header Host $host;
         proxy_read_timeout 300s;
+        {% else %}
+        return 503;
+        {% endif %}
     }
-    {% endif %}
 
     {% if auth %}
     location = /_dstack_auth {
         internal;
         if ($remote_addr = 127.0.0.1) {
+            # for requests from the gateway app, e.g. from the OpenAI-compatible API
             return 200;
         }
         proxy_pass http://localhost:{{ proxy_port }}/api/auth/{{ project_name }};

dstack/_internal/proxy/gateway/services/stats.py

@@ -11,10 +11,10 @@ from pydantic import BaseModel
 
 from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
 from dstack._internal.proxy.gateway.schemas.stats import PerWindowStats, ServiceStats, Stat
+from dstack._internal.proxy.lib.errors import UnexpectedProxyError
 from dstack._internal.utils.common import run_async
 
 logger = logging.getLogger(__name__)
-IGNORE_STATUSES = {403, 404}
 WINDOWS = (30, 60, 300)
 TTL = WINDOWS[-1]
 EMPTY_STATS = {window: Stat(requests=0, request_time=0.0) for window in WINDOWS}
@@ -35,6 +35,7 @@ class LogEntry(BaseModel):
     host: str
     status: int
     request_time: float
+    is_replica_hit: bool
 
 
 class StatsCollector:
@@ -87,7 +88,8 @@ class StatsCollector:
         now = datetime.datetime.now(tz=datetime.timezone.utc)
 
         for entry in self._read_access_log(now - datetime.timedelta(seconds=TTL)):
-            if entry.status in IGNORE_STATUSES:
+            # only include requests that hit or should hit a service replica
+            if not entry.is_replica_hit:
                 continue
 
             frame_timestamp = int(entry.timestamp.timestamp())
@@ -119,7 +121,10 @@ class StatsCollector:
                 line = self._file.readline()
                 if not line:
                     break
-                timestamp_str, host, status, request_time = line.split()
+                cells = line.split()
+                if len(cells) == 4:  # compatibility with pre-0.19.11 logs
+                    cells.append("0" if cells[2] in ["403", "404"] else "1")
+                timestamp_str, host, status, request_time, dstack_replica_hit = cells
                 timestamp = datetime.datetime.fromisoformat(timestamp_str)
                 if timestamp < after:
                     continue
@@ -128,6 +133,7 @@ class StatsCollector:
                     host=host,
                     status=int(status),
                     request_time=float(request_time),
+                    is_replica_hit=_parse_nginx_bool(dstack_replica_hit),
                 )
             if os.fstat(self._file.fileno()).st_ino != st_ino:
                 # file was rotated
@@ -154,3 +160,11 @@ async def get_service_stats(
         )
         for service in services
     ]
+
+
+def _parse_nginx_bool(v: str) -> bool:
+    if v == "0":
+        return False
+    if v == "1":
+        return True
+    raise UnexpectedProxyError(f"Cannot parse boolean value: expected '0' or '1', got {v!r}")

dstack/_internal/server/services/jobs/configurators/base.py

@@ -10,6 +10,7 @@ from dstack._internal import settings
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
 from dstack._internal.core.models.common import RegistryAuth
 from dstack._internal.core.models.configurations import (
+    DEFAULT_REPO_DIR,
     PortMapping,
     PythonVersion,
     RunConfigurationType,
@@ -149,7 +150,8 @@ class JobConfigurator(ABC):
             commands = self.run_spec.configuration.commands
         elif shell_commands := self._shell_commands():
             entrypoint = [self._shell(), "-i", "-c"]
-            commands = [_join_shell_commands(shell_commands)]
+            dstack_image_commands = self._dstack_image_commands()
+            commands = [_join_shell_commands(dstack_image_commands + shell_commands)]
         else:  # custom docker image without commands
             image_config = await self._get_image_config()
             entrypoint = image_config.entrypoint or []
@@ -164,6 +166,18 @@ class JobConfigurator(ABC):
 
         return result
 
+    def _dstack_image_commands(self) -> List[str]:
+        if (
+            self.run_spec.configuration.image is not None
+            or self.run_spec.configuration.entrypoint is not None
+        ):
+            return []
+        return [
+            f"uv venv --prompt workflow --seed {DEFAULT_REPO_DIR}/.venv > /dev/null 2>&1",
+            f"echo 'source {DEFAULT_REPO_DIR}/.venv/bin/activate' >> ~/.bashrc",
+            f"source {DEFAULT_REPO_DIR}/.venv/bin/activate",
+        ]
+
     def _app_specs(self) -> List[AppSpec]:
         specs = []
         for i, pm in enumerate(filter_reserved_ports(self._ports())):

dstack/_internal/server/services/jobs/configurators/extensions/cursor.py

@@ -1,5 +1,7 @@
 from typing import List
 
+from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR
+
 
 class CursorDesktop:
     def __init__(
@@ -37,6 +39,6 @@ class CursorDesktop:
         return [
             "echo To open in Cursor, use link below:",
             "echo ''",
-            f"echo '  cursor://vscode-remote/ssh-remote+{self.run_name}/workflow'",  # TODO use $REPO_DIR
+            f"echo '  cursor://vscode-remote/ssh-remote+{self.run_name}{DEFAULT_REPO_DIR}'",  # TODO use $REPO_DIR
             "echo ''",
         ]

dstack/_internal/server/services/jobs/configurators/extensions/vscode.py

@@ -1,5 +1,7 @@
 from typing import List
 
+from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR
+
 
 class VSCodeDesktop:
     def __init__(
@@ -37,6 +39,6 @@ class VSCodeDesktop:
         return [
             "echo To open in VS Code Desktop, use link below:",
             "echo ''",
-            f"echo '  vscode://vscode-remote/ssh-remote+{self.run_name}/workflow'",  # TODO use $REPO_DIR
+            f"echo '  vscode://vscode-remote/ssh-remote+{self.run_name}{DEFAULT_REPO_DIR}'",  # TODO use $REPO_DIR
             "echo ''",
         ]

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.10"
+__version__ = "0.19.11rc1"
 __is_release__ = True
-base_image = "0.7"
+base_image = "0.8"

{dstack-0.19.10.dist-info → dstack-0.19.11rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.10
+Version: 0.19.11rc1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack

{dstack-0.19.10.dist-info → dstack-0.19.11rc1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=1w-1PVkazoEpWhJboy2UsLiK2Ezuk8Le2ZyTHPXYf7E,65
+dstack/version.py,sha256=GxG1ziHLEcRXoddW7khMHK2u0fShzLeWrp1qMzeZKMQ,68
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=GOqfcLEONWC1hGU36IYPuOhJXP_qC3Y6d2SQge_NdpY,953
@@ -15,7 +15,7 @@ dstack/_internal/cli/commands/fleet.py,sha256=J0Yi5CAYvIQF_NC4ZBsDYGc3grTOtvaF7s
 dstack/_internal/cli/commands/gateway.py,sha256=DcD6P_MvXbSL9aXkLX9hgGYSAzARjgY6RSbrCMzdNcg,6075
 dstack/_internal/cli/commands/init.py,sha256=bLhSlViNWtjflB6xNq_PuCR2o2A06h222luh1NeUgVA,1169
 dstack/_internal/cli/commands/logs.py,sha256=o8ehPAKM12Xn9thg2jjnYdr7_wKqF-00ziVry8IVVwE,1528
-dstack/_internal/cli/commands/metrics.py,sha256=nxdTcyCvHNev7Mn19zGQ7vTHUBGY6hyvPtY6Z731SOU,5373
+dstack/_internal/cli/commands/metrics.py,sha256=ZS_dHD4_YK135J09xKpVTY_3CXzhj8QhDo18Ail9Pas,5862
 dstack/_internal/cli/commands/offer.py,sha256=rf10T9ohEC77GmZexdqF_RBCQkLYzVSAwTzZlxukNYM,4085
 dstack/_internal/cli/commands/project.py,sha256=V8gJwQWQGvGKQJPUyce_M7Ij_B7bOxYzo2IQgkLLHFE,6151
 dstack/_internal/cli/commands/ps.py,sha256=5RU4mcL4-FT4PcXktfKb4htCOPqia7mIuVGgfK7x9lY,1894
@@ -58,14 +58,14 @@ dstack/_internal/core/backends/aws/resources.py,sha256=Wht_USG8ps9HQBhAYfxDxjaob
 dstack/_internal/core/backends/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/azure/auth.py,sha256=CHp3QRNx3tIwkhyOY_l8WU-ElY4Pxhavoy9JSdwr15g,1259
 dstack/_internal/core/backends/azure/backend.py,sha256=XySTor8v_tLkZctDBryrFHrbVzQgNf_RUPkXQYplvwM,746
-dstack/_internal/core/backends/azure/compute.py,sha256=3CH6m1ytCG-cvj3sAQG3BPpehZQ7XQkAFE8JOyja0V8,24433
+dstack/_internal/core/backends/azure/compute.py,sha256=waunprBzpiJBMtVDJMjFCxEdmtO52E7ckzcc6OcZLf4,24488
 dstack/_internal/core/backends/azure/configurator.py,sha256=cC0YhiayPtrOEEr8Lr-9nDDq6v2-sbxK-1QhixpM1YY,18228
 dstack/_internal/core/backends/azure/models.py,sha256=ap0WsYBSFdDCxY0G3W1sg2DBN2k0xknz6bp8Ypf6phs,3356
 dstack/_internal/core/backends/azure/resources.py,sha256=cJWS2Yjxdg-XtjpdBxfVDZaX_gTbuoEpmdjTxdA9uIw,3431
 dstack/_internal/core/backends/azure/utils.py,sha256=taHMJq6UHRzUXLUcO2P5VCKy3wJaye2bG-6QdkEPNdY,1741
 dstack/_internal/core/backends/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/base/backend.py,sha256=hdFMHED1RMV9GVfLSU0yGhGE-cXlbLvt1coDA885PMM,505
-dstack/_internal/core/backends/base/compute.py,sha256=0APW_2yo4zO-mHw6vg-5nnMnwppR4QPGrFN3GQmD-EU,28542
+dstack/_internal/core/backends/base/compute.py,sha256=XblmmEz4muUjcnlIudKEwoPSaqMBVZtbOcwgGs47Xvc,28625
 dstack/_internal/core/backends/base/configurator.py,sha256=OCv8N2oxcxy3In2zS1PKiCJ0a-COZwxGjBz2FYkQnfg,3807
 dstack/_internal/core/backends/base/models.py,sha256=Ij0osOl-T-ABsKLoVg2eY81DMkwdWkevAnjXj2QnLXI,532
 dstack/_internal/core/backends/base/offers.py,sha256=AzAAx5eSTaHv8CbWuGERTHS151x4oJuQn3tr_Bz9HKQ,6282
@@ -156,7 +156,7 @@ dstack/_internal/core/backends/vultr/models.py,sha256=TkqK_lW6w5k_7wsoFiKDb6NXuT
 dstack/_internal/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/models/common.py,sha256=XWd79dmFGMrdpTcStH5fVmNXCKE0s7FsIorPL0saQ8w,2138
 dstack/_internal/core/models/config.py,sha256=JJ7rT7dztzTWCY5TkoyxXxTvG5D4IFYhGe7EzwkLOWQ,581
-dstack/_internal/core/models/configurations.py,sha256=I-BRBls-oSZ2EZy5glC4_PJbdM-_KnoAxl6gm1u16tg,19027
+dstack/_internal/core/models/configurations.py,sha256=yG5GaCbTK4QhSh0UMYrYpFSeFvD0f-Uhi3TT_XviGhQ,19068
 dstack/_internal/core/models/envs.py,sha256=yq84YRFBILOy4x3XnGcTgYpbZ69eFTCQPgBCr9Ndov4,4969
 dstack/_internal/core/models/fleets.py,sha256=gbP2rj1ODDuy4IfTbPNB-ae050thvzTtA8uDb5iTmdY,12059
 dstack/_internal/core/models/gateways.py,sha256=_O8EWwHWLdgNoWY4P4u71KM-uEr5DDp42LXfyv1qMDI,4054
@@ -167,7 +167,7 @@ dstack/_internal/core/models/placement.py,sha256=WJVq5ENJykyRarQzL2EeYQag_9_jV7V
 dstack/_internal/core/models/profiles.py,sha256=seeysTuMv1vVUmpHAZgrMUGcbMtH7hSMFIvfx0Qk__0,10406
 dstack/_internal/core/models/projects.py,sha256=H5ZZRiyUEKifpTFAhl45KBi5ly7ooE0WmI329myK360,643
 dstack/_internal/core/models/resources.py,sha256=rsf6hAhi5bfSb_Z9VcS3UoEG0G8Ohl6ekyrOStLOAqw,14114
-dstack/_internal/core/models/runs.py,sha256=Xkv1kY68JA0eJUeCVJjM9YWRkfy6P1RGXslBIMtox2E,18985
+dstack/_internal/core/models/runs.py,sha256=6egFnxV0qGs-XCr22RvJc-rSpc3zsNIk2UF3cRdWlX4,19017
 dstack/_internal/core/models/secrets.py,sha256=IQyemsNpSzqOCB-VlVTuc4gyPFmXXO4mhko0Ur0ey3I,221
 dstack/_internal/core/models/server.py,sha256=Hkc1v2s3KOiwslsWVmhUOAzcSeREoG-HD1SzSX9WUGg,152
 dstack/_internal/core/models/services.py,sha256=2Hpi7j0Q1shaf_0wd0C0044AJAmuYi-D3qx3PH849oI,3076
@@ -204,9 +204,9 @@ dstack/_internal/proxy/gateway/models.py,sha256=OLarOKJj-6teeg9dvxp3ueRv2_7yDJJf
 dstack/_internal/proxy/gateway/repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/gateway/repo/repo.py,sha256=0i9FaLwDLnf8hijtRjT8hBooWj7EJ6L0_WJMbFaIBmo,4629
 dstack/_internal/proxy/gateway/repo/state_v1.py,sha256=fngXqJWONuoJZEQcKHgiOGvs-ZQ3NATMzUFn67w4xxY,5943
-dstack/_internal/proxy/gateway/resources/nginx/00-log-format.conf,sha256=hqo9-5Ss1htl6s0dxXnnj1Mgjgs3ijJS8QmZ529CQaI,68
+dstack/_internal/proxy/gateway/resources/nginx/00-log-format.conf,sha256=-nVcJRFb4qbxy-t9BOq03dvdBtZ-P-Fi81gOzS0GnPc,361
 dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2,sha256=umnUlm34wfoLNik3GFr1Gn2jtZMAx1-kgjBYcY3sLAU,817
-dstack/_internal/proxy/gateway/resources/nginx/service.jinja2,sha256=9ulU4qh1fQwJvh478srAlRcay6s9nONsEnO7NYcMa7A,2547
+dstack/_internal/proxy/gateway/resources/nginx/service.jinja2,sha256=sccX4pIfdErti7VPhvN_9ObEJgbvsZc-COTJk1dUe8U,2780
 dstack/_internal/proxy/gateway/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/gateway/routers/auth.py,sha256=PCMYEy2HLbiSZgqrtMUH90hbMAMXHp1nSZ0j13Nlrhs,253
 dstack/_internal/proxy/gateway/routers/config.py,sha256=1UKHAPSYkeG8ULl9X3Eu6nPg94Xp-fecTObRa5Yw294,908
@@ -221,7 +221,7 @@ dstack/_internal/proxy/gateway/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5J
 dstack/_internal/proxy/gateway/services/nginx.py,sha256=tGvs0Ci1X0aO8HjaflCgkxEXop0Z7o6euw7ADfCUOro,6149
 dstack/_internal/proxy/gateway/services/registry.py,sha256=Gqyri5PaJW79iRYufawS09XHO_q7a1Re3tx2dGjBTfQ,13580
 dstack/_internal/proxy/gateway/services/server_client.py,sha256=KSnpO8iPm5wxQPU0DcW7kqUzwliaQrb3b0K8Jmk8iA0,3630
-dstack/_internal/proxy/gateway/services/stats.py,sha256=i88GpWMUlB8tZTWAQHPNh420qbgSGAhnaxNgQN2zK78,5326
+dstack/_internal/proxy/gateway/services/stats.py,sha256=F1LI34oXk_vF3VkWqpRnf_aO2ykECv6_WmBlZVa2jU8,5945
 dstack/_internal/proxy/gateway/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/gateway/testing/common.py,sha256=K50IHpdxM9NpojVPrFFEaZyMwTAHvDU76VLB3qlZBn8,269
 dstack/_internal/proxy/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -402,14 +402,14 @@ dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBW
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
 dstack/_internal/server/services/jobs/__init__.py,sha256=GU3vMC0SZKyvL564A7t_QRoDjf83-8GsUkguDWK5x6c,25578
 dstack/_internal/server/services/jobs/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/jobs/configurators/base.py,sha256=kgCbEom7hGPunrUxXI0NjXGlTdNq_Q27u56lrExDnT8,11390
+dstack/_internal/server/services/jobs/configurators/base.py,sha256=WLRIBM50P8mfvpotzYjK3Gtf5ScQsuijGieH_zaYqCg,11984
 dstack/_internal/server/services/jobs/configurators/dev.py,sha256=ufN6Sd8TwIsjQYNZE32fkAqJI7o2zjgoZThbrP-bd7U,2378
 dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YFSMuGqjOwYTLMV4FuIM5lCMDFjS0l0CoLI,929
 dstack/_internal/server/services/jobs/configurators/task.py,sha256=0-B3oO-61Eq4-mmlLmqJPliFKHhvvIV0tqc12slcQuA,1436
 dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
-dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=00HB1kC_eMlioEW0nZI7Ly78b-RSehySVNhC9pajBP8,1636
-dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
+dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=r6Cgdd-P5C63xXyrqC6E0yNKIeTX6KabRLO92VEpJRY,1719
+dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=WRg_jqi4eSqbs3XRVASMFF7OQHYRDtVBNn6NCroJY1I,1694
 dstack/_internal/server/services/logs/__init__.py,sha256=NAjO1KeYvuDznN2EkfAaJt9S6Y00fo_dl3ob3WmsdGQ,3088
 dstack/_internal/server/services/logs/aws.py,sha256=MuRVGgYwkh0TA11Y2H0upSj56-k_Bs-vT_oa2X-5yZE,13834
 dstack/_internal/server/services/logs/base.py,sha256=7eCB6SPF9iWviv4vZ_34SvfM_Bvi9DeVj1t7TxmrXKs,1273
@@ -584,8 +584,8 @@ dstack/plugins/__init__.py,sha256=buT1pcyORLgVbl89ATkRWJPhvejriVz7sNBjvuZRCRE,40
 dstack/plugins/_base.py,sha256=-etiB-EozaJCg2wtmONfj8ic-K03qXvXyl_TIDp-kNE,2662
 dstack/plugins/_models.py,sha256=1Gw--mDQ1_0FFr9Zur9LE8UbMoWESUpTdHHt12AyIZo,341
 dstack/plugins/_utils.py,sha256=FqeWYb7zOrgZkO9Bd8caL5I81_TUEsysIzvxsULrmzk,392
-dstack-0.19.10.dist-info/METADATA,sha256=kOxSPHWY8taNRczMVwanWtP_p-0KmFJbOSE7vCKQsKk,20509
-dstack-0.19.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dstack-0.19.10.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.10.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.10.dist-info/RECORD,,
+dstack-0.19.11rc1.dist-info/METADATA,sha256=ZU6h89CiVMvwc_lXF_Cv4LI2DPdA6_zvPr7rZF1iTJY,20512
+dstack-0.19.11rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dstack-0.19.11rc1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.11rc1.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.11rc1.dist-info/RECORD,,