dstack 0.18.43__py3-none-any.whl → 0.18.44__py3-none-any.whl

dstack/_internal/server/statics/static/media/entraID.d65d1f3e9486a8e56d24fc07b3230885.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg id="uuid-f8d4d392-7c12-4bd9-baff-66fbf7814b91" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 18 18">
+  <path d="m3.802,14.032c.388.242,1.033.511,1.715.511.621,0,1.198-.18,1.676-.487,0,0,.001,0,.002-.001l1.805-1.128v4.073c-.286,0-.574-.078-.824-.234l-4.374-2.734Z" fill="#225086"/>
+  <path d="m7.853,1.507L.353,9.967c-.579.654-.428,1.642.323,2.111,0,0,2.776,1.735,3.126,1.954.388.242,1.033.511,1.715.511.621,0,1.198-.18,1.676-.487,0,0,.001,0,.002-.001l1.805-1.128-4.364-2.728,4.365-4.924V1s0,0,0,0c-.424,0-.847.169-1.147.507Z" fill="#6df"/>
+  <polygon points="4.636 10.199 4.688 10.231 9 12.927 9.001 12.927 9.001 12.927 9.001 5.276 9 5.275 4.636 10.199" fill="#cbf8ff"/>
+  <path d="m17.324,12.078c.751-.469.902-1.457.323-2.111l-4.921-5.551c-.397-.185-.842-.291-1.313-.291-.925,0-1.752.399-2.302,1.026l-.109.123h0s4.364,4.924,4.364,4.924h0s0,0,0,0l-4.365,2.728v4.073c.287,0,.573-.078.823-.234l7.5-4.688Z" fill="#074793"/>
+  <path d="m9.001,1v4.275s.109-.123.109-.123c.55-.627,1.377-1.026,2.302-1.026.472,0,.916.107,1.313.291l-2.579-2.909c-.299-.338-.723-.507-1.146-.507Z" fill="#0294e4"/>
+  <polygon points="13.365 10.199 13.365 10.199 13.365 10.199 9.001 5.276 9.001 12.926 13.365 10.199" fill="#96bcc2"/>
+</svg>

dstack/_internal/server/testing/common.py

@@ -69,6 +69,7 @@ from dstack._internal.server.models import (
     InstanceModel,
     JobMetricsPoint,
     JobModel,
+    JobPrometheusMetrics,
     PlacementGroupModel,
     PoolModel,
     ProjectModel,
@@ -858,6 +859,22 @@ async def create_job_metrics_point(
     return jmp
 
 
+async def create_job_prometheus_metrics(
+    session: AsyncSession,
+    job: JobModel,
+    collected_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+    text: str = "# Prometheus metrics\n",
+):
+    metrics = JobPrometheusMetrics(
+        job_id=job.id,
+        collected_at=collected_at,
+        text=text,
+    )
+    session.add(metrics)
+    await session.commit()
+    return metrics
+
+
 def get_private_key_string() -> str:
     return """
 -----BEGIN RSA PRIVATE KEY-----

dstack/api/_public/runs.py

@@ -25,6 +25,7 @@ from dstack._internal.core.models.profiles import (
     ProfileRetryPolicy,
     SpotPolicy,
     TerminationPolicy,
+    UtilizationPolicy,
 )
 from dstack._internal.core.models.repos.base import Repo
 from dstack._internal.core.models.resources import ResourcesSpec
@@ -485,6 +486,7 @@ class RunCollection:
         resources: Optional[ResourcesSpec] = None,
         spot_policy: Optional[SpotPolicy] = None,
         retry_policy: Optional[ProfileRetryPolicy] = None,
+        utilization_policy: Optional[UtilizationPolicy] = None,
         max_duration: Optional[Union[int, str]] = None,
         max_price: Optional[float] = None,
         working_dir: Optional[str] = None,
@@ -535,6 +537,7 @@ class RunCollection:
             spot_policy=spot_policy,
             retry=None,
             retry_policy=retry_policy,
+            utilization_policy=utilization_policy,
             max_duration=max_duration,
             stop_duration=stop_duration,
             max_price=max_price,

dstack/api/server/_fleets.py

@@ -104,6 +104,8 @@ def _get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[_ExcludeDict]:
         profile_excludes.add("availability_zones")
     if fleet_spec.configuration.blocks == 1:
         configuration_excludes["blocks"] = True
+    if fleet_spec.profile is not None and fleet_spec.profile.utilization_policy is None:
+        profile_excludes.add("utilization_policy")
 
     if ssh_hosts_excludes:
         ssh_config_excludes["hosts"] = {"__all__": ssh_hosts_excludes}

dstack/api/server/_runs.py

@@ -185,6 +185,10 @@ def _get_run_spec_excludes(run_spec: RunSpec) -> Optional[dict]:
         and configuration.inactivity_duration is None
     ):
         configuration_excludes["inactivity_duration"] = True
+    if configuration.utilization_policy is None:
+        configuration_excludes["utilization_policy"] = True
+    if profile is not None and profile.utilization_policy is None:
+        profile_excludes.add("utilization_policy")
 
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes

dstack/api/utils.py

@@ -2,6 +2,7 @@ from pathlib import Path
 from typing import Optional, Tuple
 
 import yaml
+from pydantic import ValidationError
 
 from dstack._internal.core.errors import ConfigurationError
 from dstack._internal.core.models.configurations import AnyRunConfiguration
@@ -96,6 +97,8 @@ def _load_profile_from_path(profiles_path: Path, profile_name: Optional[str]) ->
             config = ProfilesConfig.parse_obj(yaml.safe_load(f))
     except FileNotFoundError:
         return None
+    except ValidationError as e:
+        raise ConfigurationError(e)
 
     if profile_name is None:
         return config.default()

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.18.43"
+__version__ = "0.18.44"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.18.43.dist-info → dstack-0.18.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.18.43
+Version: 0.18.44
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov
@@ -60,6 +60,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "all"
 Requires-Dist: asyncpg; extra == "all"
 Requires-Dist: cachetools; extra == "all"
 Requires-Dist: python-json-logger>=3.1.0; extra == "all"
+Requires-Dist: prometheus-client; extra == "all"
 Requires-Dist: grpcio>=1.50; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: botocore; extra == "all"
@@ -100,6 +101,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "aws"
 Requires-Dist: asyncpg; extra == "aws"
 Requires-Dist: cachetools; extra == "aws"
 Requires-Dist: python-json-logger>=3.1.0; extra == "aws"
+Requires-Dist: prometheus-client; extra == "aws"
 Requires-Dist: grpcio>=1.50; extra == "aws"
 Requires-Dist: boto3; extra == "aws"
 Requires-Dist: botocore; extra == "aws"
@@ -124,6 +126,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "azure"
 Requires-Dist: asyncpg; extra == "azure"
 Requires-Dist: cachetools; extra == "azure"
 Requires-Dist: python-json-logger>=3.1.0; extra == "azure"
+Requires-Dist: prometheus-client; extra == "azure"
 Requires-Dist: grpcio>=1.50; extra == "azure"
 Requires-Dist: azure-identity>=1.12.0; extra == "azure"
 Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == "azure"
@@ -152,6 +155,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "datacrunch"
 Requires-Dist: asyncpg; extra == "datacrunch"
 Requires-Dist: cachetools; extra == "datacrunch"
 Requires-Dist: python-json-logger>=3.1.0; extra == "datacrunch"
+Requires-Dist: prometheus-client; extra == "datacrunch"
 Requires-Dist: grpcio>=1.50; extra == "datacrunch"
 Requires-Dist: datacrunch; extra == "datacrunch"
 Provides-Extra: gateway
@@ -183,6 +187,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "gcp"
 Requires-Dist: asyncpg; extra == "gcp"
 Requires-Dist: cachetools; extra == "gcp"
 Requires-Dist: python-json-logger>=3.1.0; extra == "gcp"
+Requires-Dist: prometheus-client; extra == "gcp"
 Requires-Dist: grpcio>=1.50; extra == "gcp"
 Requires-Dist: google-auth>=2.3.0; extra == "gcp"
 Requires-Dist: google-cloud-storage>=2.0.0; extra == "gcp"
@@ -212,6 +217,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "kubernetes"
 Requires-Dist: asyncpg; extra == "kubernetes"
 Requires-Dist: cachetools; extra == "kubernetes"
 Requires-Dist: python-json-logger>=3.1.0; extra == "kubernetes"
+Requires-Dist: prometheus-client; extra == "kubernetes"
 Requires-Dist: grpcio>=1.50; extra == "kubernetes"
 Requires-Dist: kubernetes; extra == "kubernetes"
 Provides-Extra: lambda
@@ -235,6 +241,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "lambda"
 Requires-Dist: asyncpg; extra == "lambda"
 Requires-Dist: cachetools; extra == "lambda"
 Requires-Dist: python-json-logger>=3.1.0; extra == "lambda"
+Requires-Dist: prometheus-client; extra == "lambda"
 Requires-Dist: grpcio>=1.50; extra == "lambda"
 Requires-Dist: boto3; extra == "lambda"
 Requires-Dist: botocore; extra == "lambda"
@@ -259,6 +266,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "oci"
 Requires-Dist: asyncpg; extra == "oci"
 Requires-Dist: cachetools; extra == "oci"
 Requires-Dist: python-json-logger>=3.1.0; extra == "oci"
+Requires-Dist: prometheus-client; extra == "oci"
 Requires-Dist: grpcio>=1.50; extra == "oci"
 Requires-Dist: oci; extra == "oci"
 Provides-Extra: server
@@ -282,6 +290,7 @@ Requires-Dist: alembic-postgresql-enum; extra == "server"
 Requires-Dist: asyncpg; extra == "server"
 Requires-Dist: cachetools; extra == "server"
 Requires-Dist: python-json-logger>=3.1.0; extra == "server"
+Requires-Dist: prometheus-client; extra == "server"
 Requires-Dist: grpcio>=1.50; extra == "server"
 
 <div style="text-align: center;">

{dstack-0.18.43.dist-info → dstack-0.18.44.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=d3JTxTwIqwsUKYkzkrgMvw4t7I2HgxbJ2Bl3oR5u-g4,65
+dstack/version.py,sha256=Ct5iULpJsRMebp_vbEGziNAlFoo4aB8f00ddufM56Js,65
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
@@ -31,14 +31,14 @@ dstack/_internal/cli/services/configurators/__init__.py,sha256=z94VPBFqybP8Zpwy3
 dstack/_internal/cli/services/configurators/base.py,sha256=DxVuF9pDqghq9mFCcTDG1CxO06IAAwvzlc9nHflCipc,3443
 dstack/_internal/cli/services/configurators/fleet.py,sha256=bKnWXhr7r8GndxedNdG_TWvwcxV4gab_LFt4s3Q17D0,14822
 dstack/_internal/cli/services/configurators/gateway.py,sha256=czB2s89s7IowOmWnpDwWErPAUlW3FvFMizImhrkQiBM,8927
-dstack/_internal/cli/services/configurators/run.py,sha256=kRbT-yczl8h4rjCjiWMd2bB44x7iBWqDC1qKCOPi2HU,23072
+dstack/_internal/cli/services/configurators/run.py,sha256=bmp4-adMi0DAsPpaL4ntxZhoAB7to3hLgEOxDhv1Mgo,23135
 dstack/_internal/cli/services/configurators/volume.py,sha256=riMXLQbgvHIIFwLKdHfad-_0iE9wE3G_rUmXU5P3ZS8,8519
 dstack/_internal/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/cli/utils/common.py,sha256=rfmzqrsgR3rXW3wj0vxDdvrhUUg2aIy4A6E9MZbd55g,1763
 dstack/_internal/cli/utils/fleet.py,sha256=NpqTdcMVcuUGke9drroxDoNnmifbKV7iziqdGPLc3lk,3885
 dstack/_internal/cli/utils/gateway.py,sha256=jMytH6u3x8hctMhm9bcmXLJxSgTXmpW8M9abATQrw3c,1474
 dstack/_internal/cli/utils/rich.py,sha256=Gx1MJU929kMKsbdo9qF7XHARNta2426Ssb-xMLVhwbQ,5710
-dstack/_internal/cli/utils/run.py,sha256=4NDhlcdE4bccQ4gWzhZ2EP8tHryZt4eI3BBho425VOA,7737
+dstack/_internal/cli/utils/run.py,sha256=TCKTdtwrgEIV1ME3g4q_GVGzp2KHBVoJwcKjXZmS-nc,8399
 dstack/_internal/cli/utils/updates.py,sha256=9KQcm_TEE_PiU0yc5WH4xRse0WIeHOubi9Et55odgvk,2601
 dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14rHudNQN4,1975
 dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -47,12 +47,12 @@ dstack/_internal/core/errors.py,sha256=UvqMIBVHG1NIUP0w0g5dWjnxOz37sfFi4lh4yuYK0
 dstack/_internal/core/backends/__init__.py,sha256=P7uX9QYj-aWSHOub1vIwr-MVERVCwTi8pcG4MQVuEYw,958
 dstack/_internal/core/backends/aws/__init__.py,sha256=gGXM9op3UJ3-OGDjNr4ywa_fNf--oywRbrOGUH2tTDA,850
 dstack/_internal/core/backends/aws/auth.py,sha256=lRBwNUdEWMWlblDnFSGlIcNkZMY2tSjapdq1OjwollU,1253
-dstack/_internal/core/backends/aws/compute.py,sha256=PIxXZEzUQoLqIiplPkDyP5pWUgTPXz8XVe5Bg9Ge3hQ,35945
+dstack/_internal/core/backends/aws/compute.py,sha256=UsSYadzCzkCM1jIH64fQRaQz94PKwyKGSVzegXJx-iQ,35964
 dstack/_internal/core/backends/aws/config.py,sha256=bd4juOaiqsUsWeorIgqCW0xNigCFIRQIvaRxEHyhpwg,533
 dstack/_internal/core/backends/aws/resources.py,sha256=8Pm1zwejYESlH559GB0zNg5zVBgC5D4bPyGx7zBRhmg,22540
 dstack/_internal/core/backends/azure/__init__.py,sha256=J-mWp413v0DQFaTochlvMJXlGf8ukQCH_6oGuYFR31U,718
 dstack/_internal/core/backends/azure/auth.py,sha256=CZZlcUMkjh18wwtMl75K7kfob1bc9aI1TRr5QwMn_u8,1523
-dstack/_internal/core/backends/azure/compute.py,sha256=ERtj3cEWVUR6MeA8W4dr4fo4SqtVMPfbkomoge5IIZs,23290
+dstack/_internal/core/backends/azure/compute.py,sha256=og9darSt5c8mM3vDFbwZS825eApVietTlP4diO8kBC4,23290
 dstack/_internal/core/backends/azure/config.py,sha256=nb7e9rBt5glSOcXW1NZUxlyjfe2W1CiEPERNlxLcLUo,391
 dstack/_internal/core/backends/azure/resources.py,sha256=keE3ruSSNWhSYMCkG7832TYppMMe8lDEyph0tulhd18,3048
 dstack/_internal/core/backends/azure/utils.py,sha256=taHMJq6UHRzUXLUcO2P5VCKy3wJaye2bG-6QdkEPNdY,1741
@@ -70,7 +70,7 @@ dstack/_internal/core/backends/datacrunch/compute.py,sha256=8ldjO2CtiOHTj9LsoIv2
 dstack/_internal/core/backends/datacrunch/config.py,sha256=bShWTAb02PYP8fuBz7H7r-YhRkhy6_yfD0nuaKd9xd4,281
 dstack/_internal/core/backends/gcp/__init__.py,sha256=nxhPNTGd1pPag2DwBRKg3X5gEKlh32p7a9Yfyi7-d5k,546
 dstack/_internal/core/backends/gcp/auth.py,sha256=hSClOujk8_kcodqK4SltdaIty6kKaQAecHYya99-LBQ,2495
-dstack/_internal/core/backends/gcp/compute.py,sha256=-tp5z4cwUY3SVCSsijRHm8QnBaAzvMh-wVs6sVHdoCE,36087
+dstack/_internal/core/backends/gcp/compute.py,sha256=YhssHfSCoKTjchzklRiVA4VIFDdRLzc_snia4tVrlBw,36092
 dstack/_internal/core/backends/gcp/config.py,sha256=AjNHgIsDqB5aKdyFXTKG9wNO2d3lgNyJW4Dk5Yb-LL4,722
 dstack/_internal/core/backends/gcp/resources.py,sha256=ZDs8rjdMCgHeeHH_vzxJjhCJN-omF4tRBApkN3GOjpg,14539
 dstack/_internal/core/backends/kubernetes/__init__.py,sha256=MpuT3Dxt2ZwZpAQomE_fXUDmgHBmfP8xvUP1YMAKTJQ,573
@@ -99,8 +99,8 @@ dstack/_internal/core/backends/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 dstack/_internal/core/backends/remote/provisioning.py,sha256=oNbomEuXIBdgYeCh6_RWB0nWtnrJ0-HRf9M6VYgoQvk,12461
 dstack/_internal/core/backends/runpod/__init__.py,sha256=CC9K4rZlSxpSIqt9akM1PIRzaOGozreY4hQWxO8dRZ4,537
 dstack/_internal/core/backends/runpod/api_client.py,sha256=Oet7CwQTu3sE57unqh-JvuMCBDEDPruAbGB4GyM0dM4,13427
-dstack/_internal/core/backends/runpod/compute.py,sha256=u6OSdGCXH5L2JX6ztEtXHW76MNv8ZE95PIhhVcfsHnU,9867
-dstack/_internal/core/backends/runpod/config.py,sha256=V9ihNEigp5szdDuffHDM47JsshyPc25x6jHwbuc2YhY,257
+dstack/_internal/core/backends/runpod/compute.py,sha256=XCOdW_vcpSrH4sEPK83jWuUphmnq7Z87Ho_O5Hu0zhA,10491
+dstack/_internal/core/backends/runpod/config.py,sha256=5orria50iwzgIu_mEA9yXwyNyoZJXa2QJU4oqtE7wgU,487
 dstack/_internal/core/backends/tensordock/__init__.py,sha256=AihBzxY8RjJHXYrpqaSGCkcOwqYsMgiyIlH16fnHxso,573
 dstack/_internal/core/backends/tensordock/api_client.py,sha256=4106Jzn_I1WTX0VZkZk3I-xkG7sHGTUBRapi8Fk2H7o,3805
 dstack/_internal/core/backends/tensordock/compute.py,sha256=njDKvNxN7Fupoxw_eMiItimOvadvdLXEAivM3e75ro0,5584
@@ -116,7 +116,7 @@ dstack/_internal/core/backends/vultr/config.py,sha256=TqjT2yhW9rAiDpJ96gBFTM638F
 dstack/_internal/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/models/common.py,sha256=XNfEP_lHAs398oDfSVn8kCX79GIWvl0hDs05Rz1CBic,2579
 dstack/_internal/core/models/config.py,sha256=JJ7rT7dztzTWCY5TkoyxXxTvG5D4IFYhGe7EzwkLOWQ,581
-dstack/_internal/core/models/configurations.py,sha256=lC5Pp_sQUb8GzR2z3QL1vMEMRgC8YwMPQlgpKt0z9dM,14726
+dstack/_internal/core/models/configurations.py,sha256=uIasiOLgU6Xy9Tuh6Vchx2MWnw5O_A4QcrF3aBVrkR4,14770
 dstack/_internal/core/models/envs.py,sha256=ls2ahIhEb3LYLX_auH3F46uTRp2zdlMWKEpjmU_V3S4,5017
 dstack/_internal/core/models/fleets.py,sha256=MzJFMpzsuXBE1Fxlbf8dtsqes5qWMCMRSUWGoWIcRnA,10772
 dstack/_internal/core/models/gateways.py,sha256=n9lPq6X0n2a7fF-CseiBVDDwasJsU3VQumTetMsfZcc,3465
@@ -125,10 +125,10 @@ dstack/_internal/core/models/logs.py,sha256=Lsmtd_NrnChMjBJahUZpFb1j8Xobix9FHWf1
 dstack/_internal/core/models/metrics.py,sha256=Xb8hCXUL-ncQ3PMsErIUAJTe9gwh5jyrQ4UQoZbibsc,269
 dstack/_internal/core/models/placement.py,sha256=WJVq5ENJykyRarQzL2EeYQag_9_jV7VSAtR_xoFvPVM,720
 dstack/_internal/core/models/pools.py,sha256=SmxNQsSYolUMkraLi0BvEXDacuhp1cvbGQn33tyhcc8,1244
-dstack/_internal/core/models/profiles.py,sha256=Y0qsIV0DvvI-YxtcMLqmsrqulGKzJsE_0SdWU2Wusv8,8208
+dstack/_internal/core/models/profiles.py,sha256=WO0hHc-mnjd30k5iLey2PURtjpHOA-J0JQYz6rYu-2Y,9488
 dstack/_internal/core/models/projects.py,sha256=hY3rhLWkuDRsavAdNvQaK6IhnTr7yBHTmNwy9k7zjVE,643
 dstack/_internal/core/models/resources.py,sha256=AzBqGEZLHmg-HpjQtcb_W4pJQRxRtDyeuiHO4berdBA,12408
-dstack/_internal/core/models/runs.py,sha256=ziWnqFRL1hORWJ10evwxLySAe9umiP7kTc3IUvEniSc,17940
+dstack/_internal/core/models/runs.py,sha256=8DW9q6k-6z9dc_Zw5Ulnd-GsAOgbf9iK1DGxkrc4rTQ,18181
 dstack/_internal/core/models/secrets.py,sha256=OlnBI8ESBnpjSqB0-Vr3z8JcqB2Ydfiwo8IJUuM5jAc,219
 dstack/_internal/core/models/server.py,sha256=Hkc1v2s3KOiwslsWVmhUOAzcSeREoG-HD1SzSX9WUGg,152
 dstack/_internal/core/models/services.py,sha256=2Hpi7j0Q1shaf_0wd0C0044AJAmuYi-D3qx3PH849oI,3076
@@ -147,7 +147,7 @@ dstack/_internal/core/models/backends/kubernetes.py,sha256=v0buKTfvcrET1QNgAQatj
 dstack/_internal/core/models/backends/lambdalabs.py,sha256=PrcdI4qU2uJOb5G1PyrInNJF1TFYBiZNF8Jgx0wYAFI,1111
 dstack/_internal/core/models/backends/nebius.py,sha256=CR91JIPJHQsBKju51cr7vgnsP-UiGcNDtUKAyX1Kz84,1306
 dstack/_internal/core/models/backends/oci.py,sha256=8vbnkaCXImnkpAIW6c68xsH7sRYZLMpFJiaIdKRIO-g,2893
-dstack/_internal/core/models/backends/runpod.py,sha256=462H4cbO5An2AmaUTw-PiyZ9OKHFsDw0kminvBmHF1g,982
+dstack/_internal/core/models/backends/runpod.py,sha256=lYiqVALScjR7FqPpKaP5fQSlkJuf__1i_4Y5GIP-Gkg,1061
 dstack/_internal/core/models/backends/tensordock.py,sha256=hoD50FTSPdE-tEVFh-AMIW8Jq4g1AVhdDPVMjH8rhW4,1269
 dstack/_internal/core/models/backends/vastai.py,sha256=Q3Y5HdtNrUuVCMBQMDMBb0dM2u--YYsUnt07hWJuvOQ,1111
 dstack/_internal/core/models/backends/vultr.py,sha256=uebitIgmcZ_Vy0Y7pMdEuuh_Ym5CTMk-pAtcOVlNx-Y,1026
@@ -221,23 +221,24 @@ dstack/_internal/proxy/lib/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 dstack/_internal/proxy/lib/testing/auth.py,sha256=-firWTnis9Eogoi58BURv1S-te4Hf9x1Q_aYLZMDll4,465
 dstack/_internal/proxy/lib/testing/common.py,sha256=5e2VYboMqjBnUxkvidaWLoQ-uaBGh_hnURb_VJc38q4,1518
 dstack/_internal/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/app.py,sha256=pPto7JaBEkwvPF2FR8sqDzwIyMovI_85xk6WXfnnCYU,11317
+dstack/_internal/server/app.py,sha256=WoiFacXv8pfksEGhOQ1Urj7pZzLZKDD95vgxit5V1Ww,11581
 dstack/_internal/server/db.py,sha256=WjuqmjG3QAZmSMCeUaJ_ynbowlHuNAvYCZO649cTPHc,3210
 dstack/_internal/server/deps.py,sha256=31e8SU_ogPJWHIDLkgl7cuC_5V91xbJoLyAj17VanfM,670
 dstack/_internal/server/main.py,sha256=kztKhCYNoHSDyJJQScWfZXE0naNleJOCQULW6dd8SGw,109
-dstack/_internal/server/models.py,sha256=VKh_lfgGUlc0Xy_CGo02IDgwPRp3membAJg07jm27j4,27405
-dstack/_internal/server/settings.py,sha256=4Vazy6qel63ToJzQSjlOHAt_5UpM99inAQYEINUgUAw,3170
-dstack/_internal/server/background/__init__.py,sha256=jNDwTgQwKhK1NYl0RJaKU4mwKIMOSIxPTKVyhizU1PM,3436
+dstack/_internal/server/models.py,sha256=skqZ0hvM3nB_rAeTVqsu2FPuDBy0CqyHLQnuoIXZ5wg,27770
+dstack/_internal/server/settings.py,sha256=1iqXWgvvsr19jXX1javGdPj6UhOfOHuuXSXgmGtjO2A,3335
+dstack/_internal/server/background/__init__.py,sha256=8kTbhEHCeXTibsOlHY1HwqIO6gGb4q8fUa2fcDrah1c,3893
 dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
 dstack/_internal/server/background/tasks/process_fleets.py,sha256=lKXUvN_b7DNjD3psHzyCt_JYsTxPFuQ86iXi8fj8GkM,3202
 dstack/_internal/server/background/tasks/process_gateways.py,sha256=KN5n9r0cKizJ3oGpeLsv3euZKZOMf-02X0Sv09GE8N8,7141
 dstack/_internal/server/background/tasks/process_instances.py,sha256=xRV0tJbSPean2E09opTYLUH2Z4Xpm1Lw-o-LqPcecqc,37046
 dstack/_internal/server/background/tasks/process_metrics.py,sha256=BPv0S7pmMX91vObZiW51AJiNR0ULIj-OQlo5n97KISY,5328
-dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=DZctiMF2TXKbYeygoz--ZiC9MXVL1sQJgq0oqRIc7hg,3858
-dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=8YH4hQXvkSxzrWz9wW0lYoMZmNgC3iUHBWR-tG_LCZI,29829
-dstack/_internal/server/background/tasks/process_runs.py,sha256=JWJqMzXXqvwQSxyKs4bKH2aBP8YQbiyO4tuwyMPVduk,16673
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=xv1aW2umlOdbdZn4TOvlhl6td3Uam378yIwosfSO9ks,27022
+dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=rLlSGEjSrJXcXHDDr_2mBXqSYmYz-4S2x4qz-iNxryo,3931
+dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=Z3eMPv1gYaYzfLOLPu5Btv-RnF4HcxfxQbQhQ6Cg3pk,5089
+dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=ABocGGQOOlWi-pjqk2sZbLKW8LvVhKlJfrmRuneU9z0,31978
+dstack/_internal/server/background/tasks/process_runs.py,sha256=qYPclhUHAXZd7VHl2Dz3ET8YLsdkvPLTcDKOVrSIaoQ,16735
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=qL587QyBCo6DjdYnWH5F15HbmbgY0HxazllPcJLsLrk,27211
 dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=0Z3Q409RwSxOL_pgK8JktBthjtESEUH3ahwTLsTdYPk,3800
 dstack/_internal/server/background/tasks/process_volumes.py,sha256=YafZml-b-bD7M6S0UmY-rLV-pIIX-SdzuyaErqxrnyk,4939
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -266,6 +267,7 @@ dstack/_internal/server/migrations/versions/555138b1f77f_change_instancemodel_fo
 dstack/_internal/server/migrations/versions/58aa5162dcc3_add_gatewaymodel_configuration.py,sha256=pT07eQypORZtWp6K9hwP-wq_K_Tx4EYeKJ9FghIHKyI,834
 dstack/_internal/server/migrations/versions/5ad8debc8fe6_fixes_for_psql.py,sha256=OoZKEtzkvLPb5v0CHeM_bhJn_I5ZxKtjXk7S1_I-wJI,9429
 dstack/_internal/server/migrations/versions/5ec538b70e71_replace_instansestatus.py,sha256=8mFsp2g5pmQm--jjBEgOiuYELS9_Lah66nbURd7sHCo,807
+dstack/_internal/server/migrations/versions/60e444118b6d_add_jobprometheusmetrics.py,sha256=PKqFqe6x6bkO8hrv73fes4uyBpzsemqwp3c-i5XzaS8,1195
 dstack/_internal/server/migrations/versions/63c3f19cb184_add_jobterminationreason_inactivity_.py,sha256=UyNT3d8Osa2MpuODQRdmmbHlt-qT-wHNNOUTzUEcLLI,2512
 dstack/_internal/server/migrations/versions/686fb8341ea5_add_user_emails.py,sha256=0FlRf5Mis9ZIc7X6M9yqPtyFqR2SKCTUFfZeDbqAXBU,809
 dstack/_internal/server/migrations/versions/710e5b3fac8f_add_encryption.py,sha256=nBURp4A6TpT13H1ccH4WyzkU2GGy3uDGKCNG88cAciw,1827
@@ -276,6 +278,7 @@ dstack/_internal/server/migrations/versions/866ec1d67184_replace_retrypolicy_lim
 dstack/_internal/server/migrations/versions/91a12fff6c76_add_repocredsmodel.py,sha256=aN4X-SLhgFEr7UtHkBf2PbiB9QxfL61N9MysThaOw_w,1463
 dstack/_internal/server/migrations/versions/91ac5e543037_extend_repos_creds_column.py,sha256=PjMznHl1fc2ZDCWwcbPbrbq-rcSWRMnomre4Mg7wmr4,873
 dstack/_internal/server/migrations/versions/98cd9c8b5927_add_volumemodel.py,sha256=s77lNvH24LRj-IR5RHI2_s61GV_2SOdsPHR4RLT9MBo,2619
+dstack/_internal/server/migrations/versions/98d1b92988bc_add_jobterminationreason_terminated_due_.py,sha256=6E-T41R61zaHmMzHzzmDVnTW4SwB9NGbAKzmUhHVYsg,4626
 dstack/_internal/server/migrations/versions/99b4c8c954ea_add_termination_reason_message.py,sha256=Bf4IorZ-REe-LGz9F_LLHGLo6r4fpvuBnO8VZmh5YKY,2271
 dstack/_internal/server/migrations/versions/9eea6af28e10_added_fail_reason_for_instancemodel.py,sha256=n0iWQ3L1DWQqPIT-IRs9TR74psf-ZDjcYvSIfn80etY,1027
 dstack/_internal/server/migrations/versions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -308,9 +311,10 @@ dstack/_internal/server/routers/fleets.py,sha256=Tmw4hlviCBuntQPpp4J8XyQZyoK2-4o
 dstack/_internal/server/routers/gateways.py,sha256=am5HnyWUydgUbEjIdjYBcob8CEIrx4bRtdj9kQhutkA,3149
 dstack/_internal/server/routers/instances.py,sha256=ELxXF4BiS9e4YCKFmBp8U4SsjP8eGmbj9Q4LiLfgsgY,1592
 dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
-dstack/_internal/server/routers/metrics.py,sha256=WEICSvIbs1-8zCgxF9WqgsR_9p0afDX_HjNW4-1LCOk,1581
+dstack/_internal/server/routers/metrics.py,sha256=wyKAD8fvqzjeqBj1WqySHgwx1_f76Wmmu14DQCfhThI,2240
 dstack/_internal/server/routers/pools.py,sha256=3NnwGQVLp5GFMypaXNKFqPjop746EEi4LSPMmx5NKek,4889
 dstack/_internal/server/routers/projects.py,sha256=0R-w_6WXUbNo6fREAexFUQ3RoOJF2D_Iz35elKjym14,2717
+dstack/_internal/server/routers/prometheus.py,sha256=yKIe07ctzjaW73lc-u3h47Jg-XOnZzq1MfHSkjH5IhU,1229
 dstack/_internal/server/routers/repos.py,sha256=NvVYnkoc_GHwiZ1_Z5xb76ml0RE8QnnpnuBzeUTB7n0,3456
 dstack/_internal/server/routers/runs.py,sha256=WVA3KwRU3KKgaH0FwMUuBgVh_-OhGYJzHzfebQx76jU,7057
 dstack/_internal/server/routers/secrets.py,sha256=50_qJCTYRpnGSlLyS93gqoV17wWewOVmM65PcG1bT_Y,856
@@ -333,24 +337,25 @@ dstack/_internal/server/schemas/secrets.py,sha256=mfqLSM7PqxVQ-GIWB6RfPRUOvSvvaR
 dstack/_internal/server/schemas/users.py,sha256=FuDqwRVe3mOmv497vOZKjI0a_d4Wt2g4ZiCJcyfHEKA,495
 dstack/_internal/server/schemas/volumes.py,sha256=9iwaQLMhA6aj9XmtdU_9jWVhpzNOtFbDByAe-WYfYh0,673
 dstack/_internal/server/security/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/security/permissions.py,sha256=K756I8fnBLBrq4PZwz1ttt74HI0UWBSmjaIfMHN5d1s,4425
+dstack/_internal/server/security/permissions.py,sha256=ZIUIP5HGk1KG7vGfpZCxP4LsMuKHAz64iOUsAC7FebY,4425
 dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/config.py,sha256=4X9KCh7Gta3WrB735YuVt7aNlH5m7G-ULRBSwHatXOo,29955
+dstack/_internal/server/services/config.py,sha256=vy0zXfZ60AMrdF10ntiyp6RX5VzAjZBBOgy-HNPiaMU,30369
 dstack/_internal/server/services/docker.py,sha256=3EcYPiVsrNBGDQYOb60QJ241mzTT6lJROYQXIwt-8dk,5351
-dstack/_internal/server/services/fleets.py,sha256=h5LL8FGOZMOl0kzLycP36U6k3SWl_bSnSn7oFg8ubQY,28980
+dstack/_internal/server/services/fleets.py,sha256=ny25DmvWzDynq-FQshzqc-TwhyUmQS-N8loUPQK9MTE,29040
 dstack/_internal/server/services/locking.py,sha256=UV5kc-BgROBuNDpBrp8jkcf-fHPOpHLc8JufL-8mbN8,2453
 dstack/_internal/server/services/logging.py,sha256=Nu1628kW2hqB__N0Eyr07wGWjVWxfyJnczonTJ72kSM,417
-dstack/_internal/server/services/metrics.py,sha256=_YmAQ_oCzzklgk_O5iSJDZJhRk9rBqx8REkPcRQUTAo,3490
+dstack/_internal/server/services/metrics.py,sha256=1f_JV4PUqJvA1x5Bav7XeKAkCD9e3C5fc5exh3jjErw,5706
 dstack/_internal/server/services/offers.py,sha256=GIaICztEu4zbcpIGHvNsfMVYmPvoYg5zcQ0apHNdgBw,6954
 dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
 dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBiq7lj41LiAP1PfoEs,1773
 dstack/_internal/server/services/pools.py,sha256=-lF9jrAosNpRxRG3PnbzLWgwPFidGFccjF7tgKyC67A,28911
 dstack/_internal/server/services/projects.py,sha256=HWOnFOC6LmvbjDzRKADv7tXKIDWthbO11zrsHU7vkew,14709
+dstack/_internal/server/services/prometheus.py,sha256=TeVLbv5v_cv_mq3W47WINlIQgK_cPf-O6JysOXPWiFo,3456
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
-dstack/_internal/server/services/runs.py,sha256=3m9oYKq08QQF1GlaOa6_2nYp9SMYoqvfjVnO3-fpTGY,36045
+dstack/_internal/server/services/runs.py,sha256=GhCRgWpMqmM9o7plNA9uQmF4ce2mT_fZcFAZKpD9mZ8,37028
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
 dstack/_internal/server/services/users.py,sha256=L-exfxHdhj3TKX-gSjezHrYK6tnrt5qsQs-zZng1tUI,7123
-dstack/_internal/server/services/volumes.py,sha256=UX-fqC3c240aG6EbdtVsXrJQbkF0O06IFZNtg6bA_tE,14551
+dstack/_internal/server/services/volumes.py,sha256=jBB6A0gdn9BLRycikLJeUkOWDmQaAhbyQJokd129zf0,14612
 dstack/_internal/server/services/backends/__init__.py,sha256=Q8Ju0Shf2RAlIdi1CapJNUKOdUAFeRnxhNYti6GZiww,14923
 dstack/_internal/server/services/backends/handlers.py,sha256=j-MhBxrpdepoDG7f2tApjFnE23RVO5I15-hxHyOWnew,3251
 dstack/_internal/server/services/backends/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -364,7 +369,7 @@ dstack/_internal/server/services/backends/configurators/kubernetes.py,sha256=HtZ
 dstack/_internal/server/services/backends/configurators/lambdalabs.py,sha256=yVFOnKTplxl-rGDUVUDUG5R5sl1qmxQwImdnaSoNqf0,3325
 dstack/_internal/server/services/backends/configurators/nebius.py,sha256=tPch1yC4Tftli-rqt36I4HfYjfvwYRP_irrBJFHdhHY,3240
 dstack/_internal/server/services/backends/configurators/oci.py,sha256=4gImMOmrRuiwVRF-UFagKSpxQlEhaP1aHOsxp17hm8I,7444
-dstack/_internal/server/services/backends/configurators/runpod.py,sha256=By74Rqk-pK8o21CZe5V8IMauQvSKs2Y3W20LxecSnmc,3261
+dstack/_internal/server/services/backends/configurators/runpod.py,sha256=SXOWYu1DD17gUKHGJm5fxZd5m-VUj5i7sWpMlTxxSrs,2733
 dstack/_internal/server/services/backends/configurators/tensordock.py,sha256=GsTy96c59AMULHxNH_0e6BjvdsBtpS9IH8BO6Fgwl5Y,3267
 dstack/_internal/server/services/backends/configurators/vastai.py,sha256=i--Iq5a-1pxo58z1fCCYP3O3Vz_bJoVcA9eXTxi-YZE,3036
 dstack/_internal/server/services/backends/configurators/vultr.py,sha256=T2nDVLkWy9jRzbIjNdA1YDwvL-vWEfjbMaC6qa7IHyY,3018
@@ -373,13 +378,13 @@ dstack/_internal/server/services/encryption/keys/__init__.py,sha256=47DEQpj8HBSa
 dstack/_internal/server/services/encryption/keys/aes.py,sha256=2He1p2_Rg6hnCeLIGJo-Yfdsij7so_338oY49RXuL3Q,2276
 dstack/_internal/server/services/encryption/keys/base.py,sha256=mqumJiidoexUPoqxhQG6J_SpC1WGYwkdjKm1MUWnXo8,352
 dstack/_internal/server/services/encryption/keys/identity.py,sha256=ryb_YSV6u4c7W1OsVfEpzJvZCrR4zZYlzLw_GpjpD2Q,741
-dstack/_internal/server/services/gateways/__init__.py,sha256=ceTzXRWu-pVghze9New465Kn61POTnvpI3RtVIjLgKE,20660
+dstack/_internal/server/services/gateways/__init__.py,sha256=qs1TldKYvGKtWELFnfot7ocOO_V3ETeyjJAQI1nwWfY,20722
 dstack/_internal/server/services/gateways/client.py,sha256=1PH9gUjRkHegp37B27pjBVoPFk64KjxcE30eEPBTLM0,7321
 dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBWeyj57nLPcLPNm316zu3jMyeWjA,5625
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
 dstack/_internal/server/services/jobs/__init__.py,sha256=euq5U73vEGWxDmNASHjnZpWRc5Kp2Dpgy_opMwUDeeU,25444
 dstack/_internal/server/services/jobs/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/jobs/configurators/base.py,sha256=I21dp1P9XC49NNJL_bfihwXfhFd6G-yQXYFYk-EJPEI,10306
+dstack/_internal/server/services/jobs/configurators/base.py,sha256=C733nLS-3OEm9UpTz14dC0-m6YSFfJjn0YCGxNWCie4,10531
 dstack/_internal/server/services/jobs/configurators/dev.py,sha256=_QOExS0ifCZMp4hLYaCuyb-ASQgAegpDguGWNdF_TUA,1953
 dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YFSMuGqjOwYTLMV4FuIM5lCMDFjS0l0CoLI,929
 dstack/_internal/server/services/jobs/configurators/task.py,sha256=0-B3oO-61Eq4-mmlLmqJPliFKHhvvIV0tqc12slcQuA,1436
@@ -400,7 +405,7 @@ dstack/_internal/server/services/proxy/routers/service_proxy.py,sha256=5oB-SX8f_
 dstack/_internal/server/services/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=4JrSxHqhBYqU1oENii89Db-bzkFWExYrOy-0mNEhWBs,4879
 dstack/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/runner/client.py,sha256=lQBQoUIMtKviSSQV9rx8VKfWXlRswGJOEI0TF7QV_oA,14961
+dstack/_internal/server/services/runner/client.py,sha256=O6bZkSZJ_eqO0V8HU0-YAa46tbNuRHAy7ncMgcSh6d4,15413
 dstack/_internal/server/services/runner/ssh.py,sha256=H-X0015ZPwYq5tc31ytFF1uNaUAr9itAsABI2oPJWrk,5017
 dstack/_internal/server/services/services/__init__.py,sha256=-kbk6Qb0mGImhNtYhG7JRsmyc0rKGl0A4rRuN1Wq2bA,10617
 dstack/_internal/server/services/services/autoscalers.py,sha256=0o_w9La-ex_P3VKG88w_XN3hkLkzryv5l1cH3pkZyAE,4315
@@ -430,10 +435,10 @@ dstack/_internal/server/statics/e467d7d60aae81ab198b.svg,sha256=_XHc9mfQZgGkcy4h
 dstack/_internal/server/statics/eb9b344b73818fe2b71a.png,sha256=2H14eOCQRyZhFGJ1Kn2LH1j70kTF1Qop4vH-tiKqyPI,85
 dstack/_internal/server/statics/f517dd626eb964120de0.png,sha256=4QQuNa8SqmcZ67HK6739OHCyjnAJseU1bkcn454KRQs,159
 dstack/_internal/server/statics/f958aecddee5d8e3222c.png,sha256=8CoZkVNgRfOAe62X1dU-AZDvwh_nESKaQblEmaX2Xrs,87
-dstack/_internal/server/statics/index.html,sha256=3BqqXx8k6XRP4kYKcj-4h0EnhHTA6HcXcmijuEzqseQ,10468
-dstack/_internal/server/statics/main-7510e71dfa9749a4e70e.css,sha256=crBuXqMqHxjUBz4ESOQLtRzot_irn6zYrZLf8Uj7gSk,1296121
-dstack/_internal/server/statics/main-fe8fd9db55df8d10e648.js,sha256=DYwOYAwdEz8wUbv7GVGuDFwC2zAZg5qKEwKI1_R5dlE,6192701
-dstack/_internal/server/statics/main-fe8fd9db55df8d10e648.js.map,sha256=K0gwTM9IZBYSuRFFPDWOSior62dC40TbdBH6D2n6bos,8122346
+dstack/_internal/server/statics/index.html,sha256=RfIoWz_lEtf_5hLEfRd9aSlDOG9njajkOlo1k4JNsQI,10468
+dstack/_internal/server/statics/main-4eb116b97819badd1e2c.js,sha256=I3ITWBhxKLM5EHGr0RT_2ldQMMP4lLjVY_rlP6PeCzE,6201517
+dstack/_internal/server/statics/main-4eb116b97819badd1e2c.js.map,sha256=R8hT6YsdRWW1yER9NNwu11tb-YU94BF2LprPkMJcCKo,8138204
+dstack/_internal/server/statics/main-da9f8c06a69c20dac23e.css,sha256=2ObS4Rg6yWkbk7nQqw-lnLVdYC3L8X5N7xZMKcavMWg,1296312
 dstack/_internal/server/statics/manifest.json,sha256=430w2BoWVmYYVr14lDvUxx-ROPt3VjigzeMqfLeiSCM,340
 dstack/_internal/server/statics/robots.txt,sha256=kNJLw79pisHhc3OVAimMzKcq3x9WT6sF9IS4xI0crdI,67
 dstack/_internal/server/statics/assets/android-chrome-144x144.png,sha256=tB3V-95O-VVEoawN5V1XFoMQRSK0I6gthraV8bATGaw,23414
@@ -505,12 +510,13 @@ dstack/_internal/server/statics/assets/mstile-310x310.png,sha256=2URcB82M6AHrJcQ
 dstack/_internal/server/statics/assets/mstile-70x70.png,sha256=Uh-iXux5J5EkcBPRHbvJggmxxL5i7PavtazOv4hflCg,9034
 dstack/_internal/server/statics/assets/yandex-browser-50x50.png,sha256=GGaavXItFc_V5P2G2_MTLFtGDMatissn8YkdBMyGv54,5403
 dstack/_internal/server/statics/assets/yandex-browser-manifest.json,sha256=wQcuGCfkNsSmCXGg2IwxcwdykLSIx7xL3Y_WtmYYLq0,147
+dstack/_internal/server/statics/static/media/entraID.d65d1f3e9486a8e56d24fc07b3230885.svg,sha256=_v0ZwN0Yi2FB4XNlplXWZ-jjOtFIFabQTcC2gyPIRE4,1302
 dstack/_internal/server/statics/static/media/github.1f7102513534c83a9d8d735d2b8c12a2.svg,sha256=mNW9cuKg4s5dYQv8ZMzPsYyLJgxUg4jYBTape-pIy8c,2179
 dstack/_internal/server/statics/static/media/logo.f602feeb138844eda97c8cb641461448.svg,sha256=z1mS8Qo_NNRDObJ41uZfbZA0ZieZUVTnROuJjoiQabE,16373
 dstack/_internal/server/statics/static/media/okta.12f178e6873a1100965f2a4dbd18fcec.svg,sha256=KqFI05gQM135zC1plF1WBRF2F7CyKL7km97WKsZjAHI,319
 dstack/_internal/server/statics/static/media/theme.3994c817bb7dda191c1c9640dee0bf42.svg,sha256=ZxFFBVZWuRLqmWH4zhwGLNtKjOzHj-5MGJRunFAtu1I,561
 dstack/_internal/server/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/testing/common.py,sha256=LKoH9pD9cJoy3JsEbP4nureES_-PBw-wg685vZVx0Rs,29614
+dstack/_internal/server/testing/common.py,sha256=twHKYVCN4iW7eb__wEyyMeIr5Rs2oSzrEqUS8o8-b10,30042
 dstack/_internal/server/testing/conf.py,sha256=-zhujfFjTHNfQDOK-hBck32By11c_kC0OeinB3esQGg,1902
 dstack/_internal/server/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/utils/common.py,sha256=PbjXtqYy1taKXpyG5ys8cIrz9MXqc9dBAsR_9D1brrk,1414
@@ -531,17 +537,17 @@ dstack/_internal/utils/path.py,sha256=YRjmRGe7TQFwsmJBHMjOQhOZo5bieVCVbxOT5J_cbp
 dstack/_internal/utils/random_names.py,sha256=lVyzAmXhaSf2bAuvRFQ_ihLy42in1ydN4gJYsMw4GG8,3537
 dstack/_internal/utils/ssh.py,sha256=ylZfj9ef_IUCvNlghx5uugBnctA6AH35UBFKPYOo6po,11794
 dstack/api/__init__.py,sha256=oiBoHhNCHiA2nDrRYUVA1DBCvj49qHlybBOA8gygMc0,1370
-dstack/api/utils.py,sha256=rS8i4fSpmJd3RQMDoALC5QPak_gnFPdmYDbgdFh864g,4082
+dstack/api/utils.py,sha256=_la3CRk1vgrPGkInPI1SOr33e8Jvh8ziJC32fjvKnaU,4188
 dstack/api/_public/__init__.py,sha256=mieMqlAjTHkbSzA-YJmObGG58f_z0QkbyIfF_UzHEDM,3506
 dstack/api/_public/backends.py,sha256=IBeotauJzlte379LzRM6ByZCl_98QemGZsm3p_WX98k,1056
 dstack/api/_public/pools.py,sha256=wlTai0px6aQNeKyL-6nOil6DCj-GdozqqDGmnTg4ubw,1033
 dstack/api/_public/repos.py,sha256=LYWy1W3Z-2y_D82Jln6YzU2F_asPULGJq1GM9HAAMIc,6173
 dstack/api/_public/resources.py,sha256=lTQM2nb7_QB2RCwmpQlBkYkpiN_aWjgF78wI7Op_RlQ,2947
-dstack/api/_public/runs.py,sha256=wYxIlEKVHr-b1X-gUC1achBq-v1LNRBtP5Vg5mtnhGc,24719
+dstack/api/_public/runs.py,sha256=Hmdpv7UlcTa19lAazmER2aeac20xDD9S5Gwm4OihQpE,24857
 dstack/api/huggingface/__init__.py,sha256=oIrEij3wttLZ1yrywEGvCMd6zswMQrX5pPjrqdSi0UA,2201
 dstack/api/server/__init__.py,sha256=1Sw1bB8NItIFWlL1bdeUN1Tuwu3uaYOJWFrg1c4VULY,5801
 dstack/api/server/_backends.py,sha256=mMePYgFMHAUma3Gycx54VsaxAL_IFhBRcI4H4eWWMg0,1908
-dstack/api/server/_fleets.py,sha256=GfM8IABpDGSi68SV7FOmpO0wXtEFx7kFeK9AfEm_8o0,5008
+dstack/api/server/_fleets.py,sha256=pmU8uv6WgBeoYvSMmPKfXYmrDnBcjXK2S1M3xiA47vQ,5148
 dstack/api/server/_gateways.py,sha256=_OcFOOxUdh4YIA15ZNQQT6mfj0cSKP2jJ_bQXxugS_w,2568
 dstack/api/server/_group.py,sha256=f9a_YvwZ7s1vDTQhOWHLwm3kO84C5R_zXToD4cxAnvY,432
 dstack/api/server/_logs.py,sha256=ng8QvFAIaoVOVChTK6Wuu5BeM6y7gAdx30KEYRsn9xA,500
@@ -549,7 +555,7 @@ dstack/api/server/_metrics.py,sha256=OPb8sLhI_U605sHOPrELgy0_6cNFLJVfpvr-qkEukRM
 dstack/api/server/_pools.py,sha256=PSs8R4FAKZk-jtD4GmATceMyipnasOy2XEg-8lmiN28,2715
 dstack/api/server/_projects.py,sha256=g6kNSU6jer8u7Kaut1I0Ft4wRMLBBCQShJf3fOB63hQ,1440
 dstack/api/server/_repos.py,sha256=gf9fAQw3rzqfPGq-uT8I2Ju_Zn6G4aWTKis8gb8Polc,1885
-dstack/api/server/_runs.py,sha256=rhgmP7JM_Rqtd2jbo1V5Fx9xb5A6vRbvIds0FZQ_SC4,7784
+dstack/api/server/_runs.py,sha256=XSC8X6QDC61R7lpcqchjczk_Y5ONLEqwWGVJdjvzugo,8011
 dstack/api/server/_secrets.py,sha256=VqLfrIcmBJtPxNDRkXTG44H5SWoY788YJapScUukvdY,1576
 dstack/api/server/_users.py,sha256=XzhgGKc5Tsr0-xkz3T6rGyWZ1tO7aYNhLux2eE7dAoY,1738
 dstack/api/server/_volumes.py,sha256=jidkfa35JjKCzRUyafC8rhLd0QpCi6WXg4DbmD8TDoA,1790
@@ -623,22 +629,24 @@ tests/_internal/server/background/tasks/test_process_gateways.py,sha256=h7jzDFQp
 tests/_internal/server/background/tasks/test_process_instances.py,sha256=-ATbgOT8XAeqGmPn4XPlSJzxnrm0haAJhX13nWOBOtM,24816
 tests/_internal/server/background/tasks/test_process_metrics.py,sha256=eirq386mVZqX_PqctuX7CxO38Prwy8RH2UesYXo3CnQ,5083
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=shHqZvS8QoNwQe8J29-aHk2X2HqX-gsxcQiZevU8yuY,1528
-tests/_internal/server/background/tasks/test_process_running_jobs.py,sha256=XzbP5WIJtgYSm7xcjPwuw6uAVAUrxVduO2nrjdmkQV4,25542
+tests/_internal/server/background/tasks/test_process_prometheus_metrics.py,sha256=4qZ-nqlqYNJ2zrhD8f_HWxRvENDSCKLpV7xTIgTe_9U,7285
+tests/_internal/server/background/tasks/test_process_running_jobs.py,sha256=Xh2fTsQdlurxQlIFTaMedIUTwzELr-b7zWQ-Gi0dvKc,30440
 tests/_internal/server/background/tasks/test_process_runs.py,sha256=Yv6f2CDZ-ejDGo7uOKyyAGxYLsvqyjNo4Zw1cCQbjgM,14489
 tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=Sf4aDa1ZYw5Sz3xAw8K0hoI-VDesv7Ew3gZdAzQvj_4,24451
 tests/_internal/server/background/tasks/test_process_submitted_volumes.py,sha256=njfRDdDzVtDL8rech_004Wf5z5PpoQVWFufKh1yUyYY,2171
 tests/_internal/server/background/tasks/test_process_terminating_jobs.py,sha256=VZsfP1NVxkcLSNWfEXDFe_vQZUQfuG0B5i-krXKP_IQ,14147
 tests/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/routers/test_backends.py,sha256=d95ZS97zSRHdmmDTxIPzYQWU-hb2_DvlNo2lcF0NNzw,65273
-tests/_internal/server/routers/test_fleets.py,sha256=ap3Jjg2iiwuklosu2_-xx-cJBlM6uZN5CQfGudORPf0,35105
+tests/_internal/server/routers/test_fleets.py,sha256=aiWfsXgJCePVLHCuvaBb99E3uC-cXZ_eqoRBezhzzZo,35201
 tests/_internal/server/routers/test_gateways.py,sha256=bb0pzm7uu1X3V8755b7ZZUHPSLk0Rd6CjoMCdionWVA,23535
 tests/_internal/server/routers/test_instances.py,sha256=BAWSCjXhe4N1px6BvPEcdAKrQHL5CkrHWvuGrzPRapc,9516
 tests/_internal/server/routers/test_logs.py,sha256=NZwyJlgjMOGq4XEx7-VDjTpniYPhZpsbZvB0dTawaog,3989
-tests/_internal/server/routers/test_metrics.py,sha256=NSdb_x_1Nwsqgrf6KXBf257qd29CXcSN30yk_BQx2CE,5923
+tests/_internal/server/routers/test_metrics.py,sha256=dCMG9PFu1NSsU48kZJoImwOTiXQUblpk4m3OwQjAijg,6552
 tests/_internal/server/routers/test_pools.py,sha256=QQ00pqFVUCo1bSfwWcA2UztyI6Fr0bfU1rj_TnEnVG0,25275
 tests/_internal/server/routers/test_projects.py,sha256=Pa_YL9lYhYt06TFmuZFEf9pxc-K3fM9L29ULf4gCELw,23950
+tests/_internal/server/routers/test_prometheus.py,sha256=4UJuEbvzzlhHr82Td5TcK8wXUyiJi2cWe4eLSd0c-dI,11386
 tests/_internal/server/routers/test_repos.py,sha256=4O4mCDlAPh8xJU_XWtwLhFWRkoTxXvZC1N2DLPyz2AI,17225
-tests/_internal/server/routers/test_runs.py,sha256=bwp0SGzD83IyIYg_nXwGOsG9VUt8G8YRYPhIiJTh48k,71547
+tests/_internal/server/routers/test_runs.py,sha256=l5i0hgBs0MYHEdrhDTp7-S7KuUM98ygUf4_u-7RYE1Q,72150
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
 tests/_internal/server/routers/test_users.py,sha256=5QSLvfn9SroGsZoBGmSTEaaqJcdEO8EUUy_YuvLL8ss,12787
 tests/_internal/server/routers/test_volumes.py,sha256=fVv7sVo9POFImal9Mlkaw8CJOAR67S_eYHxIReHebBU,16241
@@ -647,6 +655,7 @@ tests/_internal/server/services/test_config.py,sha256=KfPyYSSsLaQszBAgqYeM2Tt5Yk
 tests/_internal/server/services/test_docker.py,sha256=QsnoVJUnvzcd1Tpw0UtKXtDSCr9nh_jmtVSkOi7J4ug,5628
 tests/_internal/server/services/test_fleets.py,sha256=WqK9Bs3Nfew45n4WunZhvIvCw0VHJ2w5t0-4CakxFgU,8222
 tests/_internal/server/services/test_logs.py,sha256=z1t2rKe-egRLfK78sH6SCMB2ULwh3F8XgaB1np6qb88,29754
+tests/_internal/server/services/test_metrics.py,sha256=Ffd22qvMMXG_h-uOgr3K1_ZPDwGWVMcgOdgvWvUssmE,5634
 tests/_internal/server/services/test_offers.py,sha256=t0nQNjjm-fvIqHIOA_99hokrdTXwpYrgzjaQjqJXs8w,7814
 tests/_internal/server/services/test_pools.py,sha256=aPF4R6_VCeDR4AcjQ0H2J1BKivoVs-JUNQBJZ-IYdPg,7436
 tests/_internal/server/services/test_repos.py,sha256=o2D0xmOPYaiwDq6nNJNtqYMSuSz_xKJ_ZChVOvWIhJY,14289
@@ -678,9 +687,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.18.43.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.18.43.dist-info/METADATA,sha256=3xpGQbB7LyD6cW0q9f9y-VqJEzyi_HlhHbtCenaEdng,17769
-dstack-0.18.43.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.18.43.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.18.43.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.18.43.dist-info/RECORD,,
+dstack-0.18.44.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.18.44.dist-info/METADATA,sha256=0k_MJce8Z7jYZKDI2VVaKLWq_FFneOWj7cSjauehGjo,18232
+dstack-0.18.44.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.18.44.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.18.44.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.18.44.dist-info/RECORD,,