dstack 0.19.0__py3-none-any.whl → 0.19.1__py3-none-any.whl

dstack/_internal/server/background/tasks/process_instances.py

@@ -64,6 +64,7 @@ from dstack._internal.core.models.runs import (
     Retry,
 )
 from dstack._internal.core.services.profiles import get_retry
+from dstack._internal.server import settings as server_settings
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
@@ -529,7 +530,9 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             session=session, fleet_id=instance.fleet_id
         )
 
-    for backend, instance_offer in offers:
+    # Limit number of offers tried to prevent long-running processing
+    # in case all offers fail.
+    for backend, instance_offer in offers[: server_settings.MAX_OFFERS_TRIED]:
         if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT:
             continue
         compute = backend.compute()
@@ -578,8 +581,13 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
                 extra={"instance_name": instance.name},
             )
             continue
-        except NotImplementedError:
-            # skip a backend without create_instance support, continue with next backend and offer
+        except Exception:
+            logger.exception(
+                "Got exception when launching %s in %s/%s",
+                instance_offer.instance.name,
+                instance_offer.backend.value,
+                instance_offer.region,
+            )
             continue
 
         instance.status = InstanceStatus.PROVISIONING
@@ -607,10 +615,11 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
 
     if not should_retry:
         instance.status = InstanceStatus.TERMINATED
-        instance.termination_reason = "No offers found"
+        instance.termination_reason = "All offers failed" if offers else "No offers found"
         logger.info(
-            "No offers found. Terminated instance %s",
+            "Terminated instance %s: %s",
             instance.name,
+            instance.termination_reason,
             extra={
                 "instance_name": instance.name,
                 "instance_status": InstanceStatus.TERMINATED.value,

dstack/_internal/server/routers/prometheus.py

@@ -6,8 +6,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.server import settings
 from dstack._internal.server.db import get_session
-from dstack._internal.server.deps import Project
-from dstack._internal.server.models import ProjectModel
 from dstack._internal.server.services import prometheus
 from dstack._internal.server.utils.routers import error_not_found
 
@@ -24,13 +22,3 @@ async def get_prometheus_metrics(
     if not settings.ENABLE_PROMETHEUS_METRICS:
         raise error_not_found()
     return await prometheus.get_metrics(session=session)
-
-
-@router.get("/metrics/project/{project_name}", deprecated=True)
-async def get_project_prometheus_metrics(
-    session: Annotated[AsyncSession, Depends(get_session)],
-    project: Annotated[ProjectModel, Depends(Project())],
-) -> str:
-    if not settings.ENABLE_PROMETHEUS_METRICS:
-        raise error_not_found()
-    return await prometheus.get_project_metrics(session=session, project=project)

dstack/_internal/server/services/jobs/configurators/extensions/cursor.py

@@ -35,7 +35,7 @@ class CursorDesktop:
 
     def get_print_readme_commands(self) -> List[str]:
         return [
-            "echo To open in VS Code Desktop, use link below:",
+            "echo To open in Cursor, use link below:",
             "echo ''",
             f"echo '  cursor://vscode-remote/ssh-remote+{self.run_name}/workflow'",  # TODO use $REPO_DIR
             "echo ''",

dstack/_internal/server/services/prometheus.py

@@ -1,40 +1,38 @@
 import itertools
+from collections import defaultdict
 from collections.abc import Generator, Iterable
 from datetime import timezone
+from typing import ClassVar
+from uuid import UUID
 
 from prometheus_client import Metric
 from prometheus_client.parser import text_string_to_metric_families
 from prometheus_client.samples import Sample
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import aliased, joinedload
 
 from dstack._internal.core.models.instances import InstanceStatus
-from dstack._internal.core.models.runs import JobStatus, RunSpec
+from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus
 from dstack._internal.server.models import (
     InstanceModel,
+    JobMetricsPoint,
     JobModel,
     JobPrometheusMetrics,
     ProjectModel,
     RunModel,
+    UserModel,
 )
 from dstack._internal.server.services.instances import get_instance_offer
 from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
 from dstack._internal.utils.common import get_current_datetime
 
-_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
-_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
-_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
-_JOB_DURATION = "dstack_job_duration_seconds_total"
-_JOB_PRICE = "dstack_job_price_dollars_per_hour"
-_JOB_GPU_COUNT = "dstack_job_gpu_count"
-
 
 async def get_metrics(session: AsyncSession) -> str:
     metrics_iter = itertools.chain(
         await get_instance_metrics(session),
+        await get_run_metrics(session),
         await get_job_metrics(session),
-        await get_job_gpu_metrics(session),
     )
     return "\n".join(_render_metrics(metrics_iter)) + "\n"
 
@@ -61,19 +59,7 @@ async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
         )
     )
     instances = res.unique().scalars().all()
-    metrics: dict[str, Metric] = {
-        _INSTANCE_DURATION: Metric(
-            name=_INSTANCE_DURATION,
-            documentation="Total seconds the instance is running",
-            typ="counter",
-        ),
-        _INSTANCE_PRICE: Metric(
-            name=_INSTANCE_PRICE, documentation="Instance price, USD/hour", typ="gauge"
-        ),
-        _INSTANCE_GPU_COUNT: Metric(
-            name=_INSTANCE_GPU_COUNT, documentation="Instance GPU count", typ="gauge"
-        ),
-    }
+    metrics = _InstanceMetrics()
     now = get_current_datetime()
     for instance in instances:
         fleet = instance.fleet
@@ -94,15 +80,36 @@ async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
             "dstack_gpu": gpu,
         }
         duration = (now - instance.created_at.replace(tzinfo=timezone.utc)).total_seconds()
-        metrics[_INSTANCE_DURATION].add_sample(
-            name=_INSTANCE_DURATION, labels=labels, value=duration
-        )
-        metrics[_INSTANCE_PRICE].add_sample(
-            name=_INSTANCE_PRICE, labels=labels, value=instance.price or 0.0
-        )
-        metrics[_INSTANCE_GPU_COUNT].add_sample(
-            name=_INSTANCE_GPU_COUNT, labels=labels, value=gpu_count
-        )
+        metrics.add_sample(_INSTANCE_DURATION, labels, duration)
+        metrics.add_sample(_INSTANCE_PRICE, labels, instance.price or 0.0)
+        metrics.add_sample(_INSTANCE_GPU_COUNT, labels, gpu_count)
+    return metrics.values()
+
+
+async def get_run_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(ProjectModel.name, UserModel.name, RunModel.status, func.count(RunModel.id))
+        .join_from(RunModel, ProjectModel)
+        .join_from(RunModel, UserModel, RunModel.user_id == UserModel.id)
+        .group_by(ProjectModel.name, UserModel.name, RunModel.status)
+        .order_by(ProjectModel.name, UserModel.name, RunModel.status)
+    )
+    projects: dict[str, dict[str, dict[RunStatus, int]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(int))
+    )
+    for project_name, user_name, status, count in res.all():
+        projects[project_name][user_name][status] = count
+    metrics = _RunMetrics()
+    for project_name, users in projects.items():
+        for user_name, statuses in users.items():
+            labels: dict[str, str] = {
+                "dstack_project_name": project_name,
+                "dstack_user_name": user_name,
+            }
+            metrics.add_sample(_RUN_COUNT_TOTAL, labels, sum(statuses.values()))
+            metrics.add_sample(_RUN_COUNT_TERMINATED, labels, statuses[RunStatus.TERMINATED])
+            metrics.add_sample(_RUN_COUNT_FAILED, labels, statuses[RunStatus.FAILED])
+            metrics.add_sample(_RUN_COUNT_DONE, labels, statuses[RunStatus.DONE])
     return metrics.values()
 
 
@@ -127,106 +134,162 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
         )
     )
     jobs = res.scalars().all()
-    metrics: dict[str, Metric] = {
-        _JOB_DURATION: Metric(
-            name=_JOB_DURATION, documentation="Total seconds the job is running", typ="counter"
-        ),
-        _JOB_PRICE: Metric(
-            name=_JOB_PRICE, documentation="Job instance price, USD/hour", typ="gauge"
-        ),
-        _JOB_GPU_COUNT: Metric(name=_JOB_GPU_COUNT, documentation="Job GPU count", typ="gauge"),
-    }
+    job_ids = {job.id for job in jobs}
+    job_metrics_points = await _get_job_metrics_points(session, job_ids)
+    job_prometheus_metrics = await _get_job_prometheus_metrics(session, job_ids)
+
+    metrics = _JobMetrics()
     now = get_current_datetime()
     for job in jobs:
         jpd = get_job_provisioning_data(job)
         if jpd is None:
             continue
         jrd = get_job_runtime_data(job)
-        gpus = jpd.instance_type.resources.gpus
+        resources = jpd.instance_type.resources
         price = jpd.price
         if jrd is not None and jrd.offer is not None:
-            gpus = jrd.offer.instance.resources.gpus
+            resources = jrd.offer.instance.resources
             price = jrd.offer.price
+        gpus = resources.gpus
+        cpus = resources.cpus
         run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
-        labels = _get_job_labels(job)
-        labels["dstack_run_type"] = run_spec.configuration.type
-        labels["dstack_backend"] = jpd.get_base_backend().value
-        labels["dstack_gpu"] = gpus[0].name if gpus else ""
+        labels = {
+            "dstack_project_name": job.project.name,
+            "dstack_user_name": job.run.user.name,
+            "dstack_run_name": job.run_name,
+            "dstack_run_id": str(job.run_id),
+            "dstack_job_name": job.job_name,
+            "dstack_job_id": str(job.id),
+            "dstack_job_num": str(job.job_num),
+            "dstack_replica_num": str(job.replica_num),
+            "dstack_run_type": run_spec.configuration.type,
+            "dstack_backend": jpd.get_base_backend().value,
+            "dstack_gpu": gpus[0].name if gpus else "",
+        }
         duration = (now - job.submitted_at.replace(tzinfo=timezone.utc)).total_seconds()
-        metrics[_JOB_DURATION].add_sample(name=_JOB_DURATION, labels=labels, value=duration)
-        metrics[_JOB_PRICE].add_sample(name=_JOB_PRICE, labels=labels, value=price)
-        metrics[_JOB_GPU_COUNT].add_sample(name=_JOB_GPU_COUNT, labels=labels, value=len(gpus))
+        metrics.add_sample(_JOB_DURATION, labels, duration)
+        metrics.add_sample(_JOB_PRICE, labels, price)
+        metrics.add_sample(_JOB_GPU_COUNT, labels, len(gpus))
+        metrics.add_sample(_JOB_CPU_COUNT, labels, cpus)
+        metrics.add_sample(_JOB_MEMORY_TOTAL, labels, resources.memory_mib * 1024 * 1024)
+        jmp = job_metrics_points.get(job.id)
+        if jmp is not None:
+            metrics.add_sample(_JOB_CPU_TIME, labels, jmp.cpu_usage_micro / 1_000_000)
+            metrics.add_sample(_JOB_MEMORY_USAGE, labels, jmp.memory_usage_bytes)
+            metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
+        jpm = job_prometheus_metrics.get(job.id)
+        if jpm is not None:
+            for metric in text_string_to_metric_families(jpm.text):
+                metrics.add_metric(metric, labels)
     return metrics.values()
 
 
-async def get_job_gpu_metrics(session: AsyncSession) -> Iterable[Metric]:
-    res = await session.execute(
-        select(JobPrometheusMetrics)
-        .join(JobModel)
-        .join(ProjectModel)
-        .where(JobModel.status.in_([JobStatus.RUNNING]))
-        .order_by(ProjectModel.name, JobModel.job_name)
-        .options(
-            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
-            joinedload(JobPrometheusMetrics.job)
-            .joinedload(JobModel.run)
-            .joinedload(RunModel.user),
-        )
-    )
-    metrics_models = res.scalars().all()
-    return _parse_and_enrich_job_gpu_metrics(metrics_models)
+_COUNTER = "counter"
+_GAUGE = "gauge"
 
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_RUN_COUNT_TOTAL = "dstack_run_count_total"
+_RUN_COUNT_TERMINATED = "dstack_run_count_terminated_total"
+_RUN_COUNT_FAILED = "dstack_run_count_failed_total"
+_RUN_COUNT_DONE = "dstack_run_count_done_total"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
+_JOB_CPU_COUNT = "dstack_job_cpu_count"
+_JOB_CPU_TIME = "dstack_job_cpu_time_seconds_total"
+_JOB_MEMORY_TOTAL = "dstack_job_memory_total_bytes"
+_JOB_MEMORY_USAGE = "dstack_job_memory_usage_bytes"
+_JOB_MEMORY_WORKING_SET = "dstack_job_memory_working_set_bytes"
 
-async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> str:
-    res = await session.execute(
-        select(JobPrometheusMetrics)
-        .join(JobModel)
-        .where(
-            JobModel.project_id == project.id,
-            JobModel.status.in_([JobStatus.RUNNING]),
+
+class _Metrics(dict[str, Metric]):
+    metrics: ClassVar[list[tuple[str, str, str]]]
+
+    def __init__(self):
+        super().__init__()
+        for name, typ, documentation in self.metrics:
+            self[name] = Metric(name=name, documentation=documentation, typ=typ)
+
+    def add_sample(self, name: str, labels: dict[str, str], value: float) -> None:
+        # NOTE: Keeps reference to labels.
+        self[name].add_sample(name=name, labels=labels, value=value)
+
+    def add_metric(self, metric: Metric, labels: dict[str, str]) -> None:
+        # NOTE: Modifies and keeps reference to metric.
+        name = metric.name
+        samples = metric.samples
+        stored_metric = self.get(name)
+        if stored_metric is None:
+            stored_metric = metric
+            stored_metric.samples = []
+            self[name] = stored_metric
+        for sample in samples:
+            sample.labels.update(labels)
+            # text_string_to_metric_families "fixes" counter names appending _total,
+            # we rebuild Sample to revert this
+            stored_metric.samples.append(Sample(name, *sample[1:]))
+
+
+class _InstanceMetrics(_Metrics):
+    metrics = [
+        (_INSTANCE_DURATION, _COUNTER, "Total seconds the instance is running"),
+        (_INSTANCE_PRICE, _GAUGE, "Instance price, USD/hour"),
+        (_INSTANCE_GPU_COUNT, _GAUGE, "Instance GPU count"),
+    ]
+
+
+class _RunMetrics(_Metrics):
+    metrics = [
+        (_RUN_COUNT_TOTAL, _COUNTER, "Total runs count"),
+        (_RUN_COUNT_TERMINATED, _COUNTER, "Terminated runs count"),
+        (_RUN_COUNT_FAILED, _COUNTER, "Failed runs count"),
+        (_RUN_COUNT_DONE, _COUNTER, "Done runs count"),
+    ]
+
+
+class _JobMetrics(_Metrics):
+    metrics = [
+        (_JOB_DURATION, _COUNTER, "Total seconds the job is running"),
+        (_JOB_PRICE, _GAUGE, "Job instance price, USD/hour"),
+        (_JOB_GPU_COUNT, _GAUGE, "Job GPU count"),
+        (_JOB_CPU_COUNT, _GAUGE, "Job CPU count"),
+        (_JOB_CPU_TIME, _COUNTER, "Total CPU time consumed by the job, seconds"),
+        (_JOB_MEMORY_TOTAL, _GAUGE, "Total memory allocated for the job, bytes"),
+        (_JOB_MEMORY_USAGE, _GAUGE, "Memory used by the job (including cache), bytes"),
+        (_JOB_MEMORY_WORKING_SET, _GAUGE, "Memory used by the job (not including cache), bytes"),
+    ]
+
+
+async def _get_job_metrics_points(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobMetricsPoint]:
+    subquery = select(
+        JobMetricsPoint,
+        func.row_number()
+        .over(
+            partition_by=JobMetricsPoint.job_id,
+            order_by=JobMetricsPoint.timestamp_micro.desc(),
         )
-        .order_by(JobModel.job_name)
-        .options(
-            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
-            joinedload(JobPrometheusMetrics.job)
-            .joinedload(JobModel.run)
-            .joinedload(RunModel.user),
+        .label("row_number"),
+    ).subquery()
+    res = await session.execute(
+        select(aliased(JobMetricsPoint, subquery)).where(
+            subquery.c.row_number == 1,
+            subquery.c.job_id.in_(job_ids),
         )
     )
-    metrics_models = res.scalars().all()
-    return "\n".join(_render_metrics(_parse_and_enrich_job_gpu_metrics(metrics_models))) + "\n"
-
-
-def _parse_and_enrich_job_gpu_metrics(
-    metrics_models: Iterable[JobPrometheusMetrics],
-) -> Iterable[Metric]:
-    metrics: dict[str, Metric] = {}
-    for metrics_model in metrics_models:
-        for metric in text_string_to_metric_families(metrics_model.text):
-            samples = metric.samples
-            metric.samples = []
-            name = metric.name
-            metric = metrics.setdefault(name, metric)
-            for sample in samples:
-                labels = sample.labels
-                labels.update(_get_job_labels(metrics_model.job))
-                # text_string_to_metric_families "fixes" counter names appending _total,
-                # we rebuild Sample to revert this
-                metric.samples.append(Sample(name, labels, *sample[2:]))
-    return metrics.values()
+    return {p.job_id: p for p in res.scalars().all()}
 
 
-def _get_job_labels(job: JobModel) -> dict[str, str]:
-    return {
-        "dstack_project_name": job.project.name,
-        "dstack_user_name": job.run.user.name,
-        "dstack_run_name": job.run_name,
-        "dstack_run_id": str(job.run_id),
-        "dstack_job_name": job.job_name,
-        "dstack_job_id": str(job.id),
-        "dstack_job_num": str(job.job_num),
-        "dstack_replica_num": str(job.replica_num),
-    }
+async def _get_job_prometheus_metrics(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobPrometheusMetrics]:
+    res = await session.execute(
+        select(JobPrometheusMetrics).where(JobPrometheusMetrics.job_id.in_(job_ids))
+    )
+    return {p.job_id: p for p in res.scalars().all()}
 
 
 def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.0"
+__version__ = "0.19.1"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.19.0.dist-info → dstack-0.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.19.0
+Version: 0.19.1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov

{dstack-0.19.0.dist-info → dstack-0.19.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=1doazpT-su1BmUobtt--UmdBSkqrHjSTXH4z74aok9A,64
+dstack/version.py,sha256=ja4J6HzBpBX3wxm5CKLUUgzAwmmr8naAhq3SBch6VIw,64
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
@@ -242,7 +242,7 @@ dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
 dstack/_internal/server/background/tasks/process_fleets.py,sha256=lKXUvN_b7DNjD3psHzyCt_JYsTxPFuQ86iXi8fj8GkM,3202
 dstack/_internal/server/background/tasks/process_gateways.py,sha256=hoUI1CSqbHt_uMwnzTRAEDl-LBw0wUk_W4xobIbdvRc,7017
-dstack/_internal/server/background/tasks/process_instances.py,sha256=TvpMnK211tXcOPAMF3qngGH9DycwKJeEchoYOtKICrg,37284
+dstack/_internal/server/background/tasks/process_instances.py,sha256=Kc7CbWK4mFOsKwOqp-Pt0ewTsB5OZ5gkPyv9T6TNbpM,37674
 dstack/_internal/server/background/tasks/process_metrics.py,sha256=acySfsacpYbTPV9Yivs-oU37z1S2sUdWhRHdJkfBcCA,5332
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FqGfbzvfILdnPUfxjFPAM1ij2xd2mCDi8qufiBcUMI8,4107
 dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=u8hCXjOOek7VLEsmLy2VnDXFmIwTNjrJwcpWG7a1zW0,5093
@@ -325,7 +325,7 @@ dstack/_internal/server/routers/instances.py,sha256=XOogTC9My2Zv0ck37_PbHKoZI-j4
 dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
 dstack/_internal/server/routers/metrics.py,sha256=VFgWhkOvxVFDLlRM_kXHYFylLcfCD6UjXInvcd7H4dY,2314
 dstack/_internal/server/routers/projects.py,sha256=0R-w_6WXUbNo6fREAexFUQ3RoOJF2D_Iz35elKjym14,2717
-dstack/_internal/server/routers/prometheus.py,sha256=CIz1GoCVnSAevWhTPnbIJKUO-ntz8tvK6Q_2vicDLoo,1246
+dstack/_internal/server/routers/prometheus.py,sha256=OuC17kgKkb2ErxDD5QZ_ZdZft5A8dMIAFlIzQ_04NEo,744
 dstack/_internal/server/routers/repos.py,sha256=P_zLoEQderxhCeHQJwRkrIhVcc0-cpabfyde22bWVRk,3362
 dstack/_internal/server/routers/runs.py,sha256=oPqyIRPwkMjj12M1IdMF2UitatqvljISAXnJAjfEJyQ,5352
 dstack/_internal/server/routers/secrets.py,sha256=50_qJCTYRpnGSlLyS93gqoV17wWewOVmM65PcG1bT_Y,856
@@ -360,7 +360,7 @@ dstack/_internal/server/services/offers.py,sha256=tTld2ZcYdbhzShtMIf1YfTyIADtpN3
 dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
 dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBiq7lj41LiAP1PfoEs,1773
 dstack/_internal/server/services/projects.py,sha256=Y4LEkSvOVUHHP-F2qlrwBR7rFu0CFFhbHmDTKrrNuXE,15071
-dstack/_internal/server/services/prometheus.py,sha256=1hXSsML-xIiQHv2DKVuJTfK0FRAlseF3GESUQUvy3NE,9625
+dstack/_internal/server/services/prometheus.py,sha256=xq5G-Q2BJup9lS2F6__0wUVTs-k1Gr3dYclGzo2WoWo,12474
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
 dstack/_internal/server/services/runs.py,sha256=B2jZtTOxavUHr6WqKMXqgLzB3xWsHTkWKykcvcT2lXI,37245
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
@@ -385,7 +385,7 @@ dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YF
 dstack/_internal/server/services/jobs/configurators/task.py,sha256=0-B3oO-61Eq4-mmlLmqJPliFKHhvvIV0tqc12slcQuA,1436
 dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
-dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=r2siUu__sz86xDR_oK2xODmmzef1FzOS3veekGTS2-s,1645
+dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=00HB1kC_eMlioEW0nZI7Ly78b-RSehySVNhC9pajBP8,1636
 dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
 dstack/_internal/server/services/logs/__init__.py,sha256=NAjO1KeYvuDznN2EkfAaJt9S6Y00fo_dl3ob3WmsdGQ,3088
 dstack/_internal/server/services/logs/aws.py,sha256=949k8t9H9v_-aedDjDWkw8yPVyhZemmsszcDDEL5Tb4,13711
@@ -639,7 +639,7 @@ tests/_internal/server/background/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 tests/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/background/tasks/test_process_fleets.py,sha256=Dl31_TwxoCzYqkVNPWGLsYxmGL2sZfEK3rQXLFyPIz8,2701
 tests/_internal/server/background/tasks/test_process_gateways.py,sha256=lOP4jPXDtadAgYp0aFND_fp5R_X19M58CaOlgnDAEck,5085
-tests/_internal/server/background/tasks/test_process_instances.py,sha256=b7Shke7p3FLPT-qTVDMnwuCK4KqR4kLw9UFfsr0TSVg,24549
+tests/_internal/server/background/tasks/test_process_instances.py,sha256=WC32HvynBuSxwFtAyMTHS4eVzqCnyGufcrIUTEVoozI,27944
 tests/_internal/server/background/tasks/test_process_metrics.py,sha256=z-u4HXJE5EMVH9kwU_POHmvp55ldAvuLpEMkaebBtsg,4976
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=19LYbIMZIIeKAN0b9KOMyS-cHUx0FoOojqQuM8Oeiq4,1620
 tests/_internal/server/background/tasks/test_process_prometheus_metrics.py,sha256=I9DgIJXVGS7UvbFgm4HFnzWiCICBpy72NjDPKU_7WII,7178
@@ -656,7 +656,7 @@ tests/_internal/server/routers/test_instances.py,sha256=78HFMU9Xel8BNZL3TqnuvrKE
 tests/_internal/server/routers/test_logs.py,sha256=NZwyJlgjMOGq4XEx7-VDjTpniYPhZpsbZvB0dTawaog,3989
 tests/_internal/server/routers/test_metrics.py,sha256=xMdDFZW73Zl06QfggjatfwTut37s0soeliJivkCgBks,7620
 tests/_internal/server/routers/test_projects.py,sha256=Z3Ok7onAjUYS4ADvKvN-SwSxYKvlvf4MG5Y8baqQU14,25964
-tests/_internal/server/routers/test_prometheus.py,sha256=L3qxaJQzir2nJbFtRHZMC7Pw6xWBFcv4-0buQUSdek8,18933
+tests/_internal/server/routers/test_prometheus.py,sha256=LqJwWn5ztSLIGnvZgj-sD7BFW-JuePFt6k__ymF5Btw,22711
 tests/_internal/server/routers/test_repos.py,sha256=G4dKuFGd_UrxAHwh_XLl1xCHK_DCsiJcXBsHODw3yJk,16682
 tests/_internal/server/routers/test_runs.py,sha256=q02oBrUcp4JoJOL68jbxlfFxH9B8JO9Bkb7v_Qg-Aug,62984
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
@@ -701,9 +701,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.19.0.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.0.dist-info/METADATA,sha256=xHGGa6FKokKvEFYnt2k7cX2iFkp0YsA9o0-I3nbqagg,18231
-dstack-0.19.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.19.0.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.0.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.19.0.dist-info/RECORD,,
+dstack-0.19.1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.1.dist-info/METADATA,sha256=sj_wcanWBaGU9ecMn6I32zvXuYFniyN_6K6ehbcO3tA,18231
+dstack-0.19.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.19.1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.19.1.dist-info/RECORD,,

tests/_internal/server/background/tasks/test_process_instances.py

@@ -8,7 +8,7 @@ import pytest
 from freezegun import freeze_time
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.errors import BackendError
+from dstack._internal.core.errors import BackendError, ProvisioningError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     Gpu,
@@ -35,6 +35,8 @@ from dstack._internal.server.testing.common import (
     create_repo,
     create_run,
     create_user,
+    get_instance_offer_with_availability,
+    get_job_provisioning_data,
     get_remote_connection_info,
 )
 from dstack._internal.utils.common import get_current_datetime
@@ -557,6 +559,68 @@ class TestCreateInstance:
         assert instance.total_blocks == expected_blocks
         assert instance.busy_blocks == 0
 
+    @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
+    async def test_tries_second_offer_if_first_fails(self, session: AsyncSession, err: Exception):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        aws_mock = Mock()
+        aws_mock.TYPE = BackendType.AWS
+        offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
+        aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        aws_mock.compute.return_value.create_instance.side_effect = err
+        gcp_mock = Mock()
+        gcp_mock.TYPE = BackendType.GCP
+        offer = get_instance_offer_with_availability(backend=BackendType.GCP, price=2.0)
+        gcp_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        gcp_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        gcp_mock.compute.return_value.create_instance.return_value = get_job_provisioning_data(
+            backend=offer.backend, region=offer.region, price=offer.price
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = [aws_mock, gcp_mock]
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.PROVISIONING
+        aws_mock.compute.return_value.create_instance.assert_called_once()
+        assert instance.backend == BackendType.GCP
+
+    @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
+    async def test_fails_if_all_offers_fail(self, session: AsyncSession, err: Exception):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        aws_mock = Mock()
+        aws_mock.TYPE = BackendType.AWS
+        offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
+        aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        aws_mock.compute.return_value.create_instance.side_effect = err
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = [aws_mock]
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.TERMINATED
+        assert instance.termination_reason == "All offers failed"
+
+    async def test_fails_if_no_offers(self, session: AsyncSession):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = []
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.TERMINATED
+        assert instance.termination_reason == "No offers found"
+
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)

tests/_internal/server/routers/test_prometheus.py

@@ -9,14 +9,20 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
-from dstack._internal.core.models.runs import JobProvisioningData, JobRuntimeData, JobStatus
+from dstack._internal.core.models.runs import (
+    JobProvisioningData,
+    JobRuntimeData,
+    JobStatus,
+    RunStatus,
+)
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.models import JobModel, ProjectModel, UserModel
+from dstack._internal.server.models import JobModel, ProjectModel, RunModel, UserModel
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
     create_fleet,
     create_instance,
     create_job,
+    create_job_metrics_point,
     create_job_prometheus_metrics,
     create_project,
     create_repo,
@@ -45,11 +51,21 @@ class TestGetPrometheusMetrics:
     async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
         user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
         offer = get_instance_offer_with_availability(
-            instance_type="test-type", gpu_count=2, gpu_name="V4", price=12
+            instance_type="test-type",
+            cpu_count=32,
+            memory_gib=128,
+            gpu_count=2,
+            gpu_name="V4",
+            price=12,
         )
         project_2 = await _create_project(session, "project-2", user)
         jpd_2_1 = get_job_provisioning_data(
-            backend=BackendType.AWS, gpu_name="T4", gpu_count=2, price=16
+            backend=BackendType.AWS,
+            cpu_count=16,
+            memory_gib=64,
+            gpu_name="T4",
+            gpu_count=2,
+            price=16,
         )
         job_2_1 = await _create_job(
             session=session,
@@ -100,7 +116,41 @@ class TestGetPrometheusMetrics:
                 FIELD_2{gpu="1"} 987169 1395066363010
             """),
         )
-        job_1_2 = await _create_job(session, "run-2", project_1, user, JobStatus.RUNNING)
+        await create_job_metrics_point(
+            session=session,
+            job_model=job_1_1,
+            timestamp=FAKE_NOW - timedelta(seconds=30),
+            cpu_usage_micro=3_500_000,
+            memory_working_set_bytes=3_221_225_472,
+            memory_usage_bytes=4_294_967_296,
+        )
+        # Older, ignored
+        await create_job_metrics_point(
+            session=session,
+            job_model=job_1_1,
+            timestamp=FAKE_NOW - timedelta(seconds=60),
+            cpu_usage_micro=2_000_000,
+            memory_working_set_bytes=1_073_741_824,
+            memory_usage_bytes=2_147_483_648,
+        )
+        jpd_1_2 = get_job_provisioning_data(
+            backend=BackendType.AWS,
+            cpu_count=24,
+            memory_gib=224,
+            gpu_count=3,
+            gpu_name="L4",
+            price=12.5,
+        )
+        job_1_2 = await _create_job(
+            session=session,
+            run_name="run-2",
+            project=project_1,
+            user=user,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=jpd_1_2,
+            submitted_at=FAKE_NOW - timedelta(seconds=150),
+        )
+
         await create_job_prometheus_metrics(
             session=session,
             job=job_1_2,
@@ -124,6 +174,15 @@ class TestGetPrometheusMetrics:
                 FIELD_1{gpu="1"} 20
             """),
         )
+        await _create_run(session, "done", project_1, user, RunStatus.DONE)
+        other_user = await create_user(
+            session=session, name="other-user", global_role=GlobalRole.USER
+        )
+        await add_project_member(
+            session=session, project=project_2, user=other_user, project_role=ProjectRole.USER
+        )
+        await _create_run(session, "failed-1", project_2, other_user, RunStatus.FAILED)
+        await _create_run(session, "failed-2", project_2, other_user, RunStatus.FAILED)
         fleet = await create_fleet(session=session, project=project_1, name="test-fleet")
         instance = await create_instance(
             session=session,
@@ -149,31 +208,73 @@ class TestGetPrometheusMetrics:
             # HELP dstack_instance_gpu_count Instance GPU count
             # TYPE dstack_instance_gpu_count gauge
             dstack_instance_gpu_count{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            # HELP dstack_run_count_total Total runs count
+            # TYPE dstack_run_count_total counter
+            dstack_run_count_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 4.0
+            dstack_run_count_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 2.0
+            dstack_run_count_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 1.0
+            # HELP dstack_run_count_terminated_total Terminated runs count
+            # TYPE dstack_run_count_terminated_total counter
+            dstack_run_count_terminated_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 0.0
+            dstack_run_count_terminated_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 0.0
+            dstack_run_count_terminated_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
+            # HELP dstack_run_count_failed_total Failed runs count
+            # TYPE dstack_run_count_failed_total counter
+            dstack_run_count_failed_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 0.0
+            dstack_run_count_failed_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 2.0
+            dstack_run_count_failed_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
+            # HELP dstack_run_count_done_total Done runs count
+            # TYPE dstack_run_count_done_total counter
+            dstack_run_count_done_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 1.0
+            dstack_run_count_done_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 0.0
+            dstack_run_count_done_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
             # HELP dstack_job_duration_seconds_total Total seconds the job is running
             # TYPE dstack_job_duration_seconds_total counter
             dstack_job_duration_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 120.0
+            dstack_job_duration_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 150.0
             dstack_job_duration_seconds_total{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 100.0
             # HELP dstack_job_price_dollars_per_hour Job instance price, USD/hour
             # TYPE dstack_job_price_dollars_per_hour gauge
             dstack_job_price_dollars_per_hour{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 12.0
+            dstack_job_price_dollars_per_hour{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 12.5
             dstack_job_price_dollars_per_hour{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 16.0
             # HELP dstack_job_gpu_count Job GPU count
             # TYPE dstack_job_gpu_count gauge
             dstack_job_gpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            dstack_job_gpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 3.0
             dstack_job_gpu_count{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 2.0
+            # HELP dstack_job_cpu_count Job CPU count
+            # TYPE dstack_job_cpu_count gauge
+            dstack_job_cpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 32.0
+            dstack_job_cpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 24.0
+            dstack_job_cpu_count{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 16.0
+            # HELP dstack_job_cpu_time_seconds_total Total CPU time consumed by the job, seconds
+            # TYPE dstack_job_cpu_time_seconds_total counter
+            dstack_job_cpu_time_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 3.5
+            # HELP dstack_job_memory_total_bytes Total memory allocated for the job, bytes
+            # TYPE dstack_job_memory_total_bytes gauge
+            dstack_job_memory_total_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 137438953472.0
+            dstack_job_memory_total_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 240518168576.0
+            dstack_job_memory_total_bytes{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 68719476736.0
+            # HELP dstack_job_memory_usage_bytes Memory used by the job (including cache), bytes
+            # TYPE dstack_job_memory_usage_bytes gauge
+            dstack_job_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 4294967296.0
+            # HELP dstack_job_memory_working_set_bytes Memory used by the job (not including cache), bytes
+            # TYPE dstack_job_memory_working_set_bytes gauge
+            dstack_job_memory_working_set_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 3221225472.0
             # HELP FIELD_1 Test field 1
             # TYPE FIELD_1 gauge
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
-            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 100.0
-            FIELD_1{{gpu="1",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 200.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 350.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 1200.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 1600.0
+            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 2400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 100.0
+            FIELD_1{{gpu="1",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 200.0
             # HELP FIELD_2 Test field 2
             # TYPE FIELD_2 counter
-            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
-            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
+            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 337325.0 1395066363000
+            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 987169.0 1395066363010
         """)
 
     async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
@@ -189,110 +290,6 @@ class TestGetPrometheusMetrics:
         assert response.status_code == 404
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
-@pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
-class TestGetPrometheusProjectMetrics:
-    async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
-        user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
-        project = await _create_project(session, "project-1", user)
-        job_1 = await _create_job(session, "run-1", project, user, JobStatus.RUNNING)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_1,
-            text=dedent("""
-                # Comments should be skipped
-
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 350
-                FIELD_1{gpu="1"} 400
-
-                # HELP FIELD_2 Test field 2
-                # TYPE FIELD_2 counter
-                FIELD_2{gpu="0"} 337325 1395066363000
-                FIELD_2{gpu="1"} 987169 1395066363010
-            """),
-        )
-        job_2 = await _create_job(session, "run-2", project, user, JobStatus.RUNNING)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_2,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 1200.0
-                FIELD_1{gpu="1"} 1600.0
-                FIELD_1{gpu="2"} 2400.0
-            """),
-        )
-        # Terminated job, should not appear in the response
-        job_3 = await _create_job(session, "run-3", project, user, JobStatus.TERMINATED)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_3,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 10
-                FIELD_1{gpu="1"} 20
-            """),
-        )
-        another_project = await _create_project(session, "project-2", user)
-        another_project_job = await _create_job(
-            session, "run-4", another_project, user, JobStatus.RUNNING
-        )
-        await create_job_prometheus_metrics(
-            session=session,
-            job=another_project_job,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 100
-                FIELD_1{gpu="1"} 200
-            """),
-        )
-
-        response = await client.get("/metrics/project/project-1")
-
-        assert response.status_code == 200
-        assert response.text == dedent(f"""\
-            # HELP FIELD_1 Test field 1
-            # TYPE FIELD_1 gauge
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
-            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
-            # HELP FIELD_2 Test field 2
-            # TYPE FIELD_2 counter
-            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
-            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
-        """)
-
-    async def test_returns_empty_response_if_no_runs(
-        self, session: AsyncSession, client: AsyncClient
-    ):
-        user = await create_user(session=session, global_role=GlobalRole.USER)
-        await create_project(session=session, owner=user, name="test-project")
-        response = await client.get("/metrics/project/test-project")
-        assert response.status_code == 200
-        assert response.text == "\n"
-
-    async def test_returns_404_if_project_doesnt_exist(self, client: AsyncClient):
-        response = await client.get("/metrics/project/nonexistent")
-        assert response.status_code == 404
-
-    async def test_returns_404_if_not_enabled(
-        self, monkeypatch: pytest.MonkeyPatch, session: AsyncSession, client: AsyncClient
-    ):
-        monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", False)
-        user = await create_user(session=session, global_role=GlobalRole.USER)
-        await create_project(session=session, owner=user, name="test-project")
-        response = await client.get("/metrics/project/test-project")
-        assert response.status_code == 404
-
-
 async def _create_project(session: AsyncSession, name: str, user: UserModel) -> ProjectModel:
     project = await create_project(session=session, owner=user, name=name)
     await add_project_member(
@@ -301,26 +298,46 @@ async def _create_project(session: AsyncSession, name: str, user: UserModel) ->
     return project
 
 
-async def _create_job(
+async def _create_run(
     session: AsyncSession,
     run_name: str,
     project: ProjectModel,
     user: UserModel,
-    status: JobStatus,
-    job_provisioning_data: Optional[JobProvisioningData] = None,
-    job_runtime_data: Optional[JobRuntimeData] = None,
+    status: RunStatus,
     submitted_at: datetime = FAKE_NOW,
-) -> JobModel:
+) -> RunModel:
     repo = await create_repo(session=session, project_id=project.id, repo_name=f"{run_name}-repo")
     configuration = DevEnvironmentConfiguration(ide="vscode")
     run_spec = get_run_spec(run_name=run_name, repo_id=repo.name, configuration=configuration)
-    run = await create_run(
+    return await create_run(
         session=session,
         project=project,
         repo=repo,
         user=user,
         run_name=run_name,
         run_spec=run_spec,
+        status=status,
+        submitted_at=submitted_at,
+    )
+
+
+async def _create_job(
+    session: AsyncSession,
+    run_name: str,
+    project: ProjectModel,
+    user: UserModel,
+    status: JobStatus,
+    job_provisioning_data: Optional[JobProvisioningData] = None,
+    job_runtime_data: Optional[JobRuntimeData] = None,
+    submitted_at: datetime = FAKE_NOW,
+) -> JobModel:
+    run = await _create_run(
+        session=session,
+        run_name=run_name,
+        project=project,
+        user=user,
+        status=RunStatus.SUBMITTED,
+        submitted_at=submitted_at,
     )
     job = await create_job(
         session=session,