dstack 0.18.42__py3-none-any.whl → 0.18.44__py3-none-any.whl

tests/_internal/server/background/tasks/test_process_prometheus_metrics.py

@@ -0,0 +1,189 @@
+from collections.abc import Generator
+from datetime import datetime, timezone
+from unittest.mock import Mock, patch
+
+import pytest
+import pytest_asyncio
+from freezegun import freeze_time
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.runs import JobStatus
+from dstack._internal.core.models.users import GlobalRole, ProjectRole
+from dstack._internal.server.background.tasks.process_prometheus_metrics import (
+    collect_prometheus_metrics,
+    delete_prometheus_metrics,
+)
+from dstack._internal.server.models import JobModel, JobPrometheusMetrics
+from dstack._internal.server.services.projects import add_project_member
+from dstack._internal.server.testing.common import (
+    create_instance,
+    create_job,
+    create_job_prometheus_metrics,
+    create_pool,
+    create_project,
+    create_repo,
+    create_run,
+    create_user,
+    get_job_provisioning_data,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+@pytest.mark.usefixtures("test_db", "image_config_mock")
+class TestCollectPrometheusMetrics:
+    @pytest_asyncio.fixture
+    async def job(self, session: AsyncSession) -> JobModel:
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        project = await create_project(session=session, owner=user)
+        await add_project_member(
+            session=session, project=project, user=user, project_role=ProjectRole.USER
+        )
+        repo = await create_repo(
+            session=session,
+            project_id=project.id,
+        )
+        pool = await create_pool(session=session, project=project)
+        instance = await create_instance(
+            session=session,
+            project=project,
+            pool=pool,
+            status=InstanceStatus.BUSY,
+        )
+        run = await create_run(
+            session=session,
+            project=project,
+            repo=repo,
+            user=user,
+        )
+        job = await create_job(
+            session=session,
+            run=run,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=get_job_provisioning_data(),
+            instance_assigned=True,
+            instance=instance,
+        )
+        return job
+
+    @pytest.fixture
+    def ssh_tunnel_mock(self) -> Generator[Mock, None, None]:
+        with patch("dstack._internal.server.services.runner.ssh.SSHTunnel") as SSHTunnelMock:
+            yield SSHTunnelMock
+
+    @pytest.fixture
+    def shim_client_mock(self) -> Generator[Mock, None, None]:
+        with patch("dstack._internal.server.services.runner.client.ShimClient") as ShimClientMock:
+            yield ShimClientMock.return_value
+
+    @freeze_time(datetime(2023, 1, 2, 3, 5, 20, tzinfo=timezone.utc))
+    async def test_inserts_new_record(
+        self, session: AsyncSession, job: JobModel, ssh_tunnel_mock: Mock, shim_client_mock: Mock
+    ):
+        shim_client_mock.get_task_metrics.return_value = "# prom response"
+
+        await collect_prometheus_metrics()
+
+        ssh_tunnel_mock.assert_called_once()
+        shim_client_mock.get_task_metrics.assert_called_once()
+        res = await session.execute(
+            select(JobPrometheusMetrics).where(JobPrometheusMetrics.job_id == job.id)
+        )
+        metrics = res.scalar_one()
+        assert metrics.text == "# prom response"
+        assert metrics.collected_at == datetime(2023, 1, 2, 3, 5, 20)
+
+    @freeze_time(datetime(2023, 1, 2, 3, 5, 20, tzinfo=timezone.utc))
+    async def test_updates_record(
+        self, session: AsyncSession, job: JobModel, ssh_tunnel_mock: Mock, shim_client_mock: Mock
+    ):
+        metrics = await create_job_prometheus_metrics(
+            session=session,
+            job=job,
+            collected_at=datetime(2023, 1, 2, 3, 5, 0),
+            text="# prom old response",
+        )
+        shim_client_mock.get_task_metrics.return_value = "# prom new response"
+
+        await collect_prometheus_metrics()
+
+        ssh_tunnel_mock.assert_called_once()
+        shim_client_mock.get_task_metrics.assert_called_once()
+        res = await session.execute(
+            select(JobPrometheusMetrics)
+            .where(JobPrometheusMetrics.job_id == job.id)
+            .execution_options(populate_existing=True)
+        )
+        metrics = res.scalar_one()
+        assert metrics.text == "# prom new response"
+        assert metrics.collected_at == datetime(2023, 1, 2, 3, 5, 20)
+
+    @freeze_time(datetime(2023, 1, 2, 3, 5, 20, tzinfo=timezone.utc))
+    async def test_skips_recently_updated(
+        self, session: AsyncSession, job: JobModel, ssh_tunnel_mock: Mock, shim_client_mock: Mock
+    ):
+        metrics = await create_job_prometheus_metrics(
+            session=session,
+            job=job,
+            collected_at=datetime(2023, 1, 2, 3, 5, 15),
+            text="# prom old response",
+        )
+        shim_client_mock.get_task_metrics.return_value = "# prom new response"
+
+        await collect_prometheus_metrics()
+
+        ssh_tunnel_mock.assert_not_called()
+        shim_client_mock.get_task_metrics.assert_not_called()
+        res = await session.execute(
+            select(JobPrometheusMetrics)
+            .where(JobPrometheusMetrics.job_id == job.id)
+            .execution_options(populate_existing=True)
+        )
+        metrics = res.scalar_one()
+        assert metrics.text == "# prom old response"
+        assert metrics.collected_at == datetime(2023, 1, 2, 3, 5, 15)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+@pytest.mark.usefixtures("test_db", "image_config_mock")
+class TestDeletePrometheusMetrics:
+    @freeze_time(datetime(2023, 1, 2, 3, 5, 20, tzinfo=timezone.utc))
+    async def test_deletes_old_metrics(self, session: AsyncSession):
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        project = await create_project(session=session, owner=user)
+        await add_project_member(
+            session=session, project=project, user=user, project_role=ProjectRole.USER
+        )
+        repo = await create_repo(session=session, project_id=project.id)
+        run_1 = await create_run(
+            session=session, project=project, repo=repo, user=user, run_name="run-1"
+        )
+        job_1 = await create_job(session=session, run=run_1)
+        # old metrics
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_1,
+            collected_at=datetime(2023, 1, 2, 2, 3, 30),
+        )
+        run_2 = await create_run(
+            session=session, project=project, repo=repo, user=user, run_name="run-2"
+        )
+        job_2 = await create_job(session=session, run=run_2)
+        # recent metrics
+        metrics_2 = await create_job_prometheus_metrics(
+            session=session,
+            job=job_2,
+            collected_at=datetime(2023, 1, 2, 3, 5, 0),
+        )
+
+        await delete_prometheus_metrics()
+
+        res = await session.execute(
+            select(JobPrometheusMetrics).join(JobModel).where(JobModel.project_id == project.id)
+        )
+        all_metrics = res.scalars().all()
+        assert len(all_metrics) == 1
+        assert all_metrics[0] == metrics_2

tests/_internal/server/background/tasks/test_process_running_jobs.py

@@ -4,6 +4,7 @@ from typing import Optional
 from unittest.mock import MagicMock, Mock, patch
 
 import pytest
+from freezegun import freeze_time
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.errors import SSHError
@@ -11,6 +12,7 @@ from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import NetworkMode
 from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
 from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.profiles import UtilizationPolicy
 from dstack._internal.core.models.runs import (
     JobRuntimeData,
     JobStatus,
@@ -39,6 +41,7 @@ from dstack._internal.server.services.volumes import (
 from dstack._internal.server.testing.common import (
     create_instance,
     create_job,
+    create_job_metrics_point,
     create_pool,
     create_project,
     create_repo,
@@ -335,7 +338,7 @@ class TestProcessRunningJobs:
             name="test-run-0-0",
             registry_username="",
             registry_password="",
-            image_name="dstackai/base:py3.13-0.6-cuda-12.1",
+            image_name="dstackai/base:py3.13-0.7-cuda-12.1",
             container_user="root",
             privileged=privileged,
             gpu=None,
@@ -688,3 +691,125 @@ class TestProcessRunningJobs:
         assert job.status == expected_status
         assert job.termination_reason == expected_termination_reason
         assert job.inactivity_secs == expected_inactivity_secs
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+    @pytest.mark.parametrize(
+        ["samples", "expected_status"],
+        [
+            pytest.param(
+                [
+                    (datetime(2023, 1, 1, 12, 25, 20, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 25, 30, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 29, 50, tzinfo=timezone.utc), 40),
+                ],
+                JobStatus.RUNNING,
+                id="not-enough-points",
+            ),
+            pytest.param(
+                [
+                    (datetime(2023, 1, 1, 12, 20, 10, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 20, 20, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 29, 50, tzinfo=timezone.utc), 80),
+                ],
+                JobStatus.RUNNING,
+                id="any-above-min",
+            ),
+            pytest.param(
+                [
+                    (datetime(2023, 1, 1, 12, 10, 10, tzinfo=timezone.utc), 80),  # outside window
+                    (datetime(2023, 1, 1, 12, 10, 20, tzinfo=timezone.utc), 80),  # outside window
+                    (datetime(2023, 1, 1, 12, 20, 10, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 20, 20, tzinfo=timezone.utc), 30),
+                    (datetime(2023, 1, 1, 12, 29, 50, tzinfo=timezone.utc), 40),
+                ],
+                JobStatus.TERMINATING,
+                id="all-below-min",
+            ),
+        ],
+    )
+    @freeze_time(datetime(2023, 1, 1, 12, 30, tzinfo=timezone.utc))
+    async def test_gpu_utilization(
+        self,
+        test_db,
+        session: AsyncSession,
+        samples: list[tuple[datetime, int]],
+        expected_status: JobStatus,
+    ) -> None:
+        project = await create_project(session=session)
+        user = await create_user(session=session)
+        repo = await create_repo(
+            session=session,
+            project_id=project.id,
+        )
+        run = await create_run(
+            session=session,
+            project=project,
+            repo=repo,
+            user=user,
+            status=RunStatus.RUNNING,
+            run_name="test-run",
+            run_spec=get_run_spec(
+                run_name="test-run",
+                repo_id=repo.name,
+                configuration=DevEnvironmentConfiguration(
+                    name="test-run",
+                    ide="vscode",
+                    utilization_policy=UtilizationPolicy(
+                        min_gpu_utilization=80,
+                        time_window=600,
+                    ),
+                ),
+            ),
+        )
+        pool = await create_pool(session=session, project=project)
+        instance = await create_instance(
+            session=session,
+            project=project,
+            pool=pool,
+            status=InstanceStatus.BUSY,
+        )
+        job = await create_job(
+            session=session,
+            run=run,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=get_job_provisioning_data(),
+            instance=instance,
+            instance_assigned=True,
+        )
+        for timestamp, gpu_util in samples:
+            # two GPUs, the second one always 100% utilized
+            await create_job_metrics_point(
+                session=session,
+                job_model=job,
+                timestamp=timestamp,
+                gpus_memory_usage_bytes=[1024, 1024],
+                gpus_util_percent=[gpu_util, 100],
+            )
+        with (
+            patch("dstack._internal.server.services.runner.ssh.SSHTunnel") as SSHTunnelMock,
+            patch(
+                "dstack._internal.server.services.runner.client.RunnerClient"
+            ) as RunnerClientMock,
+        ):
+            runner_client_mock = RunnerClientMock.return_value
+            runner_client_mock.pull.return_value = PullResponse(
+                job_states=[],
+                job_logs=[],
+                runner_logs=[],
+                last_updated=0,
+                no_connections_secs=0,
+            )
+            await process_running_jobs()
+            SSHTunnelMock.assert_called_once()
+            runner_client_mock.pull.assert_called_once()
+        await session.refresh(job)
+        assert job.status == expected_status
+        if expected_status == JobStatus.TERMINATING:
+            assert job.termination_reason == JobTerminationReason.TERMINATED_BY_SERVER
+            assert job.termination_reason_message == (
+                "The job GPU utilization below 80% for 600 seconds"
+            )
+        else:
+            assert job.termination_reason is None
+            assert job.termination_reason_message is None

tests/_internal/server/conftest.py

@@ -8,6 +8,7 @@ from dstack._internal.server.main import app
 from dstack._internal.server.services import encryption as encryption  # import for side-effect
 from dstack._internal.server.services import logs as logs_services
 from dstack._internal.server.services.docker import ImageConfig, ImageConfigObject
+from dstack._internal.server.services.logs.filelog import FileLogStorage
 from dstack._internal.server.testing.conf import postgres_container, session, test_db  # noqa: F401
 
 
@@ -18,13 +19,11 @@ def client(event_loop):
 
 
 @pytest.fixture
-def test_log_storage(
-    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> logs_services.FileLogStorage:
+def test_log_storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FileLogStorage:
     root = tmp_path / "test_logs"
     root.mkdir()
-    storage = logs_services.FileLogStorage(root)
-    monkeypatch.setattr(logs_services, "_default_log_storage", storage)
+    storage = FileLogStorage(root)
+    monkeypatch.setattr(logs_services, "_log_storage", storage)
     return storage
 
 

tests/_internal/server/routers/test_backends.py

@@ -1335,6 +1335,7 @@ class TestGetConfigInfo:
             "vpc_ids": None,
             "default_vpcs": None,
             "public_ips": None,
+            "iam_instance_profile": None,
             "tags": None,
             "os_images": None,
             "creds": json.loads(backend.auth.plaintext),

tests/_internal/server/routers/test_fleets.py

@@ -370,6 +370,7 @@ class TestCreateFleet:
                     "idle_duration": None,
                     "termination_policy": None,
                     "termination_idle_time": None,
+                    "utilization_policy": None,
                     "name": "",
                     "default": False,
                     "reservation": None,
@@ -495,6 +496,7 @@ class TestCreateFleet:
                     "idle_duration": None,
                     "termination_policy": None,
                     "termination_idle_time": None,
+                    "utilization_policy": None,
                     "name": "",
                     "default": False,
                     "reservation": None,

tests/_internal/server/routers/test_logs.py

@@ -3,7 +3,7 @@ from httpx import AsyncClient
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.services.logs import FileLogStorage
+from dstack._internal.server.services.logs.filelog import FileLogStorage
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import create_project, create_user, get_auth_headers
 

tests/_internal/server/routers/test_metrics.py

@@ -55,11 +55,25 @@ class TestGetJobMetrics:
             session=session,
             run=run,
         )
+        await create_job_metrics_point(
+            session=session,
+            job_model=job,
+            timestamp=datetime(2023, 1, 2, 3, 4, 5, tzinfo=timezone.utc),
+            cpu_usage_micro=2 * 1_000_000,
+            memory_usage_bytes=256,
+            memory_working_set_bytes=128,
+            gpus_memory_usage_bytes=[256],
+            gpus_util_percent=[2],
+        )
         await create_job_metrics_point(
             session=session,
             job_model=job,
             timestamp=datetime(2023, 1, 2, 3, 4, 15, tzinfo=timezone.utc),
             cpu_usage_micro=4 * 1_000_000,
+            memory_usage_bytes=512,
+            memory_working_set_bytes=256,
+            gpus_memory_usage_bytes=[512],
+            gpus_util_percent=[6],
         )
         await create_job_metrics_point(
             session=session,
@@ -76,6 +90,7 @@ class TestGetJobMetrics:
             headers=get_auth_headers(user.token),
         )
         assert response.status_code == 200
+        # Returns one last sample by default. Filtering is tested in services/test_metrics.py
         assert response.json() == {
             "metrics": [
                 {

tests/_internal/server/routers/test_prometheus.py

@@ -0,0 +1,244 @@
+from textwrap import dedent
+
+import pytest
+from httpx import AsyncClient
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.core.models.runs import JobStatus
+from dstack._internal.core.models.users import GlobalRole, ProjectRole
+from dstack._internal.server.models import JobModel, ProjectModel, UserModel
+from dstack._internal.server.services.projects import add_project_member
+from dstack._internal.server.testing.common import (
+    create_job,
+    create_job_prometheus_metrics,
+    create_project,
+    create_repo,
+    create_run,
+    create_user,
+)
+
+
+@pytest.fixture
+def enable_metrics(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+@pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
+class TestGetPrometheusMetrics:
+    async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        project_2 = await _create_project(session, "project-2", user)
+        job_2_1 = await _create_job(session, "run-1", project_2, user, JobStatus.RUNNING)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_2_1,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 100
+                FIELD_1{gpu="1"} 200
+            """),
+        )
+        project_1 = await _create_project(session, "project-1", user)
+        job_1_1 = await _create_job(session, "run-1", project_1, user, JobStatus.RUNNING)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_1_1,
+            text=dedent("""
+                # Comments should be skipped
+
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 350
+                FIELD_1{gpu="1"} 400
+
+                # HELP FIELD_2 Test field 2
+                # TYPE FIELD_2 counter
+                FIELD_2{gpu="0"} 337325 1395066363000
+                FIELD_2{gpu="1"} 987169 1395066363010
+            """),
+        )
+        job_1_2 = await _create_job(session, "run-2", project_1, user, JobStatus.RUNNING)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_1_2,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 1200.0
+                FIELD_1{gpu="1"} 1600.0
+                FIELD_1{gpu="2"} 2400.0
+            """),
+        )
+        # Terminated job, should not appear in the response
+        job_1_3 = await _create_job(session, "run-3", project_1, user, JobStatus.TERMINATED)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_1_3,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 10
+                FIELD_1{gpu="1"} 20
+            """),
+        )
+
+        response = await client.get("/metrics")
+
+        assert response.status_code == 200
+        assert response.text == dedent("""\
+            # HELP FIELD_1 Test field 1
+            # TYPE FIELD_1 gauge
+            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 350.0
+            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 400.0
+            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1200.0
+            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1600.0
+            FIELD_1{gpu="2",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 2400.0
+            FIELD_1{gpu="0",dstack_project_name="project-2",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 100.0
+            FIELD_1{gpu="1",dstack_project_name="project-2",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 200.0
+            # HELP FIELD_2 Test field 2
+            # TYPE FIELD_2 counter
+            FIELD_2{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 337325.0 1395066363000
+            FIELD_2{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 987169.0 1395066363010
+        """)
+
+    async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
+        response = await client.get("/metrics")
+        assert response.status_code == 200
+        assert response.text == ""
+
+    async def test_returns_404_if_not_enabled(
+        self, monkeypatch: pytest.MonkeyPatch, client: AsyncClient
+    ):
+        monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", False)
+        response = await client.get("/metrics")
+        assert response.status_code == 404
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+@pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
+class TestGetPrometheusProjectMetrics:
+    async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        project = await _create_project(session, "project-1", user)
+        job_1 = await _create_job(session, "run-1", project, user, JobStatus.RUNNING)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_1,
+            text=dedent("""
+                # Comments should be skipped
+
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 350
+                FIELD_1{gpu="1"} 400
+
+                # HELP FIELD_2 Test field 2
+                # TYPE FIELD_2 counter
+                FIELD_2{gpu="0"} 337325 1395066363000
+                FIELD_2{gpu="1"} 987169 1395066363010
+            """),
+        )
+        job_2 = await _create_job(session, "run-2", project, user, JobStatus.RUNNING)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_2,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 1200.0
+                FIELD_1{gpu="1"} 1600.0
+                FIELD_1{gpu="2"} 2400.0
+            """),
+        )
+        # Terminated job, should not appear in the response
+        job_3 = await _create_job(session, "run-3", project, user, JobStatus.TERMINATED)
+        await create_job_prometheus_metrics(
+            session=session,
+            job=job_3,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 10
+                FIELD_1{gpu="1"} 20
+            """),
+        )
+        another_project = await _create_project(session, "project-2", user)
+        another_project_job = await _create_job(
+            session, "run-4", another_project, user, JobStatus.RUNNING
+        )
+        await create_job_prometheus_metrics(
+            session=session,
+            job=another_project_job,
+            text=dedent("""
+                # HELP FIELD_1 Test field 1
+                # TYPE FIELD_1 gauge
+                FIELD_1{gpu="0"} 100
+                FIELD_1{gpu="1"} 200
+            """),
+        )
+
+        response = await client.get("/metrics/project/project-1")
+
+        assert response.status_code == 200
+        assert response.text == dedent("""\
+            # HELP FIELD_1 Test field 1
+            # TYPE FIELD_1 gauge
+            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 350.0
+            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 400.0
+            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1200.0
+            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1600.0
+            FIELD_1{gpu="2",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 2400.0
+            # HELP FIELD_2 Test field 2
+            # TYPE FIELD_2 counter
+            FIELD_2{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 337325.0 1395066363000
+            FIELD_2{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 987169.0 1395066363010
+        """)
+
+    async def test_returns_empty_response_if_no_runs(
+        self, session: AsyncSession, client: AsyncClient
+    ):
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        await create_project(session=session, owner=user, name="test-project")
+        response = await client.get("/metrics/project/test-project")
+        assert response.status_code == 200
+        assert response.text == ""
+
+    async def test_returns_404_if_project_doesnt_exist(self, client: AsyncClient):
+        response = await client.get("/metrics/project/nonexistent")
+        assert response.status_code == 404
+
+    async def test_returns_404_if_not_enabled(
+        self, monkeypatch: pytest.MonkeyPatch, session: AsyncSession, client: AsyncClient
+    ):
+        monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", False)
+        user = await create_user(session=session, global_role=GlobalRole.USER)
+        await create_project(session=session, owner=user, name="test-project")
+        response = await client.get("/metrics/project/test-project")
+        assert response.status_code == 404
+
+
+async def _create_project(session: AsyncSession, name: str, user: UserModel) -> ProjectModel:
+    project = await create_project(session=session, owner=user, name=name)
+    await add_project_member(
+        session=session, project=project, user=user, project_role=ProjectRole.USER
+    )
+    return project
+
+
+async def _create_job(
+    session: AsyncSession, run_name: str, project: ProjectModel, user: UserModel, status: JobStatus
+) -> JobModel:
+    repo = await create_repo(session=session, project_id=project.id, repo_name=f"{run_name}-repo")
+    run = await create_run(
+        session=session,
+        project=project,
+        repo=repo,
+        user=user,
+        run_name=run_name,
+    )
+    job = await create_job(session=session, run=run, status=status)
+    return job