dstack 0.18.44__py3-none-any.whl → 0.19.0rc1__py3-none-any.whl

tests/_internal/server/routers/test_fleets.py

@@ -27,7 +27,6 @@ from dstack._internal.server.testing.common import (
     create_fleet,
     create_instance,
     create_job,
-    create_pool,
     create_project,
     create_repo,
     create_run,
@@ -346,8 +345,6 @@ class TestCreateFleet:
                     "retry": None,
                     "max_price": None,
                     "idle_duration": None,
-                    "termination_policy": None,
-                    "termination_idle_time": None,
                     "type": "fleet",
                     "name": "test-fleet",
                     "reservation": None,
@@ -360,16 +357,11 @@ class TestCreateFleet:
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
-                    "retry_policy": None,
                     "max_duration": None,
                     "stop_duration": None,
                     "max_price": None,
-                    "pool_name": None,
-                    "instance_name": None,
                     "creation_policy": None,
                     "idle_duration": None,
-                    "termination_policy": None,
-                    "termination_idle_time": None,
                     "utilization_policy": None,
                     "name": "",
                     "default": False,
@@ -394,7 +386,6 @@ class TestCreateFleet:
                     "unreachable": False,
                     "termination_reason": None,
                     "created": "2023-01-02T03:04:00+00:00",
-                    "pool_name": None,
                     "backend": None,
                     "region": None,
                     "availability_zone": None,
@@ -472,8 +463,6 @@ class TestCreateFleet:
                     "retry": None,
                     "max_price": None,
                     "idle_duration": None,
-                    "termination_policy": None,
-                    "termination_idle_time": None,
                     "type": "fleet",
                     "name": spec.configuration.name,
                     "reservation": None,
@@ -486,16 +475,11 @@ class TestCreateFleet:
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
-                    "retry_policy": None,
                     "max_duration": None,
                     "stop_duration": None,
                     "max_price": None,
-                    "pool_name": None,
-                    "instance_name": None,
                     "creation_policy": None,
                     "idle_duration": None,
-                    "termination_policy": None,
-                    "termination_idle_time": None,
                     "utilization_policy": None,
                     "name": "",
                     "default": False,
@@ -526,7 +510,6 @@ class TestCreateFleet:
                     "fleet_id": "1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e",
                     "fleet_name": spec.configuration.name,
                     "instance_num": 0,
-                    "pool_name": None,
                     "job_name": None,
                     "hostname": "1.1.1.1",
                     "status": "pending",
@@ -627,12 +610,10 @@ class TestDeleteFleets:
         await add_project_member(
             session=session, project=project, user=user, project_role=ProjectRole.USER
         )
-        pool = await create_pool(session=session, project=project)
         fleet = await create_fleet(session=session, project=project)
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
         )
         fleet.instances.append(instance)
         await session.commit()
@@ -654,7 +635,6 @@ class TestDeleteFleets:
     ):
         user = await create_user(session, global_role=GlobalRole.USER)
         project = await create_project(session)
-        pool = await create_pool(session=session, project=project)
         await add_project_member(
             session=session, project=project, user=user, project_role=ProjectRole.USER
         )
@@ -676,7 +656,6 @@ class TestDeleteFleets:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
             job=job,
         )
@@ -744,18 +723,15 @@ class TestDeleteFleetInstances:
         await add_project_member(
             session=session, project=project, user=user, project_role=ProjectRole.USER
         )
-        pool = await create_pool(session=session, project=project)
         fleet = await create_fleet(session=session, project=project)
         instance1 = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             instance_num=1,
         )
         instance2 = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             instance_num=2,
         )
         fleet.instances.append(instance1)
@@ -785,7 +761,6 @@ class TestDeleteFleetInstances:
         await add_project_member(
             session=session, project=project, user=user, project_role=ProjectRole.USER
         )
-        pool = await create_pool(session=session, project=project)
         fleet = await create_fleet(session=session, project=project)
         repo = await create_repo(
             session=session,
@@ -804,7 +779,6 @@ class TestDeleteFleetInstances:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             instance_num=1,
             status=InstanceStatus.BUSY,
             job=job,

tests/_internal/server/routers/test_gateways.py

@@ -13,6 +13,7 @@ from dstack._internal.server.services.gateways import (
 )
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
+    ComputeMockSpec,
     create_backend,
     create_gateway,
     create_gateway_compute,
@@ -174,7 +175,14 @@ class TestCreateGateway:
         backend = await create_backend(session, project.id, backend_type=BackendType.AWS)
         response = await client.post(
             f"/api/project/{project.name}/gateways/create",
-            json={"name": "test", "backend_type": "aws", "region": "us"},
+            json={
+                "configuration": {
+                    "type": "gateway",
+                    "name": "test",
+                    "backend": "aws",
+                    "region": "us",
+                },
+            },
             headers=get_auth_headers(user.token),
         )
         assert response.status_code == 200
@@ -217,7 +225,14 @@ class TestCreateGateway:
             g.return_value = "random-name"
             response = await client.post(
                 f"/api/project/{project.name}/gateways/create",
-                json={"name": None, "backend_type": "aws", "region": "us"},
+                json={
+                    "configuration": {
+                        "type": "gateway",
+                        "name": None,
+                        "backend": "aws",
+                        "region": "us",
+                    },
+                },
                 headers=get_auth_headers(user.token),
             )
             g.assert_called_once()
@@ -258,7 +273,14 @@ class TestCreateGateway:
         )
         response = await client.post(
             f"/api/project/{project.name}/gateways/create",
-            json={"name": "test", "backend_type": "aws", "region": "us"},
+            json={
+                "configuration": {
+                    "type": "gateway",
+                    "name": "test",
+                    "backend": "aws",
+                    "region": "us",
+                },
+            },
             headers=get_auth_headers(user.token),
         )
         assert response.status_code == 400
@@ -437,8 +459,10 @@ class TestDeleteGateway:
             "dstack._internal.server.services.gateways.get_project_backend_by_type_or_error"
         ) as m:
             aws = Mock()
+            aws.compute.return_value = Mock(spec=ComputeMockSpec)
             aws.compute.return_value.terminate_gateway.return_value = None  # success
             gcp = Mock()
+            gcp.compute.return_value = Mock(spec=ComputeMockSpec)
             gcp.compute.return_value.terminate_gateway.side_effect = DstackError()  # fail
 
             def get_backend(project, backend_type):

tests/_internal/server/routers/test_instances.py

@@ -15,7 +15,6 @@ from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
     create_fleet,
     create_instance,
-    create_pool,
     create_project,
     create_user,
     get_auth_headers,
@@ -59,11 +58,6 @@ class TestListInstances:
         await add_project_member(
             session, project=projects[2], user=users[1], project_role=ProjectRole.USER
         )
-        pools = [
-            await create_pool(session, projects[0]),
-            await create_pool(session, projects[1]),
-            await create_pool(session, projects[2]),
-        ]
         fleets = [
             await create_fleet(
                 session,
@@ -88,7 +82,6 @@ class TestListInstances:
             await create_instance(
                 session=session,
                 project=projects[0],
-                pool=pools[0],
                 fleet=fleets[0],
                 created_at=dt.datetime(2024, 1, 1, tzinfo=dt.timezone.utc),
                 name="fleet0-0",
@@ -96,7 +89,6 @@ class TestListInstances:
             await create_instance(
                 session=session,
                 project=projects[1],
-                pool=pools[1],
                 fleet=fleets[1],
                 created_at=dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc),
                 name="fleet1-0",
@@ -104,7 +96,6 @@ class TestListInstances:
             await create_instance(
                 session=session,
                 project=projects[2],
-                pool=pools[2],
                 fleet=fleets[2],
                 created_at=dt.datetime(2024, 1, 3, tzinfo=dt.timezone.utc),
                 name="fleet2-0",
@@ -112,7 +103,6 @@ class TestListInstances:
             await create_instance(
                 session=session,
                 project=projects[2],
-                pool=pools[2],
                 fleet=fleets[2],
                 created_at=dt.datetime(2024, 1, 4, tzinfo=dt.timezone.utc),
                 instance_num=1,

tests/_internal/server/routers/test_metrics.py

@@ -14,6 +14,9 @@ from dstack._internal.server.testing.common import (
     create_run,
     create_user,
     get_auth_headers,
+    get_instance_offer_with_availability,
+    get_job_provisioning_data,
+    get_job_runtime_data,
 )
 
 pytestmark = pytest.mark.usefixtures("image_config_mock")
@@ -51,9 +54,18 @@ class TestGetJobMetrics:
             repo=repo,
             user=user,
         )
+        jpd = get_job_provisioning_data(
+            cpu_count=128, memory_gib=256, gpu_count=2, gpu_memory_gib=32
+        )
+        offer = get_instance_offer_with_availability(
+            cpu_count=64, memory_gib=128, gpu_count=1, gpu_memory_gib=32
+        )
+        jrd = get_job_runtime_data(offer=offer)
         job = await create_job(
             session=session,
             run=run,
+            job_provisioning_data=jpd,
+            job_runtime_data=jrd,
         )
         await create_job_metrics_point(
             session=session,
@@ -108,11 +120,26 @@ class TestGetJobMetrics:
                     "timestamps": ["2023-01-02T03:04:25+00:00"],
                     "values": [512],
                 },
+                {
+                    "name": "cpus_detected_num",
+                    "timestamps": ["2023-01-02T03:04:25+00:00"],
+                    "values": [64],
+                },
+                {
+                    "name": "memory_total_bytes",
+                    "timestamps": ["2023-01-02T03:04:25+00:00"],
+                    "values": [137438953472],
+                },
                 {
                     "name": "gpus_detected_num",
                     "timestamps": ["2023-01-02T03:04:25+00:00"],
                     "values": [1],
                 },
+                {
+                    "name": "gpu_memory_total_bytes",
+                    "timestamps": ["2023-01-02T03:04:25+00:00"],
+                    "values": [34359738368],
+                },
                 {
                     "name": "gpu_memory_usage_bytes_gpu0",
                     "timestamps": ["2023-01-02T03:04:25+00:00"],

tests/_internal/server/routers/test_projects.py

@@ -503,6 +503,62 @@ class TestSetProjectMembers:
         members = res.scalars().all()
         assert len(members) == 3
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+    async def test_sets_project_members_by_email(
+        self, test_db, session: AsyncSession, client: AsyncClient
+    ):
+        project = await create_project(
+            session=session,
+            created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+        )
+        admin = await create_user(
+            session=session,
+            created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+            global_role=GlobalRole.ADMIN,
+        )
+        user1 = await create_user(
+            session=session,
+            name="user1",
+            created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+            email="testemail@example.com",
+        )
+        members = [
+            {
+                "username": user1.email,
+                "project_role": ProjectRole.ADMIN,
+            },
+        ]
+        body = {"members": members}
+        response = await client.post(
+            f"/api/projects/{project.name}/set_members",
+            headers=get_auth_headers(admin.token),
+            json=body,
+        )
+        assert response.status_code == 200, response.json()
+        assert response.json()["members"] == [
+            {
+                "user": {
+                    "id": str(user1.id),
+                    "username": user1.name,
+                    "created_at": "2023-01-02T03:04:00+00:00",
+                    "global_role": user1.global_role,
+                    "email": user1.email,
+                    "active": True,
+                    "permissions": {
+                        "can_create_projects": True,
+                    },
+                },
+                "project_role": ProjectRole.ADMIN,
+                "permissions": {
+                    "can_manage_ssh_fleets": True,
+                },
+            },
+        ]
+        res = await session.execute(select(MemberModel))
+        members = res.scalars().all()
+        assert len(members) == 1
+
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_manager_cannot_set_project_admins(

tests/_internal/server/routers/test_prometheus.py

@@ -1,20 +1,31 @@
+from datetime import datetime, timedelta, timezone
 from textwrap import dedent
+from typing import Optional
 
 import pytest
+from freezegun import freeze_time
 from httpx import AsyncClient
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.models.runs import JobStatus
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
+from dstack._internal.core.models.runs import JobProvisioningData, JobRuntimeData, JobStatus
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
 from dstack._internal.server.models import JobModel, ProjectModel, UserModel
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
+    create_fleet,
+    create_instance,
     create_job,
     create_job_prometheus_metrics,
     create_project,
     create_repo,
     create_run,
     create_user,
+    get_instance_offer_with_availability,
+    get_job_provisioning_data,
+    get_job_runtime_data,
+    get_run_spec,
 )
 
 
@@ -23,14 +34,32 @@ def enable_metrics(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", True)
 
 
+FAKE_NOW = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc)
+
+
+@freeze_time(FAKE_NOW)
 @pytest.mark.asyncio
 @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
 @pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
 class TestGetPrometheusMetrics:
     async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
-        user = await create_user(session=session, global_role=GlobalRole.USER)
+        user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
+        offer = get_instance_offer_with_availability(
+            instance_type="test-type", gpu_count=2, gpu_name="V4", price=12
+        )
         project_2 = await _create_project(session, "project-2", user)
-        job_2_1 = await _create_job(session, "run-1", project_2, user, JobStatus.RUNNING)
+        jpd_2_1 = get_job_provisioning_data(
+            backend=BackendType.AWS, gpu_name="T4", gpu_count=2, price=16
+        )
+        job_2_1 = await _create_job(
+            session=session,
+            run_name="run-1",
+            project=project_2,
+            user=user,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=jpd_2_1,
+            submitted_at=FAKE_NOW - timedelta(seconds=100),
+        )
         await create_job_prometheus_metrics(
             session=session,
             job=job_2_1,
@@ -42,7 +71,18 @@ class TestGetPrometheusMetrics:
             """),
         )
         project_1 = await _create_project(session, "project-1", user)
-        job_1_1 = await _create_job(session, "run-1", project_1, user, JobStatus.RUNNING)
+        jpd_1_1 = get_job_provisioning_data(backend=BackendType.AWS, gpu_count=4, gpu_name="T4")
+        jrd_1_1 = get_job_runtime_data(offer=offer)
+        job_1_1 = await _create_job(
+            session=session,
+            run_name="run-1",
+            project=project_1,
+            user=user,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=jpd_1_1,
+            job_runtime_data=jrd_1_1,
+            submitted_at=FAKE_NOW - timedelta(seconds=120),
+        )
         await create_job_prometheus_metrics(
             session=session,
             job=job_1_1,
@@ -84,30 +124,62 @@ class TestGetPrometheusMetrics:
                 FIELD_1{gpu="1"} 20
             """),
         )
+        fleet = await create_fleet(session=session, project=project_1, name="test-fleet")
+        instance = await create_instance(
+            session=session,
+            project=project_1,
+            fleet=fleet,
+            backend=BackendType.AWS,
+            offer=offer,
+            price=14,
+            created_at=FAKE_NOW - timedelta(hours=1),
+            name="test-instance",
+        )
 
         response = await client.get("/metrics")
 
         assert response.status_code == 200
-        assert response.text == dedent("""\
+        assert response.text == dedent(f"""\
+            # HELP dstack_instance_duration_seconds_total Total seconds the instance is running
+            # TYPE dstack_instance_duration_seconds_total counter
+            dstack_instance_duration_seconds_total{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 3600.0
+            # HELP dstack_instance_price_dollars_per_hour Instance price, USD/hour
+            # TYPE dstack_instance_price_dollars_per_hour gauge
+            dstack_instance_price_dollars_per_hour{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 14.0
+            # HELP dstack_instance_gpu_count Instance GPU count
+            # TYPE dstack_instance_gpu_count gauge
+            dstack_instance_gpu_count{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            # HELP dstack_job_duration_seconds_total Total seconds the job is running
+            # TYPE dstack_job_duration_seconds_total counter
+            dstack_job_duration_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 120.0
+            dstack_job_duration_seconds_total{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 100.0
+            # HELP dstack_job_price_dollars_per_hour Job instance price, USD/hour
+            # TYPE dstack_job_price_dollars_per_hour gauge
+            dstack_job_price_dollars_per_hour{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 12.0
+            dstack_job_price_dollars_per_hour{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 16.0
+            # HELP dstack_job_gpu_count Job GPU count
+            # TYPE dstack_job_gpu_count gauge
+            dstack_job_gpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            dstack_job_gpu_count{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 2.0
             # HELP FIELD_1 Test field 1
             # TYPE FIELD_1 gauge
-            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 350.0
-            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 400.0
-            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1200.0
-            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1600.0
-            FIELD_1{gpu="2",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 2400.0
-            FIELD_1{gpu="0",dstack_project_name="project-2",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 100.0
-            FIELD_1{gpu="1",dstack_project_name="project-2",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 200.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
+            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 100.0
+            FIELD_1{{gpu="1",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 200.0
             # HELP FIELD_2 Test field 2
             # TYPE FIELD_2 counter
-            FIELD_2{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 337325.0 1395066363000
-            FIELD_2{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 987169.0 1395066363010
+            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
+            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
         """)
 
     async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
         response = await client.get("/metrics")
         assert response.status_code == 200
-        assert response.text == ""
+        assert response.text == "\n"
 
     async def test_returns_404_if_not_enabled(
         self, monkeypatch: pytest.MonkeyPatch, client: AsyncClient
@@ -122,7 +194,7 @@ class TestGetPrometheusMetrics:
 @pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
 class TestGetPrometheusProjectMetrics:
     async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
-        user = await create_user(session=session, global_role=GlobalRole.USER)
+        user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
         project = await _create_project(session, "project-1", user)
         job_1 = await _create_job(session, "run-1", project, user, JobStatus.RUNNING)
         await create_job_prometheus_metrics(
@@ -184,18 +256,18 @@ class TestGetPrometheusProjectMetrics:
         response = await client.get("/metrics/project/project-1")
 
         assert response.status_code == 200
-        assert response.text == dedent("""\
+        assert response.text == dedent(f"""\
             # HELP FIELD_1 Test field 1
             # TYPE FIELD_1 gauge
-            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 350.0
-            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 400.0
-            FIELD_1{gpu="0",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1200.0
-            FIELD_1{gpu="1",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 1600.0
-            FIELD_1{gpu="2",dstack_project_name="project-1",dstack_run_name="run-2",dstack_job_name="run-2-0-0",dstack_job_num="0",dstack_replica_num="0"} 2400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
+            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
             # HELP FIELD_2 Test field 2
             # TYPE FIELD_2 counter
-            FIELD_2{gpu="0",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 337325.0 1395066363000
-            FIELD_2{gpu="1",dstack_project_name="project-1",dstack_run_name="run-1",dstack_job_name="run-1-0-0",dstack_job_num="0",dstack_replica_num="0"} 987169.0 1395066363010
+            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
+            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
         """)
 
     async def test_returns_empty_response_if_no_runs(
@@ -205,7 +277,7 @@ class TestGetPrometheusProjectMetrics:
         await create_project(session=session, owner=user, name="test-project")
         response = await client.get("/metrics/project/test-project")
         assert response.status_code == 200
-        assert response.text == ""
+        assert response.text == "\n"
 
     async def test_returns_404_if_project_doesnt_exist(self, client: AsyncClient):
         response = await client.get("/metrics/project/nonexistent")
@@ -230,15 +302,32 @@ async def _create_project(session: AsyncSession, name: str, user: UserModel) ->
 
 
 async def _create_job(
-    session: AsyncSession, run_name: str, project: ProjectModel, user: UserModel, status: JobStatus
+    session: AsyncSession,
+    run_name: str,
+    project: ProjectModel,
+    user: UserModel,
+    status: JobStatus,
+    job_provisioning_data: Optional[JobProvisioningData] = None,
+    job_runtime_data: Optional[JobRuntimeData] = None,
+    submitted_at: datetime = FAKE_NOW,
 ) -> JobModel:
     repo = await create_repo(session=session, project_id=project.id, repo_name=f"{run_name}-repo")
+    configuration = DevEnvironmentConfiguration(ide="vscode")
+    run_spec = get_run_spec(run_name=run_name, repo_id=repo.name, configuration=configuration)
     run = await create_run(
         session=session,
         project=project,
         repo=repo,
         user=user,
         run_name=run_name,
+        run_spec=run_spec,
+    )
+    job = await create_job(
+        session=session,
+        run=run,
+        status=status,
+        job_provisioning_data=job_provisioning_data,
+        job_runtime_data=job_runtime_data,
+        submitted_at=submitted_at,
     )
-    job = await create_job(session=session, run=run, status=status)
     return job