dstack 0.19.0rc1__py3-none-any.whl → 0.19.1__py3-none-any.whl

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.0rc1"
+__version__ = "0.19.1"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.19.0rc1.dist-info → dstack-0.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.19.0rc1
+Version: 0.19.1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov

{dstack-0.19.0rc1.dist-info → dstack-0.19.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=-Q9COxDViZow52OSN2VafeyzPtL8-5rmrIp9kMGNcuc,67
+dstack/version.py,sha256=ja4J6HzBpBX3wxm5CKLUUgzAwmmr8naAhq3SBch6VIw,64
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
@@ -242,7 +242,7 @@ dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
 dstack/_internal/server/background/tasks/process_fleets.py,sha256=lKXUvN_b7DNjD3psHzyCt_JYsTxPFuQ86iXi8fj8GkM,3202
 dstack/_internal/server/background/tasks/process_gateways.py,sha256=hoUI1CSqbHt_uMwnzTRAEDl-LBw0wUk_W4xobIbdvRc,7017
-dstack/_internal/server/background/tasks/process_instances.py,sha256=TvpMnK211tXcOPAMF3qngGH9DycwKJeEchoYOtKICrg,37284
+dstack/_internal/server/background/tasks/process_instances.py,sha256=Kc7CbWK4mFOsKwOqp-Pt0ewTsB5OZ5gkPyv9T6TNbpM,37674
 dstack/_internal/server/background/tasks/process_metrics.py,sha256=acySfsacpYbTPV9Yivs-oU37z1S2sUdWhRHdJkfBcCA,5332
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FqGfbzvfILdnPUfxjFPAM1ij2xd2mCDi8qufiBcUMI8,4107
 dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=u8hCXjOOek7VLEsmLy2VnDXFmIwTNjrJwcpWG7a1zW0,5093
@@ -325,7 +325,7 @@ dstack/_internal/server/routers/instances.py,sha256=XOogTC9My2Zv0ck37_PbHKoZI-j4
 dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
 dstack/_internal/server/routers/metrics.py,sha256=VFgWhkOvxVFDLlRM_kXHYFylLcfCD6UjXInvcd7H4dY,2314
 dstack/_internal/server/routers/projects.py,sha256=0R-w_6WXUbNo6fREAexFUQ3RoOJF2D_Iz35elKjym14,2717
-dstack/_internal/server/routers/prometheus.py,sha256=CIz1GoCVnSAevWhTPnbIJKUO-ntz8tvK6Q_2vicDLoo,1246
+dstack/_internal/server/routers/prometheus.py,sha256=OuC17kgKkb2ErxDD5QZ_ZdZft5A8dMIAFlIzQ_04NEo,744
 dstack/_internal/server/routers/repos.py,sha256=P_zLoEQderxhCeHQJwRkrIhVcc0-cpabfyde22bWVRk,3362
 dstack/_internal/server/routers/runs.py,sha256=oPqyIRPwkMjj12M1IdMF2UitatqvljISAXnJAjfEJyQ,5352
 dstack/_internal/server/routers/secrets.py,sha256=50_qJCTYRpnGSlLyS93gqoV17wWewOVmM65PcG1bT_Y,856
@@ -360,7 +360,7 @@ dstack/_internal/server/services/offers.py,sha256=tTld2ZcYdbhzShtMIf1YfTyIADtpN3
 dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
 dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBiq7lj41LiAP1PfoEs,1773
 dstack/_internal/server/services/projects.py,sha256=Y4LEkSvOVUHHP-F2qlrwBR7rFu0CFFhbHmDTKrrNuXE,15071
-dstack/_internal/server/services/prometheus.py,sha256=1hXSsML-xIiQHv2DKVuJTfK0FRAlseF3GESUQUvy3NE,9625
+dstack/_internal/server/services/prometheus.py,sha256=xq5G-Q2BJup9lS2F6__0wUVTs-k1Gr3dYclGzo2WoWo,12474
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
 dstack/_internal/server/services/runs.py,sha256=B2jZtTOxavUHr6WqKMXqgLzB3xWsHTkWKykcvcT2lXI,37245
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
@@ -385,7 +385,7 @@ dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YF
 dstack/_internal/server/services/jobs/configurators/task.py,sha256=0-B3oO-61Eq4-mmlLmqJPliFKHhvvIV0tqc12slcQuA,1436
 dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
-dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=r2siUu__sz86xDR_oK2xODmmzef1FzOS3veekGTS2-s,1645
+dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=00HB1kC_eMlioEW0nZI7Ly78b-RSehySVNhC9pajBP8,1636
 dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
 dstack/_internal/server/services/logs/__init__.py,sha256=NAjO1KeYvuDznN2EkfAaJt9S6Y00fo_dl3ob3WmsdGQ,3088
 dstack/_internal/server/services/logs/aws.py,sha256=949k8t9H9v_-aedDjDWkw8yPVyhZemmsszcDDEL5Tb4,13711
@@ -431,9 +431,9 @@ dstack/_internal/server/statics/e467d7d60aae81ab198b.svg,sha256=_XHc9mfQZgGkcy4h
 dstack/_internal/server/statics/eb9b344b73818fe2b71a.png,sha256=2H14eOCQRyZhFGJ1Kn2LH1j70kTF1Qop4vH-tiKqyPI,85
 dstack/_internal/server/statics/f517dd626eb964120de0.png,sha256=4QQuNa8SqmcZ67HK6739OHCyjnAJseU1bkcn454KRQs,159
 dstack/_internal/server/statics/f958aecddee5d8e3222c.png,sha256=8CoZkVNgRfOAe62X1dU-AZDvwh_nESKaQblEmaX2Xrs,87
-dstack/_internal/server/statics/index.html,sha256=JPOmBF44MDRAYRPj8eT7Dm4ZnwVLXrRyq32Qku5-fPo,10468
-dstack/_internal/server/statics/main-4fd5a4770eff59325ee3.js,sha256=kt1vdr7aEynUtIrrzu9gVEde2kxBMmv4LsTaZDIUNV8,6201881
-dstack/_internal/server/statics/main-4fd5a4770eff59325ee3.js.map,sha256=BAYKYPDgLlBT5KAU8L8SS8LYOuU7rDPIL3PpigxAgNo,8139109
+dstack/_internal/server/statics/index.html,sha256=1RngYCyoktp3XhwHvC2jeCdolq0T-w-LDfNJHInNeI0,10468
+dstack/_internal/server/statics/main-4a0fe83e84574654e397.js,sha256=lNEeGSQNL6XYc_nXkZwtL35FOJkqbAGUexPC38feqsQ,6204421
+dstack/_internal/server/statics/main-4a0fe83e84574654e397.js.map,sha256=WoJHj0c5pkjUN-aGI-v8oyylaxU3y4KgJ_G8QiwtvA8,8143363
 dstack/_internal/server/statics/main-da9f8c06a69c20dac23e.css,sha256=2ObS4Rg6yWkbk7nQqw-lnLVdYC3L8X5N7xZMKcavMWg,1296312
 dstack/_internal/server/statics/manifest.json,sha256=430w2BoWVmYYVr14lDvUxx-ROPt3VjigzeMqfLeiSCM,340
 dstack/_internal/server/statics/robots.txt,sha256=kNJLw79pisHhc3OVAimMzKcq3x9WT6sF9IS4xI0crdI,67
@@ -639,7 +639,7 @@ tests/_internal/server/background/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 tests/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/background/tasks/test_process_fleets.py,sha256=Dl31_TwxoCzYqkVNPWGLsYxmGL2sZfEK3rQXLFyPIz8,2701
 tests/_internal/server/background/tasks/test_process_gateways.py,sha256=lOP4jPXDtadAgYp0aFND_fp5R_X19M58CaOlgnDAEck,5085
-tests/_internal/server/background/tasks/test_process_instances.py,sha256=b7Shke7p3FLPT-qTVDMnwuCK4KqR4kLw9UFfsr0TSVg,24549
+tests/_internal/server/background/tasks/test_process_instances.py,sha256=WC32HvynBuSxwFtAyMTHS4eVzqCnyGufcrIUTEVoozI,27944
 tests/_internal/server/background/tasks/test_process_metrics.py,sha256=z-u4HXJE5EMVH9kwU_POHmvp55ldAvuLpEMkaebBtsg,4976
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=19LYbIMZIIeKAN0b9KOMyS-cHUx0FoOojqQuM8Oeiq4,1620
 tests/_internal/server/background/tasks/test_process_prometheus_metrics.py,sha256=I9DgIJXVGS7UvbFgm4HFnzWiCICBpy72NjDPKU_7WII,7178
@@ -656,7 +656,7 @@ tests/_internal/server/routers/test_instances.py,sha256=78HFMU9Xel8BNZL3TqnuvrKE
 tests/_internal/server/routers/test_logs.py,sha256=NZwyJlgjMOGq4XEx7-VDjTpniYPhZpsbZvB0dTawaog,3989
 tests/_internal/server/routers/test_metrics.py,sha256=xMdDFZW73Zl06QfggjatfwTut37s0soeliJivkCgBks,7620
 tests/_internal/server/routers/test_projects.py,sha256=Z3Ok7onAjUYS4ADvKvN-SwSxYKvlvf4MG5Y8baqQU14,25964
-tests/_internal/server/routers/test_prometheus.py,sha256=L3qxaJQzir2nJbFtRHZMC7Pw6xWBFcv4-0buQUSdek8,18933
+tests/_internal/server/routers/test_prometheus.py,sha256=LqJwWn5ztSLIGnvZgj-sD7BFW-JuePFt6k__ymF5Btw,22711
 tests/_internal/server/routers/test_repos.py,sha256=G4dKuFGd_UrxAHwh_XLl1xCHK_DCsiJcXBsHODw3yJk,16682
 tests/_internal/server/routers/test_runs.py,sha256=q02oBrUcp4JoJOL68jbxlfFxH9B8JO9Bkb7v_Qg-Aug,62984
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
@@ -701,9 +701,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.19.0rc1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.0rc1.dist-info/METADATA,sha256=TB4oTDpqjbz5UruEwcMWcwY_0cQZl4kNZuhwZlS30Ww,18234
-dstack-0.19.0rc1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.19.0rc1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.0rc1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.19.0rc1.dist-info/RECORD,,
+dstack-0.19.1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.1.dist-info/METADATA,sha256=sj_wcanWBaGU9ecMn6I32zvXuYFniyN_6K6ehbcO3tA,18231
+dstack-0.19.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.19.1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.19.1.dist-info/RECORD,,

tests/_internal/server/background/tasks/test_process_instances.py

@@ -8,7 +8,7 @@ import pytest
 from freezegun import freeze_time
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.errors import BackendError
+from dstack._internal.core.errors import BackendError, ProvisioningError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     Gpu,
@@ -35,6 +35,8 @@ from dstack._internal.server.testing.common import (
     create_repo,
     create_run,
     create_user,
+    get_instance_offer_with_availability,
+    get_job_provisioning_data,
     get_remote_connection_info,
 )
 from dstack._internal.utils.common import get_current_datetime
@@ -557,6 +559,68 @@ class TestCreateInstance:
         assert instance.total_blocks == expected_blocks
         assert instance.busy_blocks == 0
 
+    @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
+    async def test_tries_second_offer_if_first_fails(self, session: AsyncSession, err: Exception):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        aws_mock = Mock()
+        aws_mock.TYPE = BackendType.AWS
+        offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
+        aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        aws_mock.compute.return_value.create_instance.side_effect = err
+        gcp_mock = Mock()
+        gcp_mock.TYPE = BackendType.GCP
+        offer = get_instance_offer_with_availability(backend=BackendType.GCP, price=2.0)
+        gcp_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        gcp_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        gcp_mock.compute.return_value.create_instance.return_value = get_job_provisioning_data(
+            backend=offer.backend, region=offer.region, price=offer.price
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = [aws_mock, gcp_mock]
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.PROVISIONING
+        aws_mock.compute.return_value.create_instance.assert_called_once()
+        assert instance.backend == BackendType.GCP
+
+    @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
+    async def test_fails_if_all_offers_fail(self, session: AsyncSession, err: Exception):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        aws_mock = Mock()
+        aws_mock.TYPE = BackendType.AWS
+        offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
+        aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
+        aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
+        aws_mock.compute.return_value.create_instance.side_effect = err
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = [aws_mock]
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.TERMINATED
+        assert instance.termination_reason == "All offers failed"
+
+    async def test_fails_if_no_offers(self, session: AsyncSession):
+        project = await create_project(session=session)
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.PENDING
+        )
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            m.return_value = []
+            await process_instances()
+
+        await session.refresh(instance)
+        assert instance.status == InstanceStatus.TERMINATED
+        assert instance.termination_reason == "No offers found"
+
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)

tests/_internal/server/routers/test_prometheus.py

@@ -9,14 +9,20 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
-from dstack._internal.core.models.runs import JobProvisioningData, JobRuntimeData, JobStatus
+from dstack._internal.core.models.runs import (
+    JobProvisioningData,
+    JobRuntimeData,
+    JobStatus,
+    RunStatus,
+)
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.models import JobModel, ProjectModel, UserModel
+from dstack._internal.server.models import JobModel, ProjectModel, RunModel, UserModel
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
     create_fleet,
     create_instance,
     create_job,
+    create_job_metrics_point,
     create_job_prometheus_metrics,
     create_project,
     create_repo,
@@ -45,11 +51,21 @@ class TestGetPrometheusMetrics:
     async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
         user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
         offer = get_instance_offer_with_availability(
-            instance_type="test-type", gpu_count=2, gpu_name="V4", price=12
+            instance_type="test-type",
+            cpu_count=32,
+            memory_gib=128,
+            gpu_count=2,
+            gpu_name="V4",
+            price=12,
         )
         project_2 = await _create_project(session, "project-2", user)
         jpd_2_1 = get_job_provisioning_data(
-            backend=BackendType.AWS, gpu_name="T4", gpu_count=2, price=16
+            backend=BackendType.AWS,
+            cpu_count=16,
+            memory_gib=64,
+            gpu_name="T4",
+            gpu_count=2,
+            price=16,
         )
         job_2_1 = await _create_job(
             session=session,
@@ -100,7 +116,41 @@ class TestGetPrometheusMetrics:
                 FIELD_2{gpu="1"} 987169 1395066363010
             """),
         )
-        job_1_2 = await _create_job(session, "run-2", project_1, user, JobStatus.RUNNING)
+        await create_job_metrics_point(
+            session=session,
+            job_model=job_1_1,
+            timestamp=FAKE_NOW - timedelta(seconds=30),
+            cpu_usage_micro=3_500_000,
+            memory_working_set_bytes=3_221_225_472,
+            memory_usage_bytes=4_294_967_296,
+        )
+        # Older, ignored
+        await create_job_metrics_point(
+            session=session,
+            job_model=job_1_1,
+            timestamp=FAKE_NOW - timedelta(seconds=60),
+            cpu_usage_micro=2_000_000,
+            memory_working_set_bytes=1_073_741_824,
+            memory_usage_bytes=2_147_483_648,
+        )
+        jpd_1_2 = get_job_provisioning_data(
+            backend=BackendType.AWS,
+            cpu_count=24,
+            memory_gib=224,
+            gpu_count=3,
+            gpu_name="L4",
+            price=12.5,
+        )
+        job_1_2 = await _create_job(
+            session=session,
+            run_name="run-2",
+            project=project_1,
+            user=user,
+            status=JobStatus.RUNNING,
+            job_provisioning_data=jpd_1_2,
+            submitted_at=FAKE_NOW - timedelta(seconds=150),
+        )
+
         await create_job_prometheus_metrics(
             session=session,
             job=job_1_2,
@@ -124,6 +174,15 @@ class TestGetPrometheusMetrics:
                 FIELD_1{gpu="1"} 20
             """),
         )
+        await _create_run(session, "done", project_1, user, RunStatus.DONE)
+        other_user = await create_user(
+            session=session, name="other-user", global_role=GlobalRole.USER
+        )
+        await add_project_member(
+            session=session, project=project_2, user=other_user, project_role=ProjectRole.USER
+        )
+        await _create_run(session, "failed-1", project_2, other_user, RunStatus.FAILED)
+        await _create_run(session, "failed-2", project_2, other_user, RunStatus.FAILED)
         fleet = await create_fleet(session=session, project=project_1, name="test-fleet")
         instance = await create_instance(
             session=session,
@@ -149,31 +208,73 @@ class TestGetPrometheusMetrics:
             # HELP dstack_instance_gpu_count Instance GPU count
             # TYPE dstack_instance_gpu_count gauge
             dstack_instance_gpu_count{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            # HELP dstack_run_count_total Total runs count
+            # TYPE dstack_run_count_total counter
+            dstack_run_count_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 4.0
+            dstack_run_count_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 2.0
+            dstack_run_count_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 1.0
+            # HELP dstack_run_count_terminated_total Terminated runs count
+            # TYPE dstack_run_count_terminated_total counter
+            dstack_run_count_terminated_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 0.0
+            dstack_run_count_terminated_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 0.0
+            dstack_run_count_terminated_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
+            # HELP dstack_run_count_failed_total Failed runs count
+            # TYPE dstack_run_count_failed_total counter
+            dstack_run_count_failed_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 0.0
+            dstack_run_count_failed_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 2.0
+            dstack_run_count_failed_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
+            # HELP dstack_run_count_done_total Done runs count
+            # TYPE dstack_run_count_done_total counter
+            dstack_run_count_done_total{{dstack_project_name="project-1",dstack_user_name="test-user"}} 1.0
+            dstack_run_count_done_total{{dstack_project_name="project-2",dstack_user_name="other-user"}} 0.0
+            dstack_run_count_done_total{{dstack_project_name="project-2",dstack_user_name="test-user"}} 0.0
             # HELP dstack_job_duration_seconds_total Total seconds the job is running
             # TYPE dstack_job_duration_seconds_total counter
             dstack_job_duration_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 120.0
+            dstack_job_duration_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 150.0
             dstack_job_duration_seconds_total{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 100.0
             # HELP dstack_job_price_dollars_per_hour Job instance price, USD/hour
             # TYPE dstack_job_price_dollars_per_hour gauge
             dstack_job_price_dollars_per_hour{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 12.0
+            dstack_job_price_dollars_per_hour{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 12.5
             dstack_job_price_dollars_per_hour{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 16.0
             # HELP dstack_job_gpu_count Job GPU count
             # TYPE dstack_job_gpu_count gauge
             dstack_job_gpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 2.0
+            dstack_job_gpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 3.0
             dstack_job_gpu_count{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 2.0
+            # HELP dstack_job_cpu_count Job CPU count
+            # TYPE dstack_job_cpu_count gauge
+            dstack_job_cpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 32.0
+            dstack_job_cpu_count{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 24.0
+            dstack_job_cpu_count{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 16.0
+            # HELP dstack_job_cpu_time_seconds_total Total CPU time consumed by the job, seconds
+            # TYPE dstack_job_cpu_time_seconds_total counter
+            dstack_job_cpu_time_seconds_total{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 3.5
+            # HELP dstack_job_memory_total_bytes Total memory allocated for the job, bytes
+            # TYPE dstack_job_memory_total_bytes gauge
+            dstack_job_memory_total_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 137438953472.0
+            dstack_job_memory_total_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 240518168576.0
+            dstack_job_memory_total_bytes{{dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 68719476736.0
+            # HELP dstack_job_memory_usage_bytes Memory used by the job (including cache), bytes
+            # TYPE dstack_job_memory_usage_bytes gauge
+            dstack_job_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 4294967296.0
+            # HELP dstack_job_memory_working_set_bytes Memory used by the job (not including cache), bytes
+            # TYPE dstack_job_memory_working_set_bytes gauge
+            dstack_job_memory_working_set_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 3221225472.0
             # HELP FIELD_1 Test field 1
             # TYPE FIELD_1 gauge
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
-            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 100.0
-            FIELD_1{{gpu="1",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 200.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 350.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 1200.0
+            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 1600.0
+            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_1_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_1_2.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="L4"}} 2400.0
+            FIELD_1{{gpu="0",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 100.0
+            FIELD_1{{gpu="1",dstack_project_name="project-2",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_2_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_2_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="T4"}} 200.0
             # HELP FIELD_2 Test field 2
             # TYPE FIELD_2 counter
-            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
-            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
+            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 337325.0 1395066363000
+            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 987169.0 1395066363010
         """)
 
     async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
@@ -189,110 +290,6 @@ class TestGetPrometheusMetrics:
         assert response.status_code == 404
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
-@pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
-class TestGetPrometheusProjectMetrics:
-    async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
-        user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
-        project = await _create_project(session, "project-1", user)
-        job_1 = await _create_job(session, "run-1", project, user, JobStatus.RUNNING)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_1,
-            text=dedent("""
-                # Comments should be skipped
-
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 350
-                FIELD_1{gpu="1"} 400
-
-                # HELP FIELD_2 Test field 2
-                # TYPE FIELD_2 counter
-                FIELD_2{gpu="0"} 337325 1395066363000
-                FIELD_2{gpu="1"} 987169 1395066363010
-            """),
-        )
-        job_2 = await _create_job(session, "run-2", project, user, JobStatus.RUNNING)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_2,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 1200.0
-                FIELD_1{gpu="1"} 1600.0
-                FIELD_1{gpu="2"} 2400.0
-            """),
-        )
-        # Terminated job, should not appear in the response
-        job_3 = await _create_job(session, "run-3", project, user, JobStatus.TERMINATED)
-        await create_job_prometheus_metrics(
-            session=session,
-            job=job_3,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 10
-                FIELD_1{gpu="1"} 20
-            """),
-        )
-        another_project = await _create_project(session, "project-2", user)
-        another_project_job = await _create_job(
-            session, "run-4", another_project, user, JobStatus.RUNNING
-        )
-        await create_job_prometheus_metrics(
-            session=session,
-            job=another_project_job,
-            text=dedent("""
-                # HELP FIELD_1 Test field 1
-                # TYPE FIELD_1 gauge
-                FIELD_1{gpu="0"} 100
-                FIELD_1{gpu="1"} 200
-            """),
-        )
-
-        response = await client.get("/metrics/project/project-1")
-
-        assert response.status_code == 200
-        assert response.text == dedent(f"""\
-            # HELP FIELD_1 Test field 1
-            # TYPE FIELD_1 gauge
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 350.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 400.0
-            FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1200.0
-            FIELD_1{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 1600.0
-            FIELD_1{{gpu="2",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-2",dstack_run_id="{job_2.run_id}",dstack_job_name="run-2-0-0",dstack_job_id="{job_2.id}",dstack_job_num="0",dstack_replica_num="0"}} 2400.0
-            # HELP FIELD_2 Test field 2
-            # TYPE FIELD_2 counter
-            FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 337325.0 1395066363000
-            FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1.id}",dstack_job_num="0",dstack_replica_num="0"}} 987169.0 1395066363010
-        """)
-
-    async def test_returns_empty_response_if_no_runs(
-        self, session: AsyncSession, client: AsyncClient
-    ):
-        user = await create_user(session=session, global_role=GlobalRole.USER)
-        await create_project(session=session, owner=user, name="test-project")
-        response = await client.get("/metrics/project/test-project")
-        assert response.status_code == 200
-        assert response.text == "\n"
-
-    async def test_returns_404_if_project_doesnt_exist(self, client: AsyncClient):
-        response = await client.get("/metrics/project/nonexistent")
-        assert response.status_code == 404
-
-    async def test_returns_404_if_not_enabled(
-        self, monkeypatch: pytest.MonkeyPatch, session: AsyncSession, client: AsyncClient
-    ):
-        monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", False)
-        user = await create_user(session=session, global_role=GlobalRole.USER)
-        await create_project(session=session, owner=user, name="test-project")
-        response = await client.get("/metrics/project/test-project")
-        assert response.status_code == 404
-
-
 async def _create_project(session: AsyncSession, name: str, user: UserModel) -> ProjectModel:
     project = await create_project(session=session, owner=user, name=name)
     await add_project_member(
@@ -301,26 +298,46 @@ async def _create_project(session: AsyncSession, name: str, user: UserModel) ->
     return project
 
 
-async def _create_job(
+async def _create_run(
     session: AsyncSession,
     run_name: str,
     project: ProjectModel,
     user: UserModel,
-    status: JobStatus,
-    job_provisioning_data: Optional[JobProvisioningData] = None,
-    job_runtime_data: Optional[JobRuntimeData] = None,
+    status: RunStatus,
     submitted_at: datetime = FAKE_NOW,
-) -> JobModel:
+) -> RunModel:
     repo = await create_repo(session=session, project_id=project.id, repo_name=f"{run_name}-repo")
     configuration = DevEnvironmentConfiguration(ide="vscode")
     run_spec = get_run_spec(run_name=run_name, repo_id=repo.name, configuration=configuration)
-    run = await create_run(
+    return await create_run(
         session=session,
         project=project,
         repo=repo,
         user=user,
         run_name=run_name,
         run_spec=run_spec,
+        status=status,
+        submitted_at=submitted_at,
+    )
+
+
+async def _create_job(
+    session: AsyncSession,
+    run_name: str,
+    project: ProjectModel,
+    user: UserModel,
+    status: JobStatus,
+    job_provisioning_data: Optional[JobProvisioningData] = None,
+    job_runtime_data: Optional[JobRuntimeData] = None,
+    submitted_at: datetime = FAKE_NOW,
+) -> JobModel:
+    run = await _create_run(
+        session=session,
+        run_name=run_name,
+        project=project,
+        user=user,
+        status=RunStatus.SUBMITTED,
+        submitted_at=submitted_at,
     )
     job = await create_job(
         session=session,