dstack 0.18.44__py3-none-any.whl → 0.19.0rc1__py3-none-any.whl

tests/_internal/server/background/tasks/test_process_submitted_jobs.py

@@ -14,7 +14,7 @@ from dstack._internal.core.models.instances import (
     InstanceType,
     Resources,
 )
-from dstack._internal.core.models.profiles import Profile, ProfileRetryPolicy
+from dstack._internal.core.models.profiles import Profile
 from dstack._internal.core.models.runs import (
     JobProvisioningData,
     JobStatus,
@@ -29,10 +29,10 @@ from dstack._internal.core.models.volumes import (
 from dstack._internal.server.background.tasks.process_submitted_jobs import process_submitted_jobs
 from dstack._internal.server.models import InstanceModel, JobModel, VolumeAttachmentModel
 from dstack._internal.server.testing.common import (
+    ComputeMockSpec,
     create_fleet,
     create_instance,
     create_job,
-    create_pool,
     create_project,
     create_repo,
     create_run,
@@ -52,7 +52,6 @@ class TestProcessSubmittedJobs:
     async def test_fails_job_when_no_backends(self, test_db, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -93,7 +92,6 @@ class TestProcessSubmittedJobs:
     ):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -151,12 +149,6 @@ class TestProcessSubmittedJobs:
         assert job is not None
         assert job.status == JobStatus.PROVISIONING
 
-        await session.refresh(pool)
-        instance_offer = InstanceOfferWithAvailability.parse_raw(pool.instances[0].offer)
-        assert offer == instance_offer
-        pool_job_provisioning_data = pool.instances[0].job_provisioning_data
-        assert pool_job_provisioning_data == job.job_provisioning_data
-
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_fails_job_when_privileged_true_and_no_offers_with_create_instance_support(
@@ -166,7 +158,6 @@ class TestProcessSubmittedJobs:
     ):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -227,9 +218,6 @@ class TestProcessSubmittedJobs:
         assert job.status == JobStatus.TERMINATING
         assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
 
-        await session.refresh(pool)
-        assert not pool.instances
-
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_fails_job_when_instance_mounts_and_no_offers_with_create_instance_support(
@@ -239,7 +227,6 @@ class TestProcessSubmittedJobs:
     ):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -300,9 +287,6 @@ class TestProcessSubmittedJobs:
         assert job.status == JobStatus.TERMINATING
         assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
 
-        await session.refresh(pool)
-        assert not pool.instances
-
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_provisions_job_with_optional_instance_volume_not_attached(
@@ -312,7 +296,6 @@ class TestProcessSubmittedJobs:
     ):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -369,18 +352,11 @@ class TestProcessSubmittedJobs:
         assert job is not None
         assert job.status == JobStatus.PROVISIONING
 
-        await session.refresh(pool)
-        instance_offer = InstanceOfferWithAvailability.parse_raw(pool.instances[0].offer)
-        assert offer == instance_offer
-        pool_job_provisioning_data = pool.instances[0].job_provisioning_data
-        assert pool_job_provisioning_data == job.job_provisioning_data
-
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_fails_job_when_no_capacity(self, test_db, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -396,7 +372,6 @@ class TestProcessSubmittedJobs:
                 repo_id=repo.name,
                 profile=Profile(
                     name="default",
-                    retry_policy=ProfileRetryPolicy(retry=True, duration=3600),
                 ),
             ),
         )
@@ -414,15 +389,12 @@ class TestProcessSubmittedJobs:
         assert job is not None
         assert job.status == JobStatus.TERMINATING
         assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
-        await session.refresh(pool)
-        assert not pool.instances
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_assignes_job_to_instance(self, test_db, session: AsyncSession):
         project = await create_project(session)
         user = await create_user(session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -430,10 +402,8 @@ class TestProcessSubmittedJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.IDLE,
         )
-        await session.refresh(pool)
         run = await create_run(
             session=session,
             project=project,
@@ -459,7 +429,6 @@ class TestProcessSubmittedJobs:
     async def test_assigns_job_to_instance_with_volumes(self, test_db, session: AsyncSession):
         project = await create_project(session)
         user = await create_user(session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -476,12 +445,10 @@ class TestProcessSubmittedJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.IDLE,
             backend=BackendType.AWS,
             region="us-east-1",
         )
-        await session.refresh(pool)
         run_spec = get_run_spec(run_name="test-run", repo_id=repo.name)
         run_spec.configuration.volumes = [
             VolumeMountPoint(name=volume.name, path="/volume"),
@@ -506,6 +473,7 @@ class TestProcessSubmittedJobs:
             backend_mock = Mock()
             m.return_value = backend_mock
             backend_mock.TYPE = BackendType.AWS
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData()
             # Submitted jobs processing happens in two steps
             await process_submitted_jobs()
@@ -532,7 +500,6 @@ class TestProcessSubmittedJobs:
     async def test_assigns_job_to_shared_instance(self, test_db, session: AsyncSession):
         project = await create_project(session)
         user = await create_user(session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -541,13 +508,11 @@ class TestProcessSubmittedJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.IDLE,
             offer=offer,
             total_blocks=4,
             busy_blocks=1,
         )
-        await session.refresh(pool)
         run = await create_run(
             session=session,
             project=project,
@@ -577,12 +542,10 @@ class TestProcessSubmittedJobs:
         project = await create_project(session)
         user = await create_user(session)
         repo = await create_repo(session=session, project_id=project.id)
-        pool = await create_pool(session=session, project=project)
         fleet = await create_fleet(session=session, project=project)
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             instance_num=0,
             status=InstanceStatus.BUSY,
         )

tests/_internal/server/background/tasks/test_process_submitted_volumes.py

@@ -7,6 +7,7 @@ from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.volumes import VolumeProvisioningData, VolumeStatus
 from dstack._internal.server.background.tasks.process_volumes import process_submitted_volumes
 from dstack._internal.server.testing.common import (
+    ComputeMockSpec,
     create_project,
     create_user,
     create_volume,
@@ -40,6 +41,7 @@ class TestProcessSubmittedVolumes:
         ) as m:
             aws_mock = Mock()
             m.return_value = aws_mock
+            aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             aws_mock.compute.return_value.create_volume.return_value = VolumeProvisioningData(
                 backend=BackendType.AWS,
                 volume_id="1234",

tests/_internal/server/background/tasks/test_process_terminating_jobs.py

@@ -16,9 +16,9 @@ from dstack._internal.server.background.tasks.process_terminating_jobs import (
 from dstack._internal.server.models import InstanceModel, JobModel, VolumeAttachmentModel
 from dstack._internal.server.services.volumes import volume_model_to_volume
 from dstack._internal.server.testing.common import (
+    ComputeMockSpec,
     create_instance,
     create_job,
-    create_pool,
     create_project,
     create_repo,
     create_run,
@@ -39,11 +39,9 @@ class TestProcessTerminatingJobs:
     async def test_terminates_job(self, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
         )
         repo = await create_repo(session=session, project_id=project.id)
@@ -78,7 +76,6 @@ class TestProcessTerminatingJobs:
     async def test_detaches_job_volumes(self, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         volume = await create_volume(
             session=session,
             project=project,
@@ -90,7 +87,6 @@ class TestProcessTerminatingJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
             volumes=[volume],
         )
@@ -114,6 +110,7 @@ class TestProcessTerminatingJobs:
         with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m:
             backend_mock = Mock()
             m.return_value = backend_mock
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.is_volume_detached.return_value = True
             await process_terminating_jobs()
             m.assert_awaited_once()
@@ -125,7 +122,6 @@ class TestProcessTerminatingJobs:
     async def test_force_detaches_job_volumes(self, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         volume = await create_volume(
             session=session,
             project=project,
@@ -137,7 +133,6 @@ class TestProcessTerminatingJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
             volumes=[volume],
         )
@@ -163,6 +158,7 @@ class TestProcessTerminatingJobs:
         with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m:
             backend_mock = Mock()
             m.return_value = backend_mock
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.is_volume_detached.return_value = False
             await process_terminating_jobs()
             m.assert_awaited_once()
@@ -188,6 +184,7 @@ class TestProcessTerminatingJobs:
             ) + timedelta(minutes=30)
             backend_mock = Mock()
             m.return_value = backend_mock
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.is_volume_detached.return_value = False
             await process_terminating_jobs()
             m.assert_awaited_once()
@@ -205,6 +202,7 @@ class TestProcessTerminatingJobs:
         with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m:
             backend_mock = Mock()
             m.return_value = backend_mock
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.is_volume_detached.return_value = True
             await process_terminating_jobs()
             m.assert_awaited_once()
@@ -223,7 +221,6 @@ class TestProcessTerminatingJobs:
     async def test_terminates_job_on_shared_instance(self, session: AsyncSession):
         project = await create_project(session)
         user = await create_user(session)
-        pool = await create_pool(session=session, project=project)
         repo = await create_repo(
             session=session,
             project_id=project.id,
@@ -231,12 +228,10 @@ class TestProcessTerminatingJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
             total_blocks=4,
             busy_blocks=3,
         )
-        await session.refresh(pool)
         run = await create_run(
             session=session,
             project=project,
@@ -270,7 +265,6 @@ class TestProcessTerminatingJobs:
     async def test_detaches_job_volumes_on_shared_instance(self, session: AsyncSession):
         project = await create_project(session=session)
         user = await create_user(session=session)
-        pool = await create_pool(session=session, project=project)
         volume_conf_1 = get_volume_configuration(name="vol-1")
         volume_1 = await create_volume(
             session=session,
@@ -294,7 +288,6 @@ class TestProcessTerminatingJobs:
         instance = await create_instance(
             session=session,
             project=project,
-            pool=pool,
             status=InstanceStatus.BUSY,
             volumes=[volume_1, volume_2],
         )
@@ -319,13 +312,15 @@ class TestProcessTerminatingJobs:
         with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m:
             backend_mock = Mock()
             m.return_value = backend_mock
+            backend_mock.compute.return_value = Mock(spec=ComputeMockSpec)
             backend_mock.compute.return_value.is_volume_detached.return_value = True
 
             await process_terminating_jobs()
 
-        m.assert_awaited_once()
-        backend_mock.compute.return_value.detach_volume.assert_called_once()
-        backend_mock.compute.return_value.is_volume_detached.assert_called_once()
+            m.assert_awaited_once()
+            backend_mock.compute.return_value.detach_volume.assert_called_once()
+            backend_mock.compute.return_value.is_volume_detached.assert_called_once()
+
         await session.refresh(job)
         await session.refresh(instance)
         assert job.status == JobStatus.TERMINATED