dstack 0.18.40rc1__py3-none-any.whl → 0.18.42__py3-none-any.whl

tests/_internal/server/routers/test_fleets.py

@@ -340,6 +340,7 @@ class TestCreateFleet:
                     },
                     "backends": None,
                     "regions": None,
+                    "availability_zones": None,
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
@@ -350,10 +351,12 @@ class TestCreateFleet:
                     "type": "fleet",
                     "name": "test-fleet",
                     "reservation": None,
+                    "blocks": 1,
                 },
                 "profile": {
                     "backends": None,
                     "regions": None,
+                    "availability_zones": None,
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
@@ -393,15 +396,18 @@ class TestCreateFleet:
                     "pool_name": None,
                     "backend": None,
                     "region": None,
+                    "availability_zone": None,
                     "instance_type": None,
                     "price": None,
+                    "total_blocks": 1,
+                    "busy_blocks": 0,
                 }
             ],
         }
         res = await session.execute(select(FleetModel))
         assert res.scalar_one()
         res = await session.execute(select(InstanceModel))
-        assert res.scalar_one()
+        assert res.unique().scalar_one()
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
@@ -444,6 +450,7 @@ class TestCreateFleet:
                         "port": None,
                         "identity_file": None,
                         "ssh_key": None,  # should not return ssh_key
+                        "proxy_jump": None,
                         "hosts": ["1.1.1.1"],
                         "network": None,
                     },
@@ -458,6 +465,7 @@ class TestCreateFleet:
                     },
                     "backends": None,
                     "regions": None,
+                    "availability_zones": None,
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
@@ -468,10 +476,12 @@ class TestCreateFleet:
                     "type": "fleet",
                     "name": spec.configuration.name,
                     "reservation": None,
+                    "blocks": 1,
                 },
                 "profile": {
                     "backends": None,
                     "regions": None,
+                    "availability_zones": None,
                     "instance_types": None,
                     "spot_policy": None,
                     "retry": None,
@@ -522,14 +532,17 @@ class TestCreateFleet:
                     "termination_reason": None,
                     "created": "2023-01-02T03:04:00+00:00",
                     "region": "remote",
+                    "availability_zone": None,
                     "price": 0.0,
+                    "total_blocks": 1,
+                    "busy_blocks": 0,
                 }
             ],
         }
         res = await session.execute(select(FleetModel))
         assert res.scalar_one()
         res = await session.execute(select(InstanceModel))
-        instance = res.scalar_one()
+        instance = res.unique().scalar_one()
         assert instance.remote_connection_info is not None
 
     @pytest.mark.asyncio

tests/_internal/server/routers/test_pools.py

@@ -332,7 +332,10 @@ class TestShowPool:
                     "created": "2023-01-02T03:04:00+00:00",
                     "pool_name": None,
                     "region": "en",
+                    "availability_zone": None,
                     "price": 1,
+                    "total_blocks": 1,
+                    "busy_blocks": 0,
                 }
             ],
         }
@@ -503,7 +506,10 @@ class TestRemoveInstance:
                     "created": "2023-01-02T03:04:00+00:00",
                     "pool_name": None,
                     "region": "en",
+                    "availability_zone": None,
                     "price": 1,
+                    "total_blocks": 1,
+                    "busy_blocks": 0,
                 }
             ],
         }

tests/_internal/server/routers/test_runs.py

@@ -18,6 +18,7 @@ from dstack._internal.core.models.gateways import GatewayStatus
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
     InstanceOfferWithAvailability,
+    InstanceStatus,
     InstanceType,
     Resources,
 )
@@ -47,7 +48,9 @@ from dstack._internal.server.testing.common import (
     create_backend,
     create_gateway,
     create_gateway_compute,
+    create_instance,
     create_job,
+    create_pool,
     create_project,
     create_repo,
     create_run,
@@ -85,6 +88,7 @@ def get_dev_env_run_plan_dict(
                 "working_dir": None,
                 "home_dir": "/root",
                 "ide": "vscode",
+                "inactivity_duration": None,
                 "version": None,
                 "image": None,
                 "user": None,
@@ -107,6 +111,7 @@ def get_dev_env_run_plan_dict(
                 "volumes": [json.loads(v.json()) for v in volumes],
                 "backends": ["local", "aws", "azure", "gcp", "lambda", "runpod"],
                 "regions": ["us"],
+                "availability_zones": None,
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
@@ -127,6 +132,7 @@ def get_dev_env_run_plan_dict(
             "profile": {
                 "backends": ["local", "aws", "azure", "gcp", "lambda", "runpod"],
                 "regions": ["us"],
+                "availability_zones": None,
                 "instance_types": None,
                 "creation_policy": None,
                 "default": False,
@@ -198,6 +204,7 @@ def get_dev_env_run_plan_dict(
                         "reservation": None,
                     },
                     "retry": None,
+                    "volumes": volumes,
                     "retry_policy": {"retry": False, "duration": None},
                     "working_dir": ".",
                 },
@@ -238,6 +245,7 @@ def get_dev_env_run_dict(
                 "home_dir": "/root",
                 "working_dir": None,
                 "ide": "vscode",
+                "inactivity_duration": None,
                 "version": None,
                 "image": None,
                 "user": None,
@@ -260,6 +268,7 @@ def get_dev_env_run_dict(
                 "volumes": [],
                 "backends": ["local", "aws", "azure", "gcp", "lambda"],
                 "regions": ["us"],
+                "availability_zones": None,
                 "instance_types": None,
                 "creation_policy": None,
                 "instance_name": None,
@@ -280,6 +289,7 @@ def get_dev_env_run_dict(
             "profile": {
                 "backends": ["local", "aws", "azure", "gcp", "lambda"],
                 "regions": ["us"],
+                "availability_zones": None,
                 "instance_types": None,
                 "creation_policy": None,
                 "default": False,
@@ -351,6 +361,7 @@ def get_dev_env_run_dict(
                         "reservation": None,
                     },
                     "retry": None,
+                    "volumes": [],
                     "retry_policy": {"retry": False, "duration": None},
                     "working_dir": ".",
                 },
@@ -361,6 +372,7 @@ def get_dev_env_run_dict(
                         "submitted_at": submitted_at,
                         "last_processed_at": last_processed_at,
                         "finished_at": finished_at,
+                        "inactivity_secs": None,
                         "status": "submitted",
                         "termination_reason": None,
                         "termination_reason_message": None,
@@ -375,6 +387,7 @@ def get_dev_env_run_dict(
             "submission_num": 0,
             "submitted_at": submitted_at,
             "last_processed_at": last_processed_at,
+            "inactivity_secs": None,
             "finished_at": finished_at,
             "status": "submitted",
             "termination_reason": None,
@@ -487,6 +500,7 @@ class TestListRuns:
                                 "submitted_at": run1_submitted_at.isoformat(),
                                 "last_processed_at": run1_submitted_at.isoformat(),
                                 "finished_at": None,
+                                "inactivity_secs": None,
                                 "status": "submitted",
                                 "termination_reason": None,
                                 "termination_reason_message": None,
@@ -502,6 +516,7 @@ class TestListRuns:
                     "submitted_at": run1_submitted_at.isoformat(),
                     "last_processed_at": run1_submitted_at.isoformat(),
                     "finished_at": None,
+                    "inactivity_secs": None,
                     "status": "submitted",
                     "termination_reason_message": None,
                     "termination_reason": None,
@@ -1303,11 +1318,20 @@ class TestStopRuns:
             user=user,
             status=RunStatus.RUNNING,
         )
+        pool = await create_pool(session=session, project=project)
+        instance = await create_instance(
+            session=session,
+            project=project,
+            pool=pool,
+            status=InstanceStatus.BUSY,
+        )
         job = await create_job(
             session=session,
             run=run,
             job_provisioning_data=get_job_provisioning_data(),
             status=JobStatus.RUNNING,
+            instance=instance,
+            instance_assigned=True,
         )
         with patch("dstack._internal.server.services.jobs._stop_runner") as stop_runner:
             response = await client.post(
@@ -1533,7 +1557,10 @@ class TestCreateInstance:
                 "created": result["created"],
                 "pool_name": None,
                 "region": None,
+                "availability_zone": None,
                 "price": None,
+                "total_blocks": 1,
+                "busy_blocks": 0,
             }
             assert result == expected
 

tests/_internal/server/routers/test_volumes.py

@@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.models import VolumeModel
+from dstack._internal.server.models import VolumeAttachmentModel, VolumeModel
 from dstack._internal.server.services.projects import add_project_member
 from dstack._internal.server.testing.common import (
     create_instance,
@@ -76,6 +76,7 @@ class TestListVolumes:
                 "deleted": False,
                 "volume_id": None,
                 "provisioning_data": None,
+                "attachments": [],
                 "attachment_data": None,
             },
             {
@@ -91,6 +92,7 @@ class TestListVolumes:
                 "deleted": False,
                 "volume_id": None,
                 "provisioning_data": None,
+                "attachments": [],
                 "attachment_data": None,
             },
         ]
@@ -117,6 +119,7 @@ class TestListVolumes:
                 "deleted": False,
                 "volume_id": None,
                 "provisioning_data": None,
+                "attachments": [],
                 "attachment_data": None,
             },
         ]
@@ -170,6 +173,7 @@ class TestListVolumes:
                 "deleted": False,
                 "volume_id": None,
                 "provisioning_data": None,
+                "attachments": [],
                 "attachment_data": None,
             },
         ]
@@ -217,6 +221,7 @@ class TestListProjectVolumes:
                 "deleted": False,
                 "volume_id": None,
                 "provisioning_data": None,
+                "attachments": [],
                 "attachment_data": None,
             }
         ]
@@ -264,6 +269,7 @@ class TestGetVolume:
             "deleted": False,
             "volume_id": None,
             "provisioning_data": None,
+            "attachments": [],
             "attachment_data": None,
         }
 
@@ -325,6 +331,7 @@ class TestCreateVolume:
             "deleted": False,
             "volume_id": None,
             "provisioning_data": None,
+            "attachments": [],
             "attachment_data": None,
         }
         res = await session.execute(select(VolumeModel))
@@ -391,7 +398,7 @@ class TestDeleteVolumes:
             project=project,
             pool=pool,
         )
-        volume.instances.append(instance)
+        volume.attachments.append(VolumeAttachmentModel(instance=instance))
         await session.commit()
         response = await client.post(
             f"/api/project/{project.name}/volumes/delete",

tests/_internal/server/services/jobs/configurators/test_base.py

@@ -0,0 +1,72 @@
+from typing import Union
+
+import pytest
+
+from dstack._internal.core.errors import ServerClientError
+from dstack._internal.core.models.volumes import InstanceMountPoint, MountPoint, VolumeMountPoint
+from dstack._internal.server.services.jobs.configurators.base import interpolate_job_volumes
+
+
+class TestInterpolateJobVolumes:
+    @pytest.mark.parametrize(
+        ["run_volumes", "job_num", "job_volumes"],
+        [
+            pytest.param(
+                [VolumeMountPoint(name="volume", path="/volume")],
+                0,
+                [VolumeMountPoint(name=["volume"], path="/volume")],
+                id="no_interpolation",
+            ),
+            pytest.param(
+                [InstanceMountPoint(instance_path="/volume", path="/volume")],
+                0,
+                [InstanceMountPoint(instance_path="/volume", path="/volume")],
+                id="instance_mount",
+            ),
+            pytest.param(
+                [
+                    VolumeMountPoint(
+                        name="job${{dstack.job_num}}-rank${{dstack.node_rank}}", path="/volume"
+                    )
+                ],
+                2,
+                [VolumeMountPoint(name=["job2-rank2"], path="/volume")],
+                id="job_num_and_node_rank",
+            ),
+        ],
+    )
+    def test_interpolates_volumes(
+        self,
+        run_volumes: list[Union[MountPoint, str]],
+        job_num: int,
+        job_volumes: list[MountPoint],
+    ):
+        assert interpolate_job_volumes(run_volumes, job_num) == job_volumes
+
+    @pytest.mark.parametrize(
+        ["run_volumes", "job_num"],
+        [
+            pytest.param(
+                [VolumeMountPoint(name="${{}", path="/volume")],
+                0,
+                id="invalid_syntax",
+            ),
+            pytest.param(
+                [VolumeMountPoint(name="${{ unknown.namespace }}", path="/volume")],
+                0,
+                id="unknown_namespace",
+            ),
+            pytest.param(
+                [VolumeMountPoint(name="${{ dstack.var }}", path="/volume")],
+                0,
+                id="unknown_var",
+            ),
+        ],
+    )
+    def test_raises_server_client_error(
+        self,
+        run_volumes: list[Union[MountPoint, str]],
+        job_num: int,
+    ):
+        with pytest.raises(ServerClientError):
+            assert interpolate_job_volumes(run_volumes, job_num)

tests/_internal/server/services/runner/test_client.py

@@ -9,7 +9,13 @@ from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import NetworkMode
 from dstack._internal.core.models.resources import Memory
-from dstack._internal.core.models.volumes import InstanceMountPoint, VolumeMountPoint
+from dstack._internal.core.models.volumes import (
+    InstanceMountPoint,
+    VolumeAttachment,
+    VolumeAttachmentData,
+    VolumeInstance,
+    VolumeMountPoint,
+)
 from dstack._internal.server.schemas.runner import (
     HealthcheckResponse,
     JobResult,
@@ -133,7 +139,12 @@ class TestShimClientV1(BaseShimClientTest):
             volume_id="vol-id",
             configuration=get_volume_configuration(backend=BackendType.GCP),
             external=False,
-            device_name="/dev/sdv",
+            attachments=[
+                VolumeAttachment(
+                    instance=VolumeInstance(name="instance", instance_num=0, instance_id="i-1"),
+                    attachment_data=VolumeAttachmentData(device_name="/dev/sdv"),
+                )
+            ],
         )
 
         submitted = client.submit(
@@ -150,6 +161,7 @@ class TestShimClientV1(BaseShimClientTest):
             mounts=[VolumeMountPoint(name="vol", path="/vol")],
             volumes=[volume],
             instance_mounts=[InstanceMountPoint(instance_path="/mnt/nfs/home", path="/home")],
+            instance_id="i-1",
         )
 
         assert submitted is True
@@ -198,6 +210,7 @@ class TestShimClientV1(BaseShimClientTest):
             mounts=[],
             volumes=[],
             instance_mounts=[],
+            instance_id="",
         )
 
         assert submitted is False
@@ -294,7 +307,12 @@ class TestShimClientV2(BaseShimClientTest):
             volume_id="vol-id",
             configuration=get_volume_configuration(backend=BackendType.GCP),
             external=False,
-            device_name="/dev/sdv",
+            attachments=[
+                VolumeAttachment(
+                    instance=VolumeInstance(name="instance", instance_num=0, instance_id="i-1"),
+                    attachment_data=VolumeAttachmentData(device_name="/dev/sdv"),
+                )
+            ],
         )
 
         client.submit_task(
@@ -316,6 +334,7 @@ class TestShimClientV2(BaseShimClientTest):
             host_ssh_user="dstack",
             host_ssh_keys=["host_key"],
             container_ssh_keys=["project_key", "user_key"],
+            instance_id="i-1",
         )
 
         assert adapter.call_count == 2

tests/_internal/server/services/test_offers.py

@@ -0,0 +1,167 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.profiles import Profile
+from dstack._internal.core.models.resources import ResourcesSpec
+from dstack._internal.core.models.runs import Requirements
+from dstack._internal.server.services.offers import get_offers_by_requirements
+from dstack._internal.server.testing.common import (
+    get_instance_offer_with_availability,
+    get_volume,
+    get_volume_configuration,
+)
+
+
+class TestGetOffersByRequirements:
+    @pytest.mark.asyncio
+    async def test_returns_all_offers(self):
+        profile = Profile(name="test")
+        requirements = Requirements(resources=ResourcesSpec())
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            aws_backend_mock = Mock()
+            aws_backend_mock.TYPE = BackendType.AWS
+            aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS)
+            aws_backend_mock.compute.return_value.get_offers_cached.return_value = [aws_offer]
+            runpod_backend_mock = Mock()
+            runpod_backend_mock.TYPE = BackendType.RUNPOD
+            runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD)
+            runpod_backend_mock.compute.return_value.get_offers_cached.return_value = [
+                runpod_offer
+            ]
+            m.return_value = [aws_backend_mock, runpod_backend_mock]
+            res = await get_offers_by_requirements(
+                project=Mock(),
+                profile=profile,
+                requirements=requirements,
+            )
+            m.assert_awaited_once()
+            assert res == [(aws_backend_mock, aws_offer), (runpod_backend_mock, runpod_offer)]
+
+    @pytest.mark.asyncio
+    async def test_returns_multinode_offers(self):
+        profile = Profile(name="test")
+        requirements = Requirements(resources=ResourcesSpec())
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            aws_backend_mock = Mock()
+            aws_backend_mock.TYPE = BackendType.AWS
+            aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS)
+            aws_backend_mock.compute.return_value.get_offers_cached.return_value = [aws_offer]
+            runpod_backend_mock = Mock()
+            runpod_backend_mock.TYPE = BackendType.RUNPOD
+            runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD)
+            runpod_backend_mock.compute.return_value.get_offers_cached.return_value = [
+                runpod_offer
+            ]
+            m.return_value = [aws_backend_mock, runpod_backend_mock]
+            res = await get_offers_by_requirements(
+                project=Mock(),
+                profile=profile,
+                requirements=requirements,
+                multinode=True,
+            )
+            m.assert_awaited_once()
+            assert res == [(aws_backend_mock, aws_offer)]
+
+    @pytest.mark.asyncio
+    async def test_returns_volume_offers(self):
+        profile = Profile(name="test")
+        requirements = Requirements(resources=ResourcesSpec())
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            aws_backend_mock = Mock()
+            aws_backend_mock.TYPE = BackendType.AWS
+            aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS)
+            aws_backend_mock.compute.return_value.get_offers_cached.return_value = [aws_offer]
+            runpod_backend_mock = Mock()
+            runpod_backend_mock.TYPE = BackendType.RUNPOD
+            runpod_offer1 = get_instance_offer_with_availability(
+                backend=BackendType.RUNPOD, region="eu"
+            )
+            runpod_offer2 = get_instance_offer_with_availability(
+                backend=BackendType.RUNPOD, region="us"
+            )
+            runpod_backend_mock.compute.return_value.get_offers_cached.return_value = [
+                runpod_offer1,
+                runpod_offer2,
+            ]
+            m.return_value = [aws_backend_mock, runpod_backend_mock]
+            res = await get_offers_by_requirements(
+                project=Mock(),
+                profile=profile,
+                requirements=requirements,
+                volumes=[
+                    [
+                        get_volume(
+                            configuration=get_volume_configuration(
+                                backend=BackendType.RUNPOD, region="us"
+                            )
+                        )
+                    ]
+                ],
+            )
+            m.assert_awaited_once()
+            assert res == [(runpod_backend_mock, runpod_offer2)]
+
+    @pytest.mark.asyncio
+    async def test_returns_az_offers(self):
+        profile = Profile(name="test", availability_zones=["az1", "az3"])
+        requirements = Requirements(resources=ResourcesSpec())
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            aws_backend_mock = Mock()
+            aws_backend_mock.TYPE = BackendType.AWS
+            aws_offer1 = get_instance_offer_with_availability(
+                backend=BackendType.AWS, availability_zones=["az1"]
+            )
+            aws_offer2 = get_instance_offer_with_availability(
+                backend=BackendType.AWS, availability_zones=["az2"]
+            )
+            aws_offer3 = get_instance_offer_with_availability(
+                backend=BackendType.AWS, availability_zones=["az2", "az3"]
+            )
+            expected_aws_offer3 = aws_offer3.copy()
+            expected_aws_offer3.availability_zones = ["az3"]
+            aws_offer4 = get_instance_offer_with_availability(
+                backend=BackendType.AWS, availability_zones=None
+            )
+            aws_backend_mock.compute.return_value.get_offers_cached.return_value = [
+                aws_offer1,
+                aws_offer2,
+                aws_offer3,
+                aws_offer4,
+            ]
+            m.return_value = [aws_backend_mock]
+            res = await get_offers_by_requirements(
+                project=Mock(),
+                profile=profile,
+                requirements=requirements,
+            )
+            m.assert_awaited_once()
+            assert res == [(aws_backend_mock, aws_offer1), (aws_backend_mock, expected_aws_offer3)]
+
+    @pytest.mark.asyncio
+    async def test_returns_no_offers_for_multinode_instance_mounts_and_non_multinode_backend(self):
+        # Regression test for https://github.com/dstackai/dstack/issues/2211
+        profile = Profile(name="test", backends=[BackendType.RUNPOD])
+        requirements = Requirements(resources=ResourcesSpec())
+        with patch("dstack._internal.server.services.backends.get_project_backends") as m:
+            aws_backend_mock = Mock()
+            aws_backend_mock.TYPE = BackendType.AWS
+            aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS)
+            aws_backend_mock.compute.return_value.get_offers_cached.return_value = [aws_offer]
+            runpod_backend_mock = Mock()
+            runpod_backend_mock.TYPE = BackendType.RUNPOD
+            runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD)
+            runpod_backend_mock.compute.return_value.get_offers_cached.return_value = [
+                runpod_offer
+            ]
+            m.return_value = [aws_backend_mock, runpod_backend_mock]
+            res = await get_offers_by_requirements(
+                project=Mock(),
+                profile=profile,
+                requirements=requirements,
+                multinode=True,
+                instance_mounts=True,
+            )
+            m.assert_awaited_once()
+            assert res == []