dstack 0.19.4rc3__py3-none-any.whl → 0.19.6rc1__py3-none-any.whl

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm import joinedload
 from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT
 from dstack._internal.core.errors import GatewayError
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import NetworkMode, RegistryAuth, is_core_model_instance
+from dstack._internal.core.models.common import NetworkMode, RegistryAuth
 from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
 from dstack._internal.core.models.instances import (
     InstanceStatus,
@@ -40,7 +40,7 @@ from dstack._internal.server.models import (
     RepoModel,
     RunModel,
 )
-from dstack._internal.server.schemas.runner import TaskStatus
+from dstack._internal.server.schemas.runner import GPUDevice, TaskStatus
 from dstack._internal.server.services import logs as logs_services
 from dstack._internal.server.services import services
 from dstack._internal.server.services.instances import get_instance_ssh_private_keys
@@ -422,9 +422,9 @@ def _process_provisioning_with_shim(
     volume_mounts: List[VolumeMountPoint] = []
     instance_mounts: List[InstanceMountPoint] = []
     for mount in run.run_spec.configuration.volumes:
-        if is_core_model_instance(mount, VolumeMountPoint):
+        if isinstance(mount, VolumeMountPoint):
             volume_mounts.append(mount.copy())
-        elif is_core_model_instance(mount, InstanceMountPoint):
+        elif isinstance(mount, InstanceMountPoint):
             instance_mounts.append(mount)
         else:
             assert False, f"unexpected mount point: {mount!r}"
@@ -438,6 +438,10 @@ def _process_provisioning_with_shim(
         job_provisioning_data.backend, job_provisioning_data.instance_type.name
     )
 
+    gpu_devices = _get_instance_specific_gpu_devices(
+        job_provisioning_data.backend, job_provisioning_data.instance_type.name
+    )
+
     container_user = "root"
 
     job_runtime_data = get_job_runtime_data(job_model)
@@ -471,6 +475,7 @@ def _process_provisioning_with_shim(
             volumes=volumes,
             volume_mounts=volume_mounts,
             instance_mounts=instance_mounts,
+            gpu_devices=gpu_devices,
             host_ssh_user=ssh_user,
             host_ssh_keys=[ssh_key] if ssh_key else [],
             container_ssh_keys=public_keys,
@@ -657,7 +662,7 @@ def _terminate_if_inactivity_duration_exceeded(
     run_model: RunModel, job_model: JobModel, no_connections_secs: Optional[int]
 ) -> None:
     conf = RunSpec.__response__.parse_raw(run_model.run_spec).configuration
-    if not is_core_model_instance(conf, DevEnvironmentConfiguration) or not isinstance(
+    if not isinstance(conf, DevEnvironmentConfiguration) or not isinstance(
         conf.inactivity_duration, int
     ):
         # reset in case inactivity_duration was disabled via in-place update
@@ -834,14 +839,60 @@ def _submit_job_to_runner(
 def _get_instance_specific_mounts(
     backend_type: BackendType, instance_type_name: str
 ) -> List[InstanceMountPoint]:
-    if backend_type == BackendType.GCP and instance_type_name == "a3-megagpu-8g":
-        return [
-            InstanceMountPoint(
-                instance_path="/dev/aperture_devices", path="/dev/aperture_devices"
-            ),
-            InstanceMountPoint(instance_path="/var/lib/tcpxo/lib64", path="/var/lib/tcpxo/lib64"),
-            InstanceMountPoint(
-                instance_path="/var/lib/fastrak/lib64", path="/var/lib/fastrak/lib64"
-            ),
-        ]
+    if backend_type == BackendType.GCP:
+        if instance_type_name == "a3-megagpu-8g":
+            return [
+                InstanceMountPoint(
+                    instance_path="/dev/aperture_devices",
+                    path="/dev/aperture_devices",
+                ),
+                InstanceMountPoint(
+                    instance_path="/var/lib/tcpxo/lib64",
+                    path="/var/lib/tcpxo/lib64",
+                ),
+                InstanceMountPoint(
+                    instance_path="/var/lib/fastrak/lib64",
+                    path="/var/lib/fastrak/lib64",
+                ),
+            ]
+        if instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]:
+            return [
+                InstanceMountPoint(
+                    instance_path="/var/lib/nvidia/lib64",
+                    path="/usr/local/nvidia/lib64",
+                ),
+                InstanceMountPoint(
+                    instance_path="/var/lib/nvidia/bin",
+                    path="/usr/local/nvidia/bin",
+                ),
+                InstanceMountPoint(
+                    instance_path="/var/lib/tcpx/lib64",
+                    path="/usr/local/tcpx/lib64",
+                ),
+                InstanceMountPoint(
+                    instance_path="/run/tcpx",
+                    path="/run/tcpx",
+                ),
+            ]
     return []
+
+
+def _get_instance_specific_gpu_devices(
+    backend_type: BackendType, instance_type_name: str
+) -> List[GPUDevice]:
+    gpu_devices = []
+    if backend_type == BackendType.GCP and instance_type_name in [
+        "a3-edgegpu-8g",
+        "a3-highgpu-8g",
+    ]:
+        for i in range(8):
+            gpu_devices.append(
+                GPUDevice(path_on_host=f"/dev/nvidia{i}", path_in_container=f"/dev/nvidia{i}")
+            )
+        gpu_devices.append(
+            GPUDevice(path_on_host="/dev/nvidia-uvm", path_in_container="/dev/nvidia-uvm")
+        )
+        gpu_devices.append(
+            GPUDevice(path_on_host="/dev/nvidiactl", path_in_container="/dev/nvidiactl")
+        )
+    return gpu_devices

dstack/_internal/server/routers/fleets.py

@@ -9,6 +9,7 @@ from dstack._internal.core.models.fleets import Fleet, FleetPlan
 from dstack._internal.server.db import get_session
 from dstack._internal.server.models import ProjectModel, UserModel
 from dstack._internal.server.schemas.fleets import (
+    ApplyFleetPlanRequest,
     CreateFleetRequest,
     DeleteFleetInstancesRequest,
     DeleteFleetsRequest,
@@ -107,6 +108,27 @@ async def get_plan(
     return plan
 
 
+@project_router.post("/apply")
+async def apply_plan(
+    body: ApplyFleetPlanRequest,
+    session: AsyncSession = Depends(get_session),
+    user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
+) -> Fleet:
+    """
+    Creates a new fleet or updates an existing fleet.
+    Errors if the expected current resource from the plan does not match the current resource.
+    Use `force: true` to apply even if the current resource does not match.
+    """
+    user, project = user_project
+    return await fleets_services.apply_plan(
+        session=session,
+        user=user,
+        project=project,
+        plan=body.plan,
+        force=body.force,
+    )
+
+
 @project_router.post("/create")
 async def create_fleet(
     body: CreateFleetRequest,

dstack/_internal/server/routers/runs.py

@@ -100,6 +100,7 @@ async def get_plan(
         project=project,
         user=user,
         run_spec=body.run_spec,
+        max_offers=body.max_offers,
     )
     return run_plan
 

dstack/_internal/server/schemas/fleets.py

@@ -1,11 +1,11 @@
 from datetime import datetime
-from typing import List, Optional
+from typing import Annotated, List, Optional
 from uuid import UUID
 
 from pydantic import Field
 
 from dstack._internal.core.models.common import CoreModel
-from dstack._internal.core.models.fleets import FleetSpec
+from dstack._internal.core.models.fleets import ApplyFleetPlanInput, FleetSpec
 
 
 class ListFleetsRequest(CoreModel):
@@ -26,6 +26,16 @@ class GetFleetPlanRequest(CoreModel):
     spec: FleetSpec
 
 
+class ApplyFleetPlanRequest(CoreModel):
+    plan: ApplyFleetPlanInput
+    force: Annotated[
+        bool,
+        Field(
+            description="Use `force: true` to apply even if the expected resource does not match."
+        ),
+    ]
+
+
 class CreateFleetRequest(CoreModel):
     spec: FleetSpec
 

dstack/_internal/server/schemas/runner.py

@@ -114,6 +114,11 @@ class TaskStatus(str, Enum):
     TERMINATED = "terminated"
 
 
+class GPUDevice(CoreModel):
+    path_on_host: str
+    path_in_container: str
+
+
 class TaskInfoResponse(CoreModel):
     id: str
     status: TaskStatus
@@ -139,6 +144,7 @@ class TaskSubmitRequest(CoreModel):
     volumes: list[ShimVolumeInfo]
     volume_mounts: list[VolumeMountPoint]
     instance_mounts: list[InstanceMountPoint]
+    gpu_devices: list[GPUDevice]
     host_ssh_user: str
     host_ssh_keys: list[str]
     container_ssh_keys: list[str]

dstack/_internal/server/schemas/runs.py

@@ -26,6 +26,9 @@ class GetRunRequest(CoreModel):
 
 class GetRunPlanRequest(CoreModel):
     run_spec: RunSpec
+    max_offers: Optional[int] = Field(
+        description="The maximum number of offers to return", ge=1, le=10000
+    )
 
 
 class SubmitRunRequest(CoreModel):

dstack/_internal/server/services/docker.py

@@ -91,8 +91,7 @@ def get_image_config(image_name: str, registry_auth: Optional[RegistryAuth]) ->
             config_resp = join_byte_stream_checked(config_stream, MAX_CONFIG_OBJECT_SIZE)
             if config_resp is None:
                 raise DockerRegistryError(
-                    "Image config object exceeds the size limit of "
-                    f"{MAX_CONFIG_OBJECT_SIZE} bytes"
+                    f"Image config object exceeds the size limit of {MAX_CONFIG_OBJECT_SIZE} bytes"
                 )
             return ImageConfigObject.__response__.parse_raw(config_resp)
 

dstack/_internal/server/services/fleets.py

@@ -15,9 +15,9 @@ from dstack._internal.core.errors import (
     ResourceExistsError,
     ServerClientError,
 )
-from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.envs import Env
 from dstack._internal.core.models.fleets import (
+    ApplyFleetPlanInput,
     Fleet,
     FleetPlan,
     FleetSpec,
@@ -234,32 +234,34 @@ async def get_plan(
     user: UserModel,
     spec: FleetSpec,
 ) -> FleetPlan:
+    effective_spec = FleetSpec.parse_obj(spec.dict())
     current_fleet: Optional[Fleet] = None
     current_fleet_id: Optional[uuid.UUID] = None
-    if spec.configuration.name is not None:
+    if effective_spec.configuration.name is not None:
         current_fleet_model = await get_project_fleet_model_by_name(
-            session=session, project=project, name=spec.configuration.name
+            session=session, project=project, name=effective_spec.configuration.name
         )
         if current_fleet_model is not None:
             current_fleet = fleet_model_to_fleet(current_fleet_model)
             current_fleet_id = current_fleet_model.id
-    await _check_ssh_hosts_not_yet_added(session, spec, current_fleet_id)
+    await _check_ssh_hosts_not_yet_added(session, effective_spec, current_fleet_id)
 
     offers = []
-    if spec.configuration.ssh_config is None:
+    if effective_spec.configuration.ssh_config is None:
         offers_with_backends = await get_create_instance_offers(
             project=project,
-            profile=spec.merged_profile,
-            requirements=_get_fleet_requirements(spec),
-            fleet_spec=spec,
-            blocks=spec.configuration.blocks,
+            profile=effective_spec.merged_profile,
+            requirements=_get_fleet_requirements(effective_spec),
+            fleet_spec=effective_spec,
+            blocks=effective_spec.configuration.blocks,
         )
         offers = [offer for _, offer in offers_with_backends]
-    _remove_fleet_spec_sensitive_info(spec)
+    _remove_fleet_spec_sensitive_info(effective_spec)
     plan = FleetPlan(
         project_name=project.name,
         user=user.name,
         spec=spec,
+        effective_spec=effective_spec,
         current_resource=current_fleet,
         offers=offers[:50],
         total_offers=len(offers),
@@ -307,6 +309,21 @@ async def get_create_instance_offers(
     return offers
 
 
+async def apply_plan(
+    session: AsyncSession,
+    user: UserModel,
+    project: ProjectModel,
+    plan: ApplyFleetPlanInput,
+    force: bool,
+) -> Fleet:
+    return await create_fleet(
+        session=session,
+        project=project,
+        user=user,
+        spec=plan.spec,
+    )
+
+
 async def create_fleet(
     session: AsyncSession,
     project: ProjectModel,
@@ -320,7 +337,7 @@ async def create_fleet(
 
     lock_namespace = f"fleet_names_{project.name}"
     if get_db().dialect_name == "sqlite":
-        # Start new transaction to see commited changes after lock
+        # Start new transaction to see committed changes after lock
         await session.commit()
     elif get_db().dialect_name == "postgresql":
         await session.execute(
@@ -402,6 +419,7 @@ async def create_fleet_instance_model(
         placement_group_name=placement_group_name,
         reservation=reservation,
         blocks=spec.configuration.blocks,
+        tags=spec.configuration.tags,
     )
     return instance_model
 
@@ -629,7 +647,7 @@ def _validate_fleet_spec(spec: FleetSpec):
         if spec.configuration.ssh_config.ssh_key is not None:
             _validate_ssh_key(spec.configuration.ssh_config.ssh_key)
         for host in spec.configuration.ssh_config.hosts:
-            if is_core_model_instance(host, SSHHostParams) and host.ssh_key is not None:
+            if isinstance(host, SSHHostParams) and host.ssh_key is not None:
                 _validate_ssh_key(host.ssh_key)
         _validate_internal_ips(spec.configuration.ssh_config)
 

dstack/_internal/server/services/gateways/__init__.py

@@ -106,6 +106,7 @@ async def create_gateway_compute(
         public_ip=configuration.public_ip,
         ssh_key_pub=gateway_ssh_public_key,
         certificate=configuration.certificate,
+        tags=configuration.tags,
     )
 
     gpd = await run_async(

dstack/_internal/server/services/instances.py

@@ -1,7 +1,7 @@
 import uuid
 from collections.abc import Container, Iterable
 from datetime import datetime, timezone
-from typing import List, Literal, Optional, Union
+from typing import Dict, List, Literal, Optional, Union
 
 import gpuhunt
 from sqlalchemy import and_, or_, select
@@ -411,6 +411,7 @@ async def create_instance_model(
     placement_group_name: Optional[str],
     reservation: Optional[str],
     blocks: Union[Literal["auto"], int],
+    tags: Optional[Dict[str, str]],
 ) -> InstanceModel:
     termination_policy, termination_idle_time = get_termination(
         profile, DEFAULT_FLEET_TERMINATION_IDLE_TIME
@@ -428,6 +429,7 @@ async def create_instance_model(
         instance_id=str(instance_id),
         placement_group_name=placement_group_name,
         reservation=reservation,
+        tags=tags,
     )
     instance = InstanceModel(
         id=instance_id,

dstack/_internal/server/services/jobs/__init__.py

@@ -20,7 +20,6 @@ from dstack._internal.core.errors import (
     SSHError,
 )
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.configurations import RunConfigurationType
 from dstack._internal.core.models.instances import InstanceStatus
 from dstack._internal.core.models.runs import (
@@ -585,7 +584,7 @@ async def get_job_configured_volume_models(
         job_volumes = interpolate_job_volumes(run_spec.configuration.volumes, job_num)
     volume_models = []
     for mount_point in job_volumes:
-        if not is_core_model_instance(mount_point, VolumeMountPoint):
+        if not isinstance(mount_point, VolumeMountPoint):
             continue
         if isinstance(mount_point.name, str):
             names = [mount_point.name]

dstack/_internal/server/services/jobs/configurators/base.py

@@ -1,13 +1,14 @@
 import shlex
 import sys
 from abc import ABC, abstractmethod
+from pathlib import PurePosixPath
 from typing import Dict, List, Optional, Union
 
 from cachetools import TTLCache, cached
 
 import dstack.version as version
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
-from dstack._internal.core.models.common import RegistryAuth, is_core_model_instance
+from dstack._internal.core.models.common import RegistryAuth
 from dstack._internal.core.models.configurations import (
     PortMapping,
     PythonVersion,
@@ -131,16 +132,24 @@ class JobConfigurator(ABC):
         )
         return job_spec
 
+    def _shell(self) -> str:
+        shell = self.run_spec.configuration.shell
+        if shell is not None:
+            path = PurePosixPath(shell)
+            if path.is_absolute():
+                return shell
+            return str("/bin" / path)
+        if self.run_spec.configuration.image is None:  # dstackai/base
+            return "/bin/bash"
+        return "/bin/sh"
+
     async def _commands(self) -> List[str]:
         if self.run_spec.configuration.entrypoint is not None:  # docker-like format
             entrypoint = shlex.split(self.run_spec.configuration.entrypoint)
             commands = self.run_spec.configuration.commands
-        elif self.run_spec.configuration.image is None:  # dstackai/base
-            entrypoint = ["/bin/bash", "-i", "-c"]
-            commands = [_join_shell_commands(self._shell_commands())]
-        elif self._shell_commands():  # custom docker image with shell commands
-            entrypoint = ["/bin/sh", "-i", "-c"]
-            commands = [_join_shell_commands(self._shell_commands())]
+        elif shell_commands := self._shell_commands():
+            entrypoint = [self._shell(), "-i", "-c"]
+            commands = [_join_shell_commands(shell_commands)]
         else:  # custom docker image without commands
             image_config = await self._get_image_config()
             entrypoint = image_config.entrypoint or []
@@ -274,7 +283,7 @@ def interpolate_job_volumes(
         if isinstance(mount_point, str):
             # pydantic validator ensures strings are converted to MountPoint
             continue
-        if not is_core_model_instance(mount_point, VolumeMountPoint):
+        if not isinstance(mount_point, VolumeMountPoint):
             job_volumes.append(mount_point.copy())
             continue
         if isinstance(mount_point.name, str):

dstack/_internal/server/services/locking.py

@@ -2,7 +2,7 @@ import asyncio
 import hashlib
 from asyncio import Lock
 from contextlib import asynccontextmanager
-from typing import Dict, List, Set, Tuple, TypeVar, Union
+from typing import AsyncGenerator, Dict, List, Set, Tuple, TypeVar, Union
 
 from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession
@@ -52,6 +52,21 @@ async def advisory_lock_ctx(
             await bind.execute(select(func.pg_advisory_unlock(string_to_lock_id(resource))))
 
 
+@asynccontextmanager
+async def try_advisory_lock_ctx(
+    bind: Union[AsyncConnection, AsyncSession], dialect_name: str, resource: str
+) -> AsyncGenerator[bool, None]:
+    locked = True
+    if dialect_name == "postgresql":
+        res = await bind.execute(select(func.pg_try_advisory_lock(string_to_lock_id(resource))))
+        locked = res.scalar_one()
+    try:
+        yield locked
+    finally:
+        if dialect_name == "postgresql" and locked:
+            await bind.execute(select(func.pg_advisory_unlock(string_to_lock_id(resource))))
+
+
 _locker = ResourceLocker()
 
 

dstack/_internal/server/services/projects.py

@@ -14,7 +14,6 @@ from dstack._internal.core.backends.dstack.models import (
 )
 from dstack._internal.core.backends.models import BackendInfo
 from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
-from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.projects import Member, MemberPermissions, Project
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
 from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
@@ -386,7 +385,7 @@ def project_model_to_project(
             backend_config = get_backend_config_from_backend_model(
                 configurator, b, include_creds=False
             )
-            if is_core_model_instance(backend_config, DstackBackendConfig):
+            if isinstance(backend_config, DstackBackendConfig):
                 for backend_type in backend_config.base_backends:
                     backends.append(
                         BackendInfo(

dstack/_internal/server/services/proxy/repo.py

@@ -7,7 +7,6 @@ from sqlalchemy.orm import joinedload
 
 import dstack._internal.server.services.jobs as jobs_services
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
-from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.instances import RemoteConnectionInfo, SSHConnectionParams
 from dstack._internal.core.models.runs import (
@@ -64,7 +63,7 @@ class ServerProxyRepo(BaseProxyRepo):
             return None
         run = jobs[0].run
         run_spec = RunSpec.__response__.parse_raw(run.run_spec)
-        if not is_core_model_instance(run_spec.configuration, ServiceConfiguration):
+        if not isinstance(run_spec.configuration, ServiceConfiguration):
             return None
         replicas = []
         for job in jobs:

dstack/_internal/server/services/runner/client.py

@@ -15,6 +15,7 @@ from dstack._internal.core.models.resources import Memory
 from dstack._internal.core.models.runs import ClusterInfo, JobSpec, RunSpec
 from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, VolumeMountPoint
 from dstack._internal.server.schemas.runner import (
+    GPUDevice,
     HealthcheckResponse,
     LegacyPullResponse,
     LegacyStopBody,
@@ -233,6 +234,7 @@ class ShimClient:
         volumes: list[Volume],
         volume_mounts: list[VolumeMountPoint],
         instance_mounts: list[InstanceMountPoint],
+        gpu_devices: list[GPUDevice],
         host_ssh_user: str,
         host_ssh_keys: list[str],
         container_ssh_keys: list[str],
@@ -256,6 +258,7 @@ class ShimClient:
             volumes=[_volume_to_shim_volume_info(v, instance_id) for v in volumes],
             volume_mounts=volume_mounts,
             instance_mounts=instance_mounts,
+            gpu_devices=gpu_devices,
             host_ssh_user=host_ssh_user,
             host_ssh_keys=host_ssh_keys,
             container_ssh_keys=container_ssh_keys,

dstack/_internal/server/services/runs.py

@@ -15,7 +15,7 @@ from dstack._internal.core.errors import (
     ResourceNotExistsError,
     ServerClientError,
 )
-from dstack._internal.core.models.common import ApplyAction, is_core_model_instance
+from dstack._internal.core.models.common import ApplyAction
 from dstack._internal.core.models.configurations import AnyRunConfiguration
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
@@ -92,6 +92,8 @@ JOB_TERMINATION_REASONS_TO_RETRY = {
     JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,
 }
 
+DEFAULT_MAX_OFFERS = 50
+
 
 async def list_user_runs(
     session: AsyncSession,
@@ -275,46 +277,46 @@ async def get_plan(
     project: ProjectModel,
     user: UserModel,
     run_spec: RunSpec,
+    max_offers: Optional[int],
 ) -> RunPlan:
-    _validate_run_spec_and_set_defaults(run_spec)
+    effective_run_spec = RunSpec.parse_obj(run_spec.dict())
+    _validate_run_spec_and_set_defaults(effective_run_spec)
 
-    profile = run_spec.merged_profile
+    profile = effective_run_spec.merged_profile
     creation_policy = profile.creation_policy
 
     current_resource = None
     action = ApplyAction.CREATE
-    if run_spec.run_name is not None:
+    if effective_run_spec.run_name is not None:
         current_resource = await get_run_by_name(
             session=session,
             project=project,
-            run_name=run_spec.run_name,
+            run_name=effective_run_spec.run_name,
         )
         if (
             current_resource is not None
             and not current_resource.status.is_finished()
-            and _can_update_run_spec(current_resource.run_spec, run_spec)
+            and _can_update_run_spec(current_resource.run_spec, effective_run_spec)
         ):
             action = ApplyAction.UPDATE
 
-    # TODO(egor-s): do we need to generate all replicas here?
-    jobs = await get_jobs_from_run_spec(run_spec, replica_num=0)
+    jobs = await get_jobs_from_run_spec(effective_run_spec, replica_num=0)
 
     volumes = await get_job_configured_volumes(
         session=session,
         project=project,
-        run_spec=run_spec,
+        run_spec=effective_run_spec,
         job_num=0,
     )
 
     pool_offers = await _get_pool_offers(
         session=session,
         project=project,
-        run_spec=run_spec,
+        run_spec=effective_run_spec,
         job=jobs[0],
         volumes=volumes,
     )
-    run_name = run_spec.run_name  # preserve run_name
-    run_spec.run_name = "dry-run"  # will regenerate jobs on submission
+    effective_run_spec.run_name = "dry-run"  # will regenerate jobs on submission
 
     # Get offers once for all jobs
     offers = []
@@ -327,7 +329,7 @@ async def get_plan(
             multinode=jobs[0].job_spec.jobs_per_replica > 1,
             volumes=volumes,
             privileged=jobs[0].job_spec.privileged,
-            instance_mounts=check_run_spec_requires_instance_mounts(run_spec),
+            instance_mounts=check_run_spec_requires_instance_mounts(effective_run_spec),
         )
 
     job_plans = []
@@ -342,17 +344,18 @@ async def get_plan(
 
         job_plan = JobPlan(
             job_spec=job_spec,
-            offers=job_offers[:50],
+            offers=job_offers[: (max_offers or DEFAULT_MAX_OFFERS)],
             total_offers=len(job_offers),
             max_price=max((offer.price for offer in job_offers), default=None),
         )
         job_plans.append(job_plan)
 
-    run_spec.run_name = run_name  # restore run_name
+    effective_run_spec.run_name = run_spec.run_name  # restore run_name
     run_plan = RunPlan(
         project_name=project.name,
         user=user.name,
         run_spec=run_spec,
+        effective_run_spec=effective_run_spec,
         job_plans=job_plans,
         current_resource=current_resource,
         action=action,
@@ -748,7 +751,7 @@ async def _generate_run_name(
 
 def check_run_spec_requires_instance_mounts(run_spec: RunSpec) -> bool:
     return any(
-        is_core_model_instance(mp, InstanceMountPoint) and not mp.optional
+        isinstance(mp, InstanceMountPoint) and not mp.optional
         for mp in run_spec.configuration.volumes
     )