dstack 0.18.40rc1__py3-none-any.whl → 0.18.42__py3-none-any.whl

dstack/_internal/server/services/jobs/__init__.py

@@ -7,14 +7,21 @@ from uuid import UUID
 import requests
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
 
 import dstack._internal.server.services.backends as backends_services
 from dstack._internal.core.backends.base import Backend
 from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT
-from dstack._internal.core.errors import BackendError, ResourceNotExistsError, SSHError
+from dstack._internal.core.errors import (
+    BackendError,
+    ResourceNotExistsError,
+    ServerClientError,
+    SSHError,
+)
 from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.configurations import RunConfigurationType
-from dstack._internal.core.models.instances import InstanceStatus, RemoteConnectionInfo
+from dstack._internal.core.models.instances import InstanceStatus
 from dstack._internal.core.models.runs import (
     Job,
     JobProvisioningData,
@@ -25,6 +32,7 @@ from dstack._internal.core.models.runs import (
     JobTerminationReason,
     RunSpec,
 )
+from dstack._internal.core.models.volumes import Volume, VolumeMountPoint, VolumeStatus
 from dstack._internal.server.models import (
     InstanceModel,
     JobModel,
@@ -33,14 +41,22 @@ from dstack._internal.server.models import (
     VolumeModel,
 )
 from dstack._internal.server.services import services
-from dstack._internal.server.services.jobs.configurators.base import JobConfigurator
+from dstack._internal.server.services import volumes as volumes_services
+from dstack._internal.server.services.jobs.configurators.base import (
+    JobConfigurator,
+    interpolate_job_volumes,
+)
 from dstack._internal.server.services.jobs.configurators.dev import DevEnvironmentJobConfigurator
 from dstack._internal.server.services.jobs.configurators.service import ServiceJobConfigurator
 from dstack._internal.server.services.jobs.configurators.task import TaskJobConfigurator
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.services.pools import get_instance_ssh_private_keys
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
-from dstack._internal.server.services.volumes import volume_model_to_volume
+from dstack._internal.server.services.volumes import (
+    list_project_volume_models,
+    volume_model_to_volume,
+)
 from dstack._internal.utils import common
 from dstack._internal.utils.common import get_or_error, run_async
 from dstack._internal.utils.logging import get_logger
@@ -115,6 +131,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         submitted_at=job_model.submitted_at.replace(tzinfo=timezone.utc),
         last_processed_at=last_processed_at,
         finished_at=finished_at,
+        inactivity_secs=job_model.inactivity_secs,
         status=job_model.status,
         termination_reason=job_model.termination_reason,
         termination_reason_message=job_model.termination_reason_message,
@@ -155,29 +172,22 @@ _configuration_type_to_configurator_class_map = {c.TYPE: c for c in _job_configu
 
 
 async def stop_runner(session: AsyncSession, job_model: JobModel):
-    project = await session.get(ProjectModel, job_model.project_id)
-    ssh_private_key = project.ssh_private_key
-
     res = await session.execute(
-        select(InstanceModel).where(
+        select(InstanceModel)
+        .where(
             InstanceModel.project_id == job_model.project_id,
-            InstanceModel.job_id == job_model.id,
+            InstanceModel.id == job_model.instance_id,
         )
+        .options(joinedload(InstanceModel.project))
     )
     instance: Optional[InstanceModel] = res.scalar()
 
-    # TODO: Drop this logic and always use project key once it's safe to assume that most on-prem
-    # fleets are (re)created after this change: https://github.com/dstackai/dstack/pull/1716
-    if instance and instance.remote_connection_info is not None:
-        remote_conn_info: RemoteConnectionInfo = RemoteConnectionInfo.__response__.parse_raw(
-            instance.remote_connection_info
-        )
-        ssh_private_key = remote_conn_info.ssh_keys[0].private
+    ssh_private_keys = get_instance_ssh_private_keys(common.get_or_error(instance))
     try:
         jpd = get_job_provisioning_data(job_model)
         if jpd is not None:
             jrd = get_job_runtime_data(job_model)
-            await run_async(_stop_runner, ssh_private_key, jpd, jrd, job_model)
+            await run_async(_stop_runner, ssh_private_keys, jpd, jrd, job_model)
     except SSHError:
         logger.debug("%s: failed to stop runner", fmt(job_model))
 
@@ -219,30 +229,41 @@ async def process_terminating_job(
         _set_job_termination_status(job_model)
         return
 
+    all_volumes_detached: bool = True
+    jrd = get_job_runtime_data(job_model)
     jpd = get_job_provisioning_data(job_model)
     if jpd is not None:
         logger.debug("%s: stopping container", fmt(job_model))
-        ssh_private_key = instance_model.project.ssh_private_key
-        # TODO: Drop this logic and always use project key once it's safe to assume that
-        # most on-prem fleets are (re)created after this change:
-        # https://github.com/dstackai/dstack/pull/1716
-        if instance_model and instance_model.remote_connection_info is not None:
-            remote_conn_info: RemoteConnectionInfo = RemoteConnectionInfo.__response__.parse_raw(
-                instance_model.remote_connection_info
+        ssh_private_keys = get_instance_ssh_private_keys(instance_model)
+        await stop_container(job_model, jpd, ssh_private_keys)
+        volume_models: list[VolumeModel]
+        if jrd is not None and jrd.volume_names is not None:
+            volume_models = await list_project_volume_models(
+                session=session, project=instance_model.project, names=jrd.volume_names
             )
-            ssh_private_key = remote_conn_info.ssh_keys[0].private
-        await stop_container(job_model, jpd, ssh_private_key)
-        if len(instance_model.volumes) > 0:
-            logger.info("Detaching volumes: %s", [v.name for v in instance_model.volumes])
-            await _detach_volumes_from_job_instance(
+        else:
+            volume_models = [va.volume for va in instance_model.volume_attachments]
+        if len(volume_models) > 0:
+            logger.info("Detaching volumes: %s", [v.name for v in volume_models])
+            all_volumes_detached = await _detach_volumes_from_job_instance(
                 project=instance_model.project,
                 job_model=job_model,
                 jpd=jpd,
                 instance_model=instance_model,
+                volume_models=volume_models,
             )
 
+    if jrd is not None and jrd.offer is not None:
+        blocks = jrd.offer.blocks
+    else:
+        # Old job submitted before jrd or blocks were introduced
+        blocks = 1
+    instance_model.busy_blocks -= blocks
+
     if instance_model.status == InstanceStatus.BUSY:
-        instance_model.status = InstanceStatus.IDLE
+        # no other jobs besides this one
+        if not [j for j in instance_model.jobs if j.id != job_model.id]:
+            instance_model.status = InstanceStatus.IDLE
     elif instance_model.status != InstanceStatus.TERMINATED:
         # instance was PROVISIONING (specially for the job)
         # schedule for termination
@@ -254,7 +275,7 @@ async def process_terminating_job(
 
     # The instance should be released even if detach fails
     # so that stuck volumes don't prevent the instance from terminating.
-    instance_model.job_id = None
+    job_model.instance_id = None
     instance_model.last_job_processed_at = common.get_current_datetime()
     logger.info(
         "%s: instance '%s' has been released, new status is %s",
@@ -263,9 +284,8 @@ async def process_terminating_job(
         instance_model.status.name,
     )
     await services.unregister_replica(session, job_model)
-    if len(instance_model.volumes) == 0:
+    if all_volumes_detached:
         # Do not terminate while some volumes are not detached.
-        # TODO: In case of multiple jobs per instance, don't consider volumes from other jobs.
         _set_job_termination_status(job_model)
 
 
@@ -280,18 +300,25 @@ async def process_volumes_detaching(
     If the volumes fail to detach, force detaches them.
     """
     jpd = get_or_error(get_job_provisioning_data(job_model))
-    logger.info("Detaching volumes: %s", [v.name for v in instance_model.volumes])
-    await _detach_volumes_from_job_instance(
+    jrd = get_job_runtime_data(job_model)
+    if jrd is not None and jrd.volume_names is not None:
+        volume_models = await list_project_volume_models(
+            session=session, project=instance_model.project, names=jrd.volume_names
+        )
+    else:
+        volume_models = [va.volume for va in instance_model.volume_attachments]
+    logger.info("Detaching volumes: %s", [v.name for v in volume_models])
+    all_volumes_detached = await _detach_volumes_from_job_instance(
         project=instance_model.project,
         job_model=job_model,
         jpd=jpd,
         instance_model=instance_model,
+        volume_models=volume_models,
     )
-    if len(instance_model.volumes) == 0:
+    if all_volumes_detached:
         # Do not terminate the job while some volumes are not detached.
         # If force detach never succeeds, the job will be stuck terminating.
         # The job releases the instance when soft detaching, so the instance won't be stuck.
-        # TODO: In case of multiple jobs per instance, don't consider volumes from other jobs.
         _set_job_termination_status(job_model)
 
 
@@ -311,14 +338,16 @@ def _set_job_termination_status(job_model: JobModel):
 
 
 async def stop_container(
-    job_model: JobModel, job_provisioning_data: JobProvisioningData, ssh_private_key: str
+    job_model: JobModel,
+    job_provisioning_data: JobProvisioningData,
+    ssh_private_keys: tuple[str, Optional[str]],
 ):
     if job_provisioning_data.dockerized:
         # send a request to the shim to terminate the docker container
         # SSHError and RequestException are caught in the `runner_ssh_tunner` decorator
         await run_async(
             _shim_submit_stop,
-            ssh_private_key,
+            ssh_private_keys,
             job_provisioning_data,
             None,
             job_model,
@@ -378,7 +407,8 @@ async def _detach_volumes_from_job_instance(
     job_model: JobModel,
     jpd: JobProvisioningData,
     instance_model: InstanceModel,
-):
+    volume_models: list[VolumeModel],
+) -> bool:
     job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
     backend = await backends_services.get_project_backend_by_type(
         project=project,
@@ -388,11 +418,11 @@ async def _detach_volumes_from_job_instance(
         logger.error(
             "Failed to detach volumes from %s. Backend not available.", instance_model.name
         )
-        return
+        return False
 
-    # TODO: In case of multiple jobs per instance, detach only volumes used by this job
+    all_detached = True
     detached_volumes = []
-    for volume_model in instance_model.volumes:
+    for volume_model in volume_models:
         detached = await _detach_volume_from_job_instance(
             backend=backend,
             job_model=job_model,
@@ -403,13 +433,16 @@ async def _detach_volumes_from_job_instance(
         )
         if detached:
             detached_volumes.append(volume_model)
+        else:
+            all_detached = False
 
     if job_model.volumes_detached_at is None:
         job_model.volumes_detached_at = common.get_current_datetime()
     detached_volumes_ids = {v.id for v in detached_volumes}
-    instance_model.volumes = [
-        v for v in instance_model.volumes if v.id not in detached_volumes_ids
+    instance_model.volume_attachments = [
+        va for va in instance_model.volume_attachments if va.volume_id not in detached_volumes_ids
     ]
+    return all_detached
 
 
 async def _detach_volume_from_job_instance(
@@ -503,3 +536,142 @@ async def get_instances_ids_with_detaching_volumes(session: AsyncSession) -> Lis
     )
     job_models = res.scalars().all()
     return [jm.used_instance_id for jm in job_models if jm.used_instance_id]
+
+
+async def get_job_configured_volumes(
+    session: AsyncSession,
+    project: ProjectModel,
+    run_spec: RunSpec,
+    job_num: int,
+    job_spec: Optional[JobSpec] = None,
+) -> List[List[Volume]]:
+    """
+    Returns a list of job volumes grouped by mount points.
+    """
+    volume_models = await get_job_configured_volume_models(
+        session=session,
+        project=project,
+        run_spec=run_spec,
+        job_num=job_num,
+        job_spec=job_spec,
+    )
+    return [
+        [volumes_services.volume_model_to_volume(v) for v in mount_point_volume_models]
+        for mount_point_volume_models in volume_models
+    ]
+
+
+async def get_job_configured_volume_models(
+    session: AsyncSession,
+    project: ProjectModel,
+    run_spec: RunSpec,
+    job_num: int,
+    job_spec: Optional[JobSpec] = None,
+) -> List[List[VolumeModel]]:
+    """
+    Returns a list of job volume models grouped by mount points.
+    """
+    job_volumes = None
+    if job_spec is not None:
+        job_volumes = job_spec.volumes
+    if job_volumes is None:
+        # job_spec not provided or a legacy job_spec without volumes
+        job_volumes = interpolate_job_volumes(run_spec.configuration.volumes, job_num)
+    volume_models = []
+    for mount_point in job_volumes:
+        if not is_core_model_instance(mount_point, VolumeMountPoint):
+            continue
+        if isinstance(mount_point.name, str):
+            names = [mount_point.name]
+        else:
+            names = mount_point.name
+        mount_point_volume_models = []
+        for name in names:
+            volume_model = await volumes_services.get_project_volume_model_by_name(
+                session=session,
+                project=project,
+                name=name,
+            )
+            if volume_model is None:
+                raise ResourceNotExistsError(f"Volume {mount_point.name} not found")
+            mount_point_volume_models.append(volume_model)
+        volume_models.append(mount_point_volume_models)
+    return volume_models
+
+
+def check_can_attach_job_volumes(volumes: List[List[Volume]]):
+    """
+    Performs basic checks if volumes can be attached.
+    This is useful to show error ASAP (when user submits the run).
+    If the attachment is to fail anyway, the error will be handled when proccessing submitted jobs.
+    """
+    if len(volumes) == 0:
+        return
+    expected_backends = {v.configuration.backend for v in volumes[0]}
+    expected_regions = {v.configuration.region for v in volumes[0]}
+    for mount_point_volumes in volumes:
+        backends = {v.configuration.backend for v in mount_point_volumes}
+        regions = {v.configuration.region for v in mount_point_volumes}
+        if backends != expected_backends:
+            raise ServerClientError(
+                "Volumes from different backends specified for different mount points"
+            )
+        if regions != expected_regions:
+            raise ServerClientError(
+                "Volumes from different regions specified for different mount points"
+            )
+        for volume in mount_point_volumes:
+            if volume.status != VolumeStatus.ACTIVE:
+                raise ServerClientError(f"Cannot mount volumes that are not active: {volume.name}")
+    volumes_names = [v.name for vs in volumes for v in vs]
+    if len(volumes_names) != len(set(volumes_names)):
+        raise ServerClientError("Cannot attach the same volume at different mount points")
+
+
+async def get_job_attached_volumes(
+    session: AsyncSession,
+    project: ProjectModel,
+    run_spec: RunSpec,
+    job_num: int,
+    job_provisioning_data: JobProvisioningData,
+) -> List[Volume]:
+    """
+    Returns volumes attached to the job.
+    """
+    job_configured_volumes = await get_job_configured_volumes(
+        session=session,
+        project=project,
+        run_spec=run_spec,
+        job_num=job_num,
+    )
+    job_volumes = []
+    for mount_point_volumes in job_configured_volumes:
+        job_volumes.append(
+            _get_job_mount_point_attached_volume(mount_point_volumes, job_provisioning_data)
+        )
+    return job_volumes
+
+
+def _get_job_mount_point_attached_volume(
+    volumes: List[Volume],
+    job_provisioning_data: JobProvisioningData,
+) -> Volume:
+    """
+    Returns the volume attached to the job among the list of possible mount point volumes.
+    """
+    for volume in volumes:
+        if (
+            volume.configuration.backend != job_provisioning_data.get_base_backend()
+            or volume.configuration.region != job_provisioning_data.region
+        ):
+            continue
+        if (
+            volume.provisioning_data is not None
+            and volume.provisioning_data.availability_zone is not None
+            and job_provisioning_data.availability_zone is not None
+            and volume.provisioning_data.availability_zone
+            != job_provisioning_data.availability_zone
+        ):
+            continue
+        return volume
+    raise ServerClientError("Failed to find an eligible volume for the mount point")

dstack/_internal/server/services/jobs/configurators/base.py

@@ -1,13 +1,13 @@
 import shlex
 import sys
 from abc import ABC, abstractmethod
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from cachetools import TTLCache, cached
 
 import dstack.version as version
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
-from dstack._internal.core.models.common import RegistryAuth
+from dstack._internal.core.models.common import RegistryAuth, is_core_model_instance
 from dstack._internal.core.models.configurations import (
     PortMapping,
     PythonVersion,
@@ -22,10 +22,12 @@ from dstack._internal.core.models.runs import (
     RunSpec,
 )
 from dstack._internal.core.models.unix import UnixUser
+from dstack._internal.core.models.volumes import MountPoint, VolumeMountPoint
 from dstack._internal.core.services.profiles import get_retry
 from dstack._internal.core.services.ssh.ports import filter_reserved_ports
 from dstack._internal.server.services.docker import ImageConfig, get_image_config
 from dstack._internal.utils.common import run_async
+from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
 
 
 def get_default_python_verison() -> str:
@@ -115,6 +117,7 @@ class JobConfigurator(ABC):
             requirements=self._requirements(),
             retry=self._retry(),
             working_dir=self._working_dir(),
+            volumes=self._volumes(job_num),
         )
         return job_spec
 
@@ -224,6 +227,48 @@ class JobConfigurator(ABC):
             return self.run_spec.configuration.python.value
         return get_default_python_verison()
 
+    def _volumes(self, job_num: int) -> List[MountPoint]:
+        return interpolate_job_volumes(self.run_spec.configuration.volumes, job_num)
+
+
+def interpolate_job_volumes(
+    run_volumes: List[Union[MountPoint, str]],
+    job_num: int,
+) -> List[MountPoint]:
+    if len(run_volumes) == 0:
+        return []
+    interpolator = VariablesInterpolator(
+        namespaces={
+            "dstack": {
+                "job_num": str(job_num),
+                "node_rank": str(job_num),  # an alias for job_num
+            }
+        }
+    )
+    job_volumes = []
+    for mount_point in run_volumes:
+        if isinstance(mount_point, str):
+            # pydantic validator ensures strings are converted to MountPoint
+            continue
+        if not is_core_model_instance(mount_point, VolumeMountPoint):
+            job_volumes.append(mount_point.copy())
+            continue
+        if isinstance(mount_point.name, str):
+            names = [mount_point.name]
+        else:
+            names = mount_point.name
+        try:
+            interpolated_names = [interpolator.interpolate_or_error(n) for n in names]
+        except InterpolatorError as e:
+            raise ServerClientError(e.args[0])
+        job_volumes.append(
+            VolumeMountPoint(
+                name=interpolated_names,
+                path=mount_point.path,
+            )
+        )
+    return job_volumes
+
 
 def _join_shell_commands(commands: List[str]) -> str:
     for i, cmd in enumerate(commands):

dstack/_internal/server/services/offers.py

@@ -1,4 +1,6 @@
-from typing import List, Optional, Tuple
+from typing import List, Literal, Optional, Tuple, Union
+
+import gpuhunt
 
 from dstack._internal.core.backends import (
     BACKENDS_WITH_CREATE_INSTANCE_SUPPORT,
@@ -7,7 +9,11 @@ from dstack._internal.core.backends import (
 )
 from dstack._internal.core.backends.base import Backend
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.instances import InstanceOfferWithAvailability
+from dstack._internal.core.models.instances import (
+    InstanceOfferWithAvailability,
+    InstanceType,
+    Resources,
+)
 from dstack._internal.core.models.profiles import Profile
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.core.models.volumes import Volume
@@ -25,6 +31,7 @@ async def get_offers_by_requirements(
     volumes: Optional[List[List[Volume]]] = None,
     privileged: bool = False,
     instance_mounts: bool = False,
+    blocks: Union[int, Literal["auto"]] = 1,
 ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]:
     backends: List[Backend] = await backends_services.get_project_backends(project=project)
 
@@ -38,33 +45,40 @@ async def get_offers_by_requirements(
 
     backend_types = profile.backends
     regions = profile.regions
+    availability_zones = profile.availability_zones
 
     if volumes:
         mount_point_volumes = volumes[0]
-        backend_types = [v.configuration.backend for v in mount_point_volumes]
-        regions = [v.configuration.region for v in mount_point_volumes]
+        volumes_backend_types = [v.configuration.backend for v in mount_point_volumes]
+        if backend_types is None:
+            backend_types = volumes_backend_types
+        backend_types = [b for b in backend_types if b in volumes_backend_types]
+        volumes_regions = [v.configuration.region for v in mount_point_volumes]
+        if regions is None:
+            regions = volumes_regions
+        regions = [r for r in regions if r in volumes_regions]
 
     if multinode:
-        if not backend_types:
+        if backend_types is None:
             backend_types = BACKENDS_WITH_MULTINODE_SUPPORT
         backend_types = [b for b in backend_types if b in BACKENDS_WITH_MULTINODE_SUPPORT]
 
     if privileged or instance_mounts:
-        if not backend_types:
+        if backend_types is None:
             backend_types = BACKENDS_WITH_CREATE_INSTANCE_SUPPORT
         backend_types = [b for b in backend_types if b in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT]
 
     if profile.reservation is not None:
-        if not backend_types:
+        if backend_types is None:
             backend_types = BACKENDS_WITH_RESERVATION_SUPPORT
         backend_types = [b for b in backend_types if b in BACKENDS_WITH_RESERVATION_SUPPORT]
 
     # For multi-node, restrict backend and region.
     # The default behavior is to provision all nodes in the same backend and region.
     if master_job_provisioning_data is not None:
-        if not backend_types:
+        if backend_types is None:
             backend_types = [master_job_provisioning_data.get_base_backend()]
-        if not regions:
+        if regions is None:
             regions = [master_job_provisioning_data.region]
         backend_types = [
             b for b in backend_types if b == master_job_provisioning_data.get_base_backend()
@@ -89,7 +103,79 @@ async def get_offers_by_requirements(
     if regions is not None:
         offers = [(b, o) for b, o in offers if o.region in regions]
 
+    if availability_zones is not None:
+        new_offers = []
+        for b, o in offers:
+            if o.availability_zones is not None:
+                new_offer = o.copy()
+                new_offer.availability_zones = [
+                    z for z in o.availability_zones if z in availability_zones
+                ]
+                if new_offer.availability_zones:
+                    new_offers.append((b, new_offer))
+        offers = new_offers
+
     if profile.instance_types is not None:
         offers = [(b, o) for b, o in offers if o.instance.name in profile.instance_types]
 
-    return offers
+    if blocks == 1:
+        return offers
+
+    shareable_offers = []
+    for backend, offer in offers:
+        resources = offer.instance.resources
+        cpu_count = resources.cpus
+        gpu_count = len(resources.gpus)
+        if gpu_count > 0 and resources.gpus[0].vendor == gpuhunt.AcceleratorVendor.GOOGLE:
+            # TPUs cannot be shared
+            gpu_count = 1
+        divisible, _blocks = is_divisible_into_blocks(cpu_count, gpu_count, blocks)
+        if not divisible:
+            continue
+        offer.total_blocks = _blocks
+        shareable_offers.append((backend, offer))
+    return shareable_offers
+
+
+def is_divisible_into_blocks(
+    cpu_count: int, gpu_count: int, blocks: Union[int, Literal["auto"]]
+) -> tuple[bool, int]:
+    """
+    Returns `True` and number of blocks the instance can be split into or `False` and `0` if
+    is not divisible.
+    Requested number of blocks can be `auto`, which means as many as possible.
+    """
+    if blocks == "auto":
+        if gpu_count == 0:
+            blocks = cpu_count
+        else:
+            blocks = min(cpu_count, gpu_count)
+    if blocks < 1 or cpu_count % blocks or gpu_count % blocks:
+        return False, 0
+    return True, blocks
+
+
+def generate_shared_offer(
+    offer: InstanceOfferWithAvailability, blocks: int, total_blocks: int
+) -> InstanceOfferWithAvailability:
+    full_resources = offer.instance.resources
+    resources = Resources(
+        cpus=full_resources.cpus // total_blocks * blocks,
+        memory_mib=full_resources.memory_mib // total_blocks * blocks,
+        gpus=full_resources.gpus[: len(full_resources.gpus) // total_blocks * blocks],
+        spot=full_resources.spot,
+        disk=full_resources.disk,
+        description=full_resources.description,
+    )
+    return InstanceOfferWithAvailability(
+        backend=offer.backend,
+        instance=InstanceType(
+            name=offer.instance.name,
+            resources=resources,
+        ),
+        region=offer.region,
+        price=offer.price,
+        availability=offer.availability,
+        blocks=blocks,
+        total_blocks=total_blocks,
+    )