dstack 0.19.28__py3-none-any.whl → 0.19.30rc1__py3-none-any.whl

dstack/_internal/core/backends/hotaisle/compute.py

@@ -9,6 +9,7 @@ from gpuhunt.providers.hotaisle import HotAisleProvider
 
 from dstack._internal.core.backends.base.compute import (
     Compute,
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     get_shim_commands,
 )
@@ -23,7 +24,7 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
 )
 from dstack._internal.core.models.placement import PlacementGroup
-from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.core.models.runs import JobProvisioningData
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -44,6 +45,7 @@ INSTANCE_TYPE_SPECS = {
 
 
 class HotAisleCompute(
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     Compute,
 ):
@@ -56,16 +58,12 @@ class HotAisleCompute(
             HotAisleProvider(api_key=config.creds.api_key, team_handle=config.team_handle)
         )
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.HOTAISLE,
             locations=self.config.regions or None,
-            requirements=requirements,
             catalog=self.catalog,
         )
-
         supported_offers = []
         for offer in offers:
             if offer.instance.name in INSTANCE_TYPE_SPECS:
@@ -78,7 +76,6 @@ class HotAisleCompute(
                 logger.warning(
                     f"Skipping unsupported Hot Aisle instance type: {offer.instance.name}"
                 )
-
         return supported_offers
 
     def get_payload_from_offer(self, instance_type) -> dict:

dstack/_internal/core/backends/kubernetes/compute.py

@@ -9,13 +9,14 @@ from kubernetes import client
 
 from dstack._internal.core.backends.base.compute import (
     Compute,
+    ComputeWithFilteredOffersCached,
     ComputeWithGatewaySupport,
     generate_unique_gateway_instance_name,
     generate_unique_instance_name_for_job,
     get_docker_commands,
     get_dstack_gateway_commands,
 )
-from dstack._internal.core.backends.base.offers import match_requirements
+from dstack._internal.core.backends.base.offers import filter_offers_by_requirements
 from dstack._internal.core.backends.kubernetes.models import (
     KubernetesConfig,
     KubernetesNetworkingConfig,
@@ -58,6 +59,7 @@ NVIDIA_GPU_NAMES = NVIDIA_GPU_NAME_TO_GPU_INFO.keys()
 
 
 class KubernetesCompute(
+    ComputeWithFilteredOffersCached,
     ComputeWithGatewaySupport,
     Compute,
 ):
@@ -70,8 +72,8 @@ class KubernetesCompute(
         self.networking_config = networking_config
         self.api = get_api_from_config_data(config.kubeconfig.data)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
+    def get_offers_by_requirements(
+        self, requirements: Requirements
     ) -> List[InstanceOfferWithAvailability]:
         nodes = self.api.list_node()
         instance_offers = []
@@ -99,7 +101,7 @@ class KubernetesCompute(
                 availability=InstanceAvailability.AVAILABLE,
                 instance_runtime=InstanceRuntime.RUNNER,
             )
-            instance_offers.extend(match_requirements([instance_offer], requirements))
+            instance_offers.extend(filter_offers_by_requirements([instance_offer], requirements))
         return instance_offers
 
     def run_job(

dstack/_internal/core/backends/lambdalabs/compute.py

@@ -7,6 +7,7 @@ from typing import Dict, List, Optional
 
 from dstack._internal.core.backends.base.compute import (
     Compute,
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     generate_unique_instance_name,
     get_shim_commands,
@@ -22,12 +23,13 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
 )
 from dstack._internal.core.models.placement import PlacementGroup
-from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.core.models.runs import JobProvisioningData
 
 MAX_INSTANCE_NAME_LEN = 60
 
 
 class LambdaCompute(
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     Compute,
 ):
@@ -36,13 +38,10 @@ class LambdaCompute(
         self.config = config
         self.api_client = LambdaAPIClient(config.creds.api_key)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.LAMBDA,
             locations=self.config.regions or None,
-            requirements=requirements,
         )
         offers_with_availability = self._get_offers_with_availability(offers)
         return offers_with_availability

dstack/_internal/core/backends/local/compute.py

@@ -28,9 +28,7 @@ class LocalCompute(
     ComputeWithVolumeSupport,
     Compute,
 ):
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_offers(self, requirements: Requirements) -> List[InstanceOfferWithAvailability]:
         return [
             InstanceOfferWithAvailability(
                 backend=BackendType.LOCAL,

dstack/_internal/core/backends/nebius/compute.py

@@ -3,7 +3,7 @@ import random
 import shlex
 import time
 from functools import cached_property
-from typing import List, Optional
+from typing import Callable, List, Optional
 
 from nebius.aio.operation import Operation as SDKOperation
 from nebius.aio.service_error import RequestError, StatusCode
@@ -12,13 +12,14 @@ from nebius.sdk import SDK
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     generate_unique_instance_name,
     get_user_data,
 )
-from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.base.offers import get_catalog_offers, get_offers_disk_modifier
 from dstack._internal.core.backends.nebius import resources
 from dstack._internal.core.backends.nebius.fabrics import get_suitable_infiniband_fabrics
 from dstack._internal.core.backends.nebius.models import NebiusConfig, NebiusServiceAccountCreds
@@ -76,6 +77,7 @@ SUPPORTED_PLATFORMS = [
 
 
 class NebiusCompute(
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
@@ -106,15 +108,11 @@ class NebiusCompute(
             ).metadata.id
         return self._subnet_id_cache[region]
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.NEBIUS,
             locations=list(self._region_to_project_id),
-            requirements=requirements,
             extra_filter=_supported_instances,
-            configurable_disk_size=CONFIGURABLE_DISK_SIZE,
         )
         return [
             InstanceOfferWithAvailability(
@@ -124,6 +122,11 @@ class NebiusCompute(
             for offer in offers
         ]
 
+    def get_offers_modifier(
+        self, requirements: Requirements
+    ) -> Callable[[InstanceOfferWithAvailability], Optional[InstanceOfferWithAvailability]]:
+        return get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements)
+
     def create_instance(
         self,
         instance_offer: InstanceOfferWithAvailability,

dstack/_internal/core/backends/oci/compute.py

@@ -1,17 +1,18 @@
 from concurrent.futures import ThreadPoolExecutor
 from functools import cached_property
-from typing import List, Optional
+from typing import Callable, List, Optional
 
 import oci
 
 from dstack._internal.core.backends.base.compute import (
     Compute,
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     generate_unique_instance_name,
     get_user_data,
 )
-from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.base.offers import get_catalog_offers, get_offers_disk_modifier
 from dstack._internal.core.backends.oci import resources
 from dstack._internal.core.backends.oci.models import OCIConfig
 from dstack._internal.core.backends.oci.region import make_region_clients_map
@@ -47,6 +48,7 @@ CONFIGURABLE_DISK_SIZE = Range[Memory](min=Memory.parse("50GB"), max=Memory.pars
 
 
 class OCICompute(
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     Compute,
@@ -60,14 +62,10 @@ class OCICompute(
     def shapes_quota(self) -> resources.ShapesQuota:
         return resources.ShapesQuota.load(self.regions, self.config.compartment_id)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.OCI,
             locations=self.config.regions,
-            requirements=requirements,
-            configurable_disk_size=CONFIGURABLE_DISK_SIZE,
             extra_filter=_supported_instances,
         )
 
@@ -96,6 +94,11 @@ class OCICompute(
 
         return offers_with_availability
 
+    def get_offers_modifier(
+        self, requirements: Requirements
+    ) -> Callable[[InstanceOfferWithAvailability], Optional[InstanceOfferWithAvailability]]:
+        return get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements)
+
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
     ) -> None:

dstack/_internal/core/backends/runpod/compute.py

@@ -1,17 +1,18 @@
 import json
 import uuid
 from datetime import timedelta
-from typing import List, Optional
+from typing import Callable, List, Optional
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    ComputeWithAllOffersCached,
     ComputeWithVolumeSupport,
     generate_unique_instance_name,
     generate_unique_volume_name,
     get_docker_commands,
     get_job_instance_name,
 )
-from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.base.offers import get_catalog_offers, get_offers_disk_modifier
 from dstack._internal.core.backends.runpod.api_client import RunpodApiClient
 from dstack._internal.core.backends.runpod.models import RunpodConfig
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
@@ -27,6 +28,7 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
     SSHKey,
 )
+from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run
 from dstack._internal.core.models.volumes import Volume, VolumeProvisioningData
 from dstack._internal.utils.common import get_current_datetime
@@ -39,8 +41,12 @@ MAX_RESOURCE_NAME_LEN = 60
 
 CONTAINER_REGISTRY_AUTH_CLEANUP_INTERVAL = 60 * 60 * 24  # 24 hour
 
+# RunPod does not seem to have any limits on the disk size.
+CONFIGURABLE_DISK_SIZE = Range[Memory](min=Memory.parse("1GB"), max=None)
+
 
 class RunpodCompute(
+    ComputeWithAllOffersCached,
     ComputeWithVolumeSupport,
     Compute,
 ):
@@ -51,13 +57,11 @@ class RunpodCompute(
         self.config = config
         self.api_client = RunpodApiClient(config.creds.api_key)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.RUNPOD,
             locations=self.config.regions or None,
-            requirements=requirements,
+            requirements=None,
             extra_filter=lambda o: _is_secure_cloud(o.region) or self.config.allow_community_cloud,
         )
         offers = [
@@ -68,6 +72,11 @@ class RunpodCompute(
         ]
         return offers
 
+    def get_offers_modifier(
+        self, requirements: Requirements
+    ) -> Callable[[InstanceOfferWithAvailability], Optional[InstanceOfferWithAvailability]]:
+        return get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements)
+
     def run_job(
         self,
         run: Run,

dstack/_internal/core/backends/template/compute.py.jinja

@@ -2,6 +2,7 @@ from typing import List, Optional
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
     ComputeWithMultinodeSupport,
@@ -28,6 +29,7 @@ logger = get_logger(__name__)
 
 class {{ backend_name }}Compute(
     # TODO: Choose ComputeWith* classes to extend and implement
+    # ComputeWithAllOffersCached,
     # ComputeWithCreateInstanceSupport,
     # ComputeWithMultinodeSupport,
     # ComputeWithReservationSupport,
@@ -42,7 +44,7 @@ class {{ backend_name }}Compute(
         self.config = config
 
     def get_offers(
-        self, requirements: Optional[Requirements] = None
+        self, requirements: Requirements
     ) -> List[InstanceOfferWithAvailability]:
         # If the provider is added to gpuhunt, you'd typically get offers
         # using `get_catalog_offers()` and extend them with availability info.

dstack/_internal/core/backends/tensordock/compute.py

@@ -39,9 +39,7 @@ class TensorDockCompute(
         self.config = config
         self.api_client = TensorDockAPIClient(config.creds.api_key, config.creds.api_token)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_offers(self, requirements: Requirements) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.TENSORDOCK,
             requirements=requirements,

dstack/_internal/core/backends/tensordock/models.py

@@ -4,6 +4,8 @@ from pydantic import Field
 
 from dstack._internal.core.models.common import CoreModel
 
+# TODO: TensorDock is deprecated and will be removed in the future
+
 
 class TensorDockAPIKeyCreds(CoreModel):
     type: Annotated[Literal["api_key"], Field(description="The type of credentials")] = "api_key"

dstack/_internal/core/backends/vastai/compute.py

@@ -5,6 +5,7 @@ from gpuhunt.providers.vastai import VastAIProvider
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    ComputeWithFilteredOffersCached,
     generate_unique_instance_name_for_job,
     get_docker_commands,
 )
@@ -30,7 +31,10 @@ logger = get_logger(__name__)
 MAX_INSTANCE_NAME_LEN = 60
 
 
-class VastAICompute(Compute):
+class VastAICompute(
+    ComputeWithFilteredOffersCached,
+    Compute,
+):
     def __init__(self, config: VastAIConfig):
         super().__init__()
         self.config = config
@@ -49,8 +53,8 @@ class VastAICompute(Compute):
             )
         )
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
+    def get_offers_by_requirements(
+        self, requirements: Requirements
     ) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.VASTAI,

dstack/_internal/core/backends/vultr/compute.py

@@ -6,6 +6,7 @@ import requests
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     generate_unique_instance_name,
@@ -23,7 +24,7 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
 )
 from dstack._internal.core.models.placement import PlacementGroup
-from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.core.models.runs import JobProvisioningData
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -32,6 +33,7 @@ MAX_INSTANCE_NAME_LEN = 64
 
 
 class VultrCompute(
+    ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithMultinodeSupport,
     Compute,
@@ -41,12 +43,10 @@ class VultrCompute(
         self.config = config
         self.api_client = VultrApiClient(config.creds.api_key)
 
-    def get_offers(
-        self, requirements: Optional[Requirements] = None
-    ) -> List[InstanceOfferWithAvailability]:
+    def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.VULTR,
-            requirements=requirements,
+            requirements=None,
             locations=self.config.regions or None,
             extra_filter=_supported_instances,
         )

dstack/_internal/core/models/projects.py

@@ -26,3 +26,11 @@ class Project(CoreModel):
     backends: List[BackendInfo]
     members: List[Member]
     is_public: bool = False
+
+
+class ProjectHookConfig(CoreModel):
+    """
+    This class can be inherited to extend the project creation configuration passed to the hooks.
+    """
+
+    pass

dstack/_internal/core/services/repos.py

@@ -122,7 +122,8 @@ def _get_repo_creds_and_default_branch_https(
 
 def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]:
     # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD"
-    output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env)
+    # Disable credential helpers to exclude any default credentials from being used
+    output: str = git.cmd.Git()(c="credential.helper=").ls_remote("--symref", url, "HEAD", env=env)
     for line in output.splitlines():
         # line format: `<oid> TAB <ref> LF`
         oid, _, ref = line.partition("\t")

dstack/_internal/server/background/tasks/process_instances.py

@@ -578,7 +578,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
                 if placement_group_model is None:  # error occurred
                     continue
                 session.add(placement_group_model)
-                await session.flush()
                 placement_group_models.append(placement_group_model)
         logger.debug(
             "Trying %s in %s/%s for $%0.4f per hour",
@@ -636,7 +635,9 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             },
         )
         if instance.fleet_id and _is_fleet_master_instance(instance):
-            # Clean up placement groups that did not end up being used
+            # Clean up placement groups that did not end up being used.
+            # Flush to update still uncommitted placement groups.
+            await session.flush()
             await schedule_fleet_placement_groups_deletion(
                 session=session,
                 fleet_id=instance.fleet_id,

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -289,7 +289,8 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
                     instance_filters=instance_filters,
                 )
             fleet_models = fleet_models_with_instances + fleet_models_without_instances
-            fleet_model, fleet_instances_with_offers = _find_optimal_fleet_with_offers(
+            fleet_model, fleet_instances_with_offers = await _find_optimal_fleet_with_offers(
+                project=project,
                 fleet_models=fleet_models,
                 run_model=run_model,
                 run_spec=run.run_spec,
@@ -492,7 +493,8 @@ async def _refetch_fleet_models_with_instances(
     return fleet_models
 
 
-def _find_optimal_fleet_with_offers(
+async def _find_optimal_fleet_with_offers(
+    project: ProjectModel,
     fleet_models: list[FleetModel],
     run_model: RunModel,
     run_spec: RunSpec,
@@ -502,58 +504,99 @@ def _find_optimal_fleet_with_offers(
 ) -> tuple[Optional[FleetModel], list[tuple[InstanceModel, InstanceOfferWithAvailability]]]:
     if run_model.fleet is not None:
         # Using the fleet that was already chosen by the master job
-        fleet_instances_with_offers = _get_fleet_instances_with_offers(
+        fleet_instances_with_pool_offers = _get_fleet_instances_with_pool_offers(
             fleet_model=run_model.fleet,
             run_spec=run_spec,
             job=job,
             master_job_provisioning_data=master_job_provisioning_data,
             volumes=volumes,
         )
-        return run_model.fleet, fleet_instances_with_offers
+        return run_model.fleet, fleet_instances_with_pool_offers
 
     if len(fleet_models) == 0:
         return None, []
 
     nodes_required_num = _get_nodes_required_num_for_run(run_spec)
-    # The current strategy is to first consider fleets that can accommodate
-    # the run without additional provisioning and choose the one with the cheapest offer.
-    # Fallback to fleet with the cheapest offer among all fleets with offers.
+    # The current strategy is first to consider fleets that can accommodate
+    # the run without additional provisioning and choose the one with the cheapest pool offer.
+    # Then choose a fleet with the cheapest pool offer among all fleets with pool offers.
+    # If there are no fleets with pool offers, choose a fleet with a cheapest backend offer.
+    # Fallback to autocreated fleet if fleets have no pool or backend offers.
+    # TODO: Consider trying all backend offers and then choosing a fleet.
     candidate_fleets_with_offers: list[
         tuple[
             Optional[FleetModel],
             list[tuple[InstanceModel, InstanceOfferWithAvailability]],
             int,
-            tuple[int, float],
+            int,
+            tuple[int, float, float],
         ]
     ] = []
     for candidate_fleet_model in fleet_models:
-        fleet_instances_with_offers = _get_fleet_instances_with_offers(
+        fleet_instances_with_pool_offers = _get_fleet_instances_with_pool_offers(
             fleet_model=candidate_fleet_model,
             run_spec=run_spec,
             job=job,
             master_job_provisioning_data=master_job_provisioning_data,
             volumes=volumes,
         )
-        fleet_available_offers = [
-            o for _, o in fleet_instances_with_offers if o.availability.is_available()
-        ]
-        fleet_has_available_capacity = nodes_required_num <= len(fleet_available_offers)
-        fleet_cheapest_offer = math.inf
-        if len(fleet_available_offers) > 0:
-            fleet_cheapest_offer = fleet_available_offers[0].price
-        fleet_priority = (not fleet_has_available_capacity, fleet_cheapest_offer)
+        fleet_has_available_capacity = nodes_required_num <= len(fleet_instances_with_pool_offers)
+        fleet_cheapest_pool_offer = math.inf
+        if len(fleet_instances_with_pool_offers) > 0:
+            fleet_cheapest_pool_offer = fleet_instances_with_pool_offers[0][1].price
+
+        candidate_fleet = fleet_model_to_fleet(candidate_fleet_model)
+        profile = combine_fleet_and_run_profiles(
+            candidate_fleet.spec.merged_profile, run_spec.merged_profile
+        )
+        fleet_requirements = get_fleet_requirements(candidate_fleet.spec)
+        requirements = combine_fleet_and_run_requirements(
+            fleet_requirements, job.job_spec.requirements
+        )
+        multinode = (
+            candidate_fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
+            or job.job_spec.jobs_per_replica > 1
+        )
+        fleet_backend_offers = []
+        if (
+            _check_can_create_new_instance_in_fleet(candidate_fleet)
+            and profile is not None
+            and requirements is not None
+        ):
+            fleet_backend_offers = await get_offers_by_requirements(
+                project=project,
+                profile=profile,
+                requirements=requirements,
+                exclude_not_available=True,
+                multinode=multinode,
+                master_job_provisioning_data=master_job_provisioning_data,
+                volumes=volumes,
+                privileged=job.job_spec.privileged,
+                instance_mounts=check_run_spec_requires_instance_mounts(run_spec),
+            )
+
+        fleet_cheapest_backend_offer = math.inf
+        if len(fleet_backend_offers) > 0:
+            fleet_cheapest_backend_offer = fleet_backend_offers[0][1].price
+
+        fleet_priority = (
+            not fleet_has_available_capacity,
+            fleet_cheapest_pool_offer,
+            fleet_cheapest_backend_offer,
+        )
         candidate_fleets_with_offers.append(
             (
                 candidate_fleet_model,
-                fleet_instances_with_offers,
-                len(fleet_available_offers),
+                fleet_instances_with_pool_offers,
+                len(fleet_instances_with_pool_offers),
+                len(fleet_backend_offers),
                 fleet_priority,
             )
         )
     if run_spec.merged_profile.fleets is None and all(
-        t[2] == 0 for t in candidate_fleets_with_offers
+        t[2] == 0 and t[3] == 0 for t in candidate_fleets_with_offers
     ):
-        # If fleets are not specified and no fleets have available offers, create a new fleet.
+        # If fleets are not specified and no fleets have available pool or backend offers, create a new fleet.
         # This is for compatibility with non-fleet-first UX when runs created new fleets
         # if there are no instances to reuse.
         return None, []
@@ -573,7 +616,7 @@ def _get_nodes_required_num_for_run(run_spec: RunSpec) -> int:
     return nodes_required_num
 
 
-def _get_fleet_instances_with_offers(
+def _get_fleet_instances_with_pool_offers(
     fleet_model: FleetModel,
     run_spec: RunSpec,
     job: Job,

dstack/_internal/server/services/backends/__init__.py

@@ -345,7 +345,7 @@ async def get_instance_offers(
     Returns list of instances satisfying minimal resource requirements sorted by price
     """
     logger.info("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends])
-    tasks = [run_async(backend.compute().get_offers_cached, requirements) for backend in backends]
+    tasks = [run_async(backend.compute().get_offers, requirements) for backend in backends]
     offers_by_backend = []
     for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)):
         if isinstance(result, BackendError):