dstack 0.18.41__py3-none-any.whl → 0.18.43__py3-none-any.whl

dstack/_internal/core/backends/nebius/compute.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 import dstack.version as version
 from dstack._internal import settings
 from dstack._internal.core.backends.base import Compute
-from dstack._internal.core.backends.base.compute import get_instance_name, get_user_data
+from dstack._internal.core.backends.base.compute import get_job_instance_name, get_user_data
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.nebius.api_client import NebiusAPIClient
 from dstack._internal.core.backends.nebius.config import NebiusConfig
@@ -130,7 +130,7 @@ class NebiusCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),  # TODO: generate name
+            instance_name=get_job_instance_name(run, job),  # TODO: generate name
             ssh_keys=[
                 SSHKey(public=project_ssh_public_key.strip()),
             ],

dstack/_internal/core/backends/oci/compute.py

@@ -4,7 +4,12 @@ from typing import List, Optional
 
 import oci
 
-from dstack._internal.core.backends.base.compute import Compute, get_instance_name, get_user_data
+from dstack._internal.core.backends.base.compute import (
+    Compute,
+    generate_unique_instance_name,
+    get_job_instance_name,
+    get_user_data,
+)
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.oci import resources
 from dstack._internal.core.backends.oci.config import OCIConfig
@@ -98,7 +103,7 @@ class OCICompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),
+            instance_name=get_job_instance_name(run, job),
             ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
             user=run.user,
         )
@@ -148,6 +153,7 @@ class OCICompute(Compute):
         ]
         cloud_init_user_data = get_user_data(instance_config.get_public_keys(), setup_commands)
 
+        display_name = generate_unique_instance_name(instance_config)
         try:
             instance = resources.launch_instance(
                 region=region,
@@ -155,7 +161,7 @@ class OCICompute(Compute):
                 compartment_id=self.config.compartment_id,
                 subnet_id=subnet.id,
                 security_group_id=security_group.id,
-                display_name=instance_config.instance_name,
+                display_name=display_name,
                 cloud_init_user_data=cloud_init_user_data,
                 shape=instance_offer.instance.name,
                 is_spot=instance_offer.instance.resources.spot,
@@ -163,7 +169,7 @@ class OCICompute(Compute):
                 image_id=package.image_id,
             )
         except oci.exceptions.ServiceError as e:
-            if e.code in ("LimitExceeded", "QuotaExceeded"):
+            if e.code in ("LimitExceeded", "QuotaExceeded") or "Out of host capacity" in e.message:
                 raise NoCapacityError(e.message)
             raise
 

dstack/_internal/core/backends/runpod/compute.py

@@ -5,8 +5,10 @@ from typing import List, Optional
 
 from dstack._internal.core.backends.base import Compute
 from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name,
+    generate_unique_volume_name,
     get_docker_commands,
-    get_instance_name,
+    get_job_instance_name,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.runpod.api_client import RunpodApiClient
@@ -31,6 +33,9 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+# Undocumented but names of len 60 work
+MAX_RESOURCE_NAME_LEN = 60
+
 CONTAINER_REGISTRY_AUTH_CLEANUP_INTERVAL = 60 * 60 * 24  # 24 hour
 
 
@@ -69,7 +74,7 @@ class RunpodCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),
+            instance_name=get_job_instance_name(run, job),
             ssh_keys=[
                 SSHKey(public=run.run_spec.ssh_key_pub.strip()),
                 SSHKey(public=project_ssh_public_key.strip()),
@@ -77,6 +82,7 @@ class RunpodCompute(Compute):
             user=run.user,
         )
 
+        pod_name = generate_unique_instance_name(instance_config, max_length=MAX_RESOURCE_NAME_LEN)
         authorized_keys = instance_config.get_public_keys()
         memory_size = round(instance_offer.instance.resources.memory_mib / 1024)
         disk_size = round(instance_offer.instance.resources.disk.size_mib / 1024)
@@ -98,7 +104,7 @@ class RunpodCompute(Compute):
             bid_per_gpu = instance_offer.price / gpu_count
 
         resp = self.api_client.create_pod(
-            name=instance_config.instance_name,
+            name=pod_name,
             image_name=job.job_spec.image_name,
             gpu_type_id=instance_offer.instance.name,
             cloud_type="SECURE",  # ["ALL", "COMMUNITY", "SECURE"]:
@@ -197,9 +203,10 @@ class RunpodCompute(Compute):
         )
 
     def create_volume(self, volume: Volume) -> VolumeProvisioningData:
+        volume_name = generate_unique_volume_name(volume, max_length=MAX_RESOURCE_NAME_LEN)
         size_gb = volume.configuration.size_gb
         volume_id = self.api_client.create_network_volume(
-            name=volume.name,
+            name=volume_name,
             region=volume.configuration.region,
             size=size_gb,
         )

dstack/_internal/core/backends/tensordock/compute.py

@@ -4,7 +4,11 @@ from typing import List, Optional
 import requests
 
 from dstack._internal.core.backends.base import Compute
-from dstack._internal.core.backends.base.compute import get_instance_name, get_shim_commands
+from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name,
+    get_job_instance_name,
+    get_shim_commands,
+)
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.tensordock.api_client import TensorDockAPIClient
 from dstack._internal.core.backends.tensordock.config import TensorDockConfig
@@ -23,6 +27,10 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+# Undocumented but names of len 60 work
+MAX_INSTANCE_NAME_LEN = 60
+
+
 class TensorDockCompute(Compute):
     def __init__(self, config: TensorDockConfig):
         super().__init__()
@@ -49,10 +57,13 @@ class TensorDockCompute(Compute):
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name(
+            instance_config, max_length=MAX_INSTANCE_NAME_LEN
+        )
         commands = get_shim_commands(authorized_keys=instance_config.get_public_keys())
         try:
             resp = self.api_client.deploy_single(
-                instance_name=instance_config.instance_name,
+                instance_name=instance_name,
                 instance=instance_offer.instance,
                 cloudinit={
                     "ssh_pwauth": False,  # disable password auth
@@ -113,7 +124,7 @@ class TensorDockCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),  # TODO: generate name
+            instance_name=get_job_instance_name(run, job),  # TODO: generate name
             ssh_keys=[
                 SSHKey(public=run.run_spec.ssh_key_pub.strip()),
                 SSHKey(public=project_ssh_public_key.strip()),

dstack/_internal/core/backends/vastai/compute.py

@@ -4,7 +4,10 @@ import gpuhunt
 from gpuhunt.providers.vastai import VastAIProvider
 
 from dstack._internal.core.backends.base import Compute
-from dstack._internal.core.backends.base.compute import get_docker_commands, get_instance_name
+from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name_for_job,
+    get_docker_commands,
+)
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.vastai.api_client import VastAIAPIClient
 from dstack._internal.core.backends.vastai.config import VastAIConfig
@@ -23,6 +26,10 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+# Undocumented but names of len 60 work
+MAX_INSTANCE_NAME_LEN = 60
+
+
 class VastAICompute(Compute):
     def __init__(self, config: VastAIConfig):
         super().__init__()
@@ -70,11 +77,14 @@ class VastAICompute(Compute):
         project_ssh_private_key: str,
         volumes: List[Volume],
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name_for_job(
+            run, job, max_length=MAX_INSTANCE_NAME_LEN
+        )
         commands = get_docker_commands(
             [run.run_spec.ssh_key_pub.strip(), project_ssh_public_key.strip()]
         )
         resp = self.api_client.create_instance(
-            instance_name=get_instance_name(run, job),
+            instance_name=instance_name,
             bundle_id=instance_offer.instance.name,
             image_name=job.job_spec.image_name,
             onstart=" && ".join(commands),

dstack/_internal/core/backends/vultr/api_client.py

@@ -20,7 +20,7 @@ class VultrApiClient:
             return False
         return True
 
-    def get_instance(self, instance_id: str, plan_type: str):
+    def get_instance(self, instance_id: str, plan_type: str) -> dict:
         if plan_type == "bare-metal":
             response = self._make_request("GET", f"/bare-metals/{instance_id}")
             return response.json()["bare_metal"]
@@ -28,7 +28,7 @@ class VultrApiClient:
             response = self._make_request("GET", f"/instances/{instance_id}")
             return response.json()["instance"]
 
-    def get_vpc_for_region(self, region: str) -> Optional[str]:
+    def get_vpc_for_region(self, region: str) -> Optional[dict]:
         response = self._make_request("GET", "/vpcs?per_page=500")
         vpcs = response.json().get("vpcs", [])
         if vpcs:
@@ -37,7 +37,7 @@ class VultrApiClient:
                     return vpc
         return None
 
-    def create_vpc(self, region: str):
+    def create_vpc(self, region: str) -> dict:
         data = {"region": region, "description": f"dstack-vpc-{region}"}
         response = self._make_request("POST", "/vpcs", data=data)
         return response.json()["vpc"]

dstack/_internal/core/backends/vultr/compute.py

@@ -6,7 +6,8 @@ import requests
 
 from dstack._internal.core.backends.base import Compute
 from dstack._internal.core.backends.base.compute import (
-    get_instance_name,
+    generate_unique_instance_name,
+    get_job_instance_name,
     get_user_data,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
@@ -27,6 +28,8 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+MAX_INSTANCE_NAME_LEN = 64
+
 
 class VultrCompute(Compute):
     def __init__(self, config: VultrConfig):
@@ -62,7 +65,7 @@ class VultrCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),
+            instance_name=get_job_instance_name(run, job),
             ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
             user=run.user,
         )
@@ -71,6 +74,9 @@ class VultrCompute(Compute):
     def create_instance(
         self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name(
+            instance_config, max_length=MAX_INSTANCE_NAME_LEN
+        )
         # create vpc
         vpc = self.api_client.get_vpc_for_region(instance_offer.region)
         if not vpc:
@@ -85,7 +91,7 @@ class VultrCompute(Compute):
         ]
         instance_id = self.api_client.launch_instance(
             region=instance_offer.region,
-            label=instance_config.instance_name,
+            label=instance_name,
             plan=instance_offer.instance.name,
             user_data=get_user_data(
                 authorized_keys=instance_config.get_public_keys(),

dstack/_internal/core/models/backends/aws.py

@@ -32,6 +32,7 @@ class AWSConfigInfo(CoreModel):
     vpc_ids: Optional[Dict[str, str]] = None
     default_vpcs: Optional[bool] = None
     public_ips: Optional[bool] = None
+    iam_instance_profile: Optional[str] = None
     tags: Optional[Dict[str, str]] = None
     os_images: Optional[AWSOSImageConfig] = None
 
@@ -70,6 +71,7 @@ class AWSConfigInfoWithCredsPartial(CoreModel):
     vpc_ids: Optional[Dict[str, str]]
     default_vpcs: Optional[bool]
     public_ips: Optional[bool]
+    iam_instance_profile: Optional[str]
     tags: Optional[Dict[str, str]]
     os_images: Optional["AWSOSImageConfig"]
 

dstack/_internal/core/models/backends/base.py

@@ -15,6 +15,7 @@ class BackendType(str, enum.Enum):
         DATACRUNCH (BackendType): DataCrunch
         KUBERNETES (BackendType): Kubernetes
         LAMBDA (BackendType): Lambda Cloud
+        OCI (BackendType): Oracle Cloud Infrastructure
         RUNPOD (BackendType): Runpod Cloud
         TENSORDOCK (BackendType): TensorDock Marketplace
         VASTAI (BackendType): Vast.ai Marketplace

dstack/_internal/core/models/configurations.py

@@ -31,7 +31,6 @@ class RunConfigurationType(str, Enum):
 
 
 class PythonVersion(str, Enum):
-    PY38 = "3.8"  # TODO(0.19 or earlier): drop 3.8, stop building Docker images with 3.8
     PY39 = "3.9"
     PY310 = "3.10"
     PY311 = "3.11"

dstack/_internal/core/models/runs.py

@@ -150,9 +150,9 @@ class JobTerminationReason(str, Enum):
 class Requirements(CoreModel):
     # TODO: Make requirements' fields required
     resources: ResourcesSpec
-    max_price: Optional[float]
-    spot: Optional[bool]
-    reservation: Optional[str]
+    max_price: Optional[float] = None
+    spot: Optional[bool] = None
+    reservation: Optional[str] = None
 
     def pretty_format(self, resources_only: bool = False):
         res = self.resources.pretty_format()

dstack/_internal/core/models/volumes.py

@@ -71,6 +71,18 @@ class VolumeAttachmentData(CoreModel):
     device_name: Optional[str] = None
 
 
+class VolumeInstance(CoreModel):
+    name: str
+    fleet_name: Optional[str] = None
+    instance_num: int
+    instance_id: Optional[str] = None
+
+
+class VolumeAttachment(CoreModel):
+    instance: VolumeInstance
+    attachment_data: Optional[VolumeAttachmentData] = None
+
+
 class Volume(CoreModel):
     id: uuid.UUID
     name: str
@@ -86,8 +98,19 @@ class Volume(CoreModel):
     deleted: bool
     volume_id: Optional[str] = None  # id of the volume in the cloud
     provisioning_data: Optional[VolumeProvisioningData] = None
+    attachments: Optional[List[VolumeAttachment]] = None
+    # attachment_data is deprecated in favor of attachments.
+    # It's only set for volumes that were attached before attachments.
     attachment_data: Optional[VolumeAttachmentData] = None
 
+    def get_attachment_data_for_instance(self, instance_id: str) -> Optional[VolumeAttachmentData]:
+        if self.attachments is not None:
+            for attachment in self.attachments:
+                if attachment.instance.instance_id == instance_id:
+                    return attachment.attachment_data
+        # volume was attached before attachments were introduced
+        return self.attachment_data
+
 
 class VolumePlan(CoreModel):
     project_name: str

dstack/_internal/core/services/__init__.py

@@ -4,5 +4,9 @@ from dstack._internal.core.errors import ServerClientError
 
 
 def validate_dstack_resource_name(resource_name: str):
-    if not re.match("^[a-z][a-z0-9-]{1,40}$", resource_name):
+    if not is_valid_dstack_resource_name(resource_name):
         raise ServerClientError("Resource name should match regex '^[a-z][a-z0-9-]{1,40}$'")
+
+
+def is_valid_dstack_resource_name(resource_name: str) -> bool:
+    return re.match("^[a-z][a-z0-9-]{1,40}$", resource_name) is not None

dstack/_internal/core/services/configs/__init__.py

@@ -65,6 +65,9 @@ class ConfigManager:
         if len(self.config.projects) == 1:
             self.config.projects[0].default = True
 
+    def list_projects(self):
+        return [project.name for project in self.config.projects]
+
     def delete_project(self, name: str):
         self.config.projects = [p for p in self.config.projects if p.name != name]
 

dstack/_internal/server/background/tasks/common.py

@@ -0,0 +1,22 @@
+from datetime import timedelta
+
+from dstack._internal.core.models.backends.base import BackendType
+
+
+def get_provisioning_timeout(backend_type: BackendType, instance_type_name: str) -> timedelta:
+    """
+    This timeout is used in a few places, but roughly refers to the max time between
+    requesting instance creation and the instance becoming ready to accept jobs.
+    For container-based backends, this also includes the image pulling time.
+    """
+    if backend_type == BackendType.LAMBDA:
+        return timedelta(minutes=30)
+    if backend_type == BackendType.RUNPOD:
+        return timedelta(minutes=20)
+    if backend_type == BackendType.KUBERNETES:
+        return timedelta(minutes=20)
+    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
+        return timedelta(minutes=20)
+    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
+        return timedelta(minutes=55)
+    return timedelta(minutes=10)

dstack/_internal/server/background/tasks/process_instances.py

@@ -45,7 +45,6 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
     InstanceRuntime,
     InstanceStatus,
-    InstanceType,
     RemoteConnectionInfo,
     SSHKey,
 )
@@ -63,6 +62,7 @@ from dstack._internal.core.models.runs import (
     Retry,
 )
 from dstack._internal.core.services.profiles import get_retry
+from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
     FleetModel,
@@ -507,9 +507,9 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         project=instance.project,
         profile=profile,
         requirements=requirements,
-        exclude_not_available=True,
         fleet_model=instance.fleet,
         blocks="auto" if instance.total_blocks is None else instance.total_blocks,
+        exclude_not_available=True,
     )
 
     if not offers and should_retry:
@@ -695,7 +695,8 @@ async def _check_instance(instance: InstanceModel) -> None:
 
     if instance.status == InstanceStatus.PROVISIONING and instance.started_at is not None:
         provisioning_deadline = _get_provisioning_deadline(
-            instance, job_provisioning_data.instance_type
+            instance=instance,
+            job_provisioning_data=job_provisioning_data,
         )
         if get_current_datetime() > provisioning_deadline:
             instance.status = InstanceStatus.TERMINATING
@@ -737,7 +738,8 @@ async def _wait_for_instance_provisioning_data(
         instance.name,
     )
     provisioning_deadline = _get_provisioning_deadline(
-        instance, job_provisioning_data.instance_type
+        instance=instance,
+        job_provisioning_data=job_provisioning_data,
     )
     if get_current_datetime() > provisioning_deadline:
         logger.warning(
@@ -915,9 +917,8 @@ def _get_instance_offer_for_instance(
         instance_offer.availability_zones = [
             z
             for z in instance_offer.availability_zones
-            if instance_offer.availability_zones == master_job_provisioning_data.availability_zone
+            if z == master_job_provisioning_data.availability_zone
         ]
-
     return instance_offer
 
 
@@ -960,24 +961,15 @@ def _get_retry_duration_deadline(instance: InstanceModel, retry: Retry) -> datet
 
 
 def _get_provisioning_deadline(
-    instance: InstanceModel, instance_type: InstanceType
+    instance: InstanceModel,
+    job_provisioning_data: JobProvisioningData,
 ) -> datetime.datetime:
-    timeout_interval = _get_instance_timeout_interval(instance.backend, instance_type.name)
+    timeout_interval = get_provisioning_timeout(
+        backend_type=job_provisioning_data.get_base_backend(),
+        instance_type_name=job_provisioning_data.instance_type.name,
+    )
     return instance.started_at.replace(tzinfo=datetime.timezone.utc) + timeout_interval
 
 
-def _get_instance_timeout_interval(
-    backend_type: BackendType, instance_type_name: str
-) -> timedelta:
-    # when changing timeouts, also consider process_running_jobs._get_runner_timeout_interval
-    if backend_type == BackendType.RUNPOD:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
-        return timedelta(seconds=3300)
-    return timedelta(seconds=600)
-
-
 def _ssh_keys_to_pkeys(ssh_keys: list[SSHKey]) -> list[PKey]:
     return [pkey_from_str(sk.private) for sk in ssh_keys if sk.private is not None]

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -1,5 +1,4 @@
 import asyncio
-from datetime import timedelta
 from typing import Dict, List, Optional
 
 from sqlalchemy import select
@@ -21,6 +20,7 @@ from dstack._internal.core.models.runs import (
     ClusterInfo,
     Job,
     JobProvisioningData,
+    JobRuntimeData,
     JobSpec,
     JobStatus,
     JobTerminationReason,
@@ -28,6 +28,7 @@ from dstack._internal.core.models.runs import (
     RunSpec,
 )
 from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, VolumeMountPoint
+from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
     InstanceModel,
@@ -148,6 +149,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
         jobs=run.jobs,
         replica_num=job.job_spec.replica_num,
         job_provisioning_data=job_provisioning_data,
+        job_runtime_data=job_submission.job_runtime_data,
     )
 
     volumes = await get_job_attached_volumes(
@@ -205,6 +207,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
                     None,
                     run,
                     job_model,
+                    job_provisioning_data,
                     volumes,
                     secrets,
                     job.job_spec.registry_auth,
@@ -241,7 +244,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
 
             if not success:
                 # check timeout
-                if job_submission.age > _get_runner_timeout_interval(
+                if job_submission.age > get_provisioning_timeout(
                     backend_type=job_provisioning_data.get_base_backend(),
                     instance_type_name=job_provisioning_data.instance_type.name,
                 ):
@@ -376,6 +379,7 @@ def _process_provisioning_with_shim(
     ports: Dict[int, int],
     run: Run,
     job_model: JobModel,
+    job_provisioning_data: JobProvisioningData,
     volumes: List[Volume],
     secrets: Dict[str, str],
     registry_auth: Optional[RegistryAuth],
@@ -459,6 +463,7 @@ def _process_provisioning_with_shim(
             host_ssh_user=ssh_user,
             host_ssh_keys=[ssh_key] if ssh_key else [],
             container_ssh_keys=public_keys,
+            instance_id=job_provisioning_data.instance_id,
         )
     else:
         submitted = shim_client.submit(
@@ -475,6 +480,7 @@ def _process_provisioning_with_shim(
             mounts=volume_mounts,
             volumes=volumes,
             instance_mounts=instance_mounts,
+            instance_id=job_provisioning_data.instance_id,
         )
         if not submitted:
             # This can happen when we lost connection to the runner (e.g., network issues), marked
@@ -667,6 +673,7 @@ def _get_cluster_info(
     jobs: List[Job],
     replica_num: int,
     job_provisioning_data: JobProvisioningData,
+    job_runtime_data: Optional[JobRuntimeData],
 ) -> ClusterInfo:
     job_ips = []
     for job in jobs:
@@ -677,10 +684,13 @@ def _get_cluster_info(
                 ).internal_ip
                 or ""
             )
+    gpus_per_job = len(job_provisioning_data.instance_type.resources.gpus)
+    if job_runtime_data is not None and job_runtime_data.offer is not None:
+        gpus_per_job = len(job_runtime_data.offer.instance.resources.gpus)
     cluster_info = ClusterInfo(
         job_ips=job_ips,
         master_job_ip=job_ips[0],
-        gpus_per_job=len(job_provisioning_data.instance_type.resources.gpus),
+        gpus_per_job=gpus_per_job,
     )
     return cluster_info
 
@@ -759,16 +769,3 @@ def _submit_job_to_runner(
     # do not log here, because the runner will send a new status
 
     return True
-
-
-def _get_runner_timeout_interval(backend_type: BackendType, instance_type_name: str) -> timedelta:
-    # when changing timeouts, also consider process_instances._get_instance_timeout_interval
-    if backend_type == BackendType.LAMBDA:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.KUBERNETES:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
-        return timedelta(seconds=3300)
-    return timedelta(seconds=600)