dstack 0.18.42__py3-none-any.whl → 0.18.43__py3-none-any.whl

dstack/_internal/core/backends/runpod/compute.py

@@ -5,8 +5,10 @@ from typing import List, Optional
 
 from dstack._internal.core.backends.base import Compute
 from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name,
+    generate_unique_volume_name,
     get_docker_commands,
-    get_instance_name,
+    get_job_instance_name,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.runpod.api_client import RunpodApiClient
@@ -31,6 +33,9 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+# Undocumented but names of len 60 work
+MAX_RESOURCE_NAME_LEN = 60
+
 CONTAINER_REGISTRY_AUTH_CLEANUP_INTERVAL = 60 * 60 * 24  # 24 hour
 
 
@@ -69,7 +74,7 @@ class RunpodCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),
+            instance_name=get_job_instance_name(run, job),
             ssh_keys=[
                 SSHKey(public=run.run_spec.ssh_key_pub.strip()),
                 SSHKey(public=project_ssh_public_key.strip()),
@@ -77,6 +82,7 @@ class RunpodCompute(Compute):
             user=run.user,
         )
 
+        pod_name = generate_unique_instance_name(instance_config, max_length=MAX_RESOURCE_NAME_LEN)
         authorized_keys = instance_config.get_public_keys()
         memory_size = round(instance_offer.instance.resources.memory_mib / 1024)
         disk_size = round(instance_offer.instance.resources.disk.size_mib / 1024)
@@ -98,7 +104,7 @@ class RunpodCompute(Compute):
             bid_per_gpu = instance_offer.price / gpu_count
 
         resp = self.api_client.create_pod(
-            name=instance_config.instance_name,
+            name=pod_name,
             image_name=job.job_spec.image_name,
             gpu_type_id=instance_offer.instance.name,
             cloud_type="SECURE",  # ["ALL", "COMMUNITY", "SECURE"]:
@@ -197,9 +203,10 @@ class RunpodCompute(Compute):
         )
 
     def create_volume(self, volume: Volume) -> VolumeProvisioningData:
+        volume_name = generate_unique_volume_name(volume, max_length=MAX_RESOURCE_NAME_LEN)
         size_gb = volume.configuration.size_gb
         volume_id = self.api_client.create_network_volume(
-            name=volume.name,
+            name=volume_name,
             region=volume.configuration.region,
             size=size_gb,
         )

dstack/_internal/core/backends/tensordock/compute.py

@@ -4,7 +4,11 @@ from typing import List, Optional
 import requests
 
 from dstack._internal.core.backends.base import Compute
-from dstack._internal.core.backends.base.compute import get_instance_name, get_shim_commands
+from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name,
+    get_job_instance_name,
+    get_shim_commands,
+)
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.tensordock.api_client import TensorDockAPIClient
 from dstack._internal.core.backends.tensordock.config import TensorDockConfig
@@ -23,6 +27,10 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+# Undocumented but names of len 60 work
+MAX_INSTANCE_NAME_LEN = 60
+
+
 class TensorDockCompute(Compute):
     def __init__(self, config: TensorDockConfig):
         super().__init__()
@@ -49,10 +57,13 @@ class TensorDockCompute(Compute):
         instance_offer: InstanceOfferWithAvailability,
         instance_config: InstanceConfiguration,
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name(
+            instance_config, max_length=MAX_INSTANCE_NAME_LEN
+        )
         commands = get_shim_commands(authorized_keys=instance_config.get_public_keys())
         try:
             resp = self.api_client.deploy_single(
-                instance_name=instance_config.instance_name,
+                instance_name=instance_name,
                 instance=instance_offer.instance,
                 cloudinit={
                     "ssh_pwauth": False,  # disable password auth
@@ -113,7 +124,7 @@ class TensorDockCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),  # TODO: generate name
+            instance_name=get_job_instance_name(run, job),  # TODO: generate name
             ssh_keys=[
                 SSHKey(public=run.run_spec.ssh_key_pub.strip()),
                 SSHKey(public=project_ssh_public_key.strip()),

dstack/_internal/core/backends/vastai/compute.py

@@ -4,7 +4,10 @@ import gpuhunt
 from gpuhunt.providers.vastai import VastAIProvider
 
 from dstack._internal.core.backends.base import Compute
-from dstack._internal.core.backends.base.compute import get_docker_commands, get_instance_name
+from dstack._internal.core.backends.base.compute import (
+    generate_unique_instance_name_for_job,
+    get_docker_commands,
+)
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.vastai.api_client import VastAIAPIClient
 from dstack._internal.core.backends.vastai.config import VastAIConfig
@@ -23,6 +26,10 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+# Undocumented but names of len 60 work
+MAX_INSTANCE_NAME_LEN = 60
+
+
 class VastAICompute(Compute):
     def __init__(self, config: VastAIConfig):
         super().__init__()
@@ -70,11 +77,14 @@ class VastAICompute(Compute):
         project_ssh_private_key: str,
         volumes: List[Volume],
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name_for_job(
+            run, job, max_length=MAX_INSTANCE_NAME_LEN
+        )
         commands = get_docker_commands(
             [run.run_spec.ssh_key_pub.strip(), project_ssh_public_key.strip()]
         )
         resp = self.api_client.create_instance(
-            instance_name=get_instance_name(run, job),
+            instance_name=instance_name,
             bundle_id=instance_offer.instance.name,
             image_name=job.job_spec.image_name,
             onstart=" && ".join(commands),

dstack/_internal/core/backends/vultr/api_client.py

@@ -20,7 +20,7 @@ class VultrApiClient:
             return False
         return True
 
-    def get_instance(self, instance_id: str, plan_type: str):
+    def get_instance(self, instance_id: str, plan_type: str) -> dict:
         if plan_type == "bare-metal":
             response = self._make_request("GET", f"/bare-metals/{instance_id}")
             return response.json()["bare_metal"]
@@ -28,7 +28,7 @@ class VultrApiClient:
             response = self._make_request("GET", f"/instances/{instance_id}")
             return response.json()["instance"]
 
-    def get_vpc_for_region(self, region: str) -> Optional[str]:
+    def get_vpc_for_region(self, region: str) -> Optional[dict]:
         response = self._make_request("GET", "/vpcs?per_page=500")
         vpcs = response.json().get("vpcs", [])
         if vpcs:
@@ -37,7 +37,7 @@ class VultrApiClient:
                     return vpc
         return None
 
-    def create_vpc(self, region: str):
+    def create_vpc(self, region: str) -> dict:
         data = {"region": region, "description": f"dstack-vpc-{region}"}
         response = self._make_request("POST", "/vpcs", data=data)
         return response.json()["vpc"]

dstack/_internal/core/backends/vultr/compute.py

@@ -6,7 +6,8 @@ import requests
 
 from dstack._internal.core.backends.base import Compute
 from dstack._internal.core.backends.base.compute import (
-    get_instance_name,
+    generate_unique_instance_name,
+    get_job_instance_name,
     get_user_data,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
@@ -27,6 +28,8 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+MAX_INSTANCE_NAME_LEN = 64
+
 
 class VultrCompute(Compute):
     def __init__(self, config: VultrConfig):
@@ -62,7 +65,7 @@ class VultrCompute(Compute):
     ) -> JobProvisioningData:
         instance_config = InstanceConfiguration(
             project_name=run.project_name,
-            instance_name=get_instance_name(run, job),
+            instance_name=get_job_instance_name(run, job),
             ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
             user=run.user,
         )
@@ -71,6 +74,9 @@ class VultrCompute(Compute):
     def create_instance(
         self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration
     ) -> JobProvisioningData:
+        instance_name = generate_unique_instance_name(
+            instance_config, max_length=MAX_INSTANCE_NAME_LEN
+        )
         # create vpc
         vpc = self.api_client.get_vpc_for_region(instance_offer.region)
         if not vpc:
@@ -85,7 +91,7 @@ class VultrCompute(Compute):
         ]
         instance_id = self.api_client.launch_instance(
             region=instance_offer.region,
-            label=instance_config.instance_name,
+            label=instance_name,
             plan=instance_offer.instance.name,
             user_data=get_user_data(
                 authorized_keys=instance_config.get_public_keys(),

dstack/_internal/core/models/backends/aws.py

@@ -32,6 +32,7 @@ class AWSConfigInfo(CoreModel):
     vpc_ids: Optional[Dict[str, str]] = None
     default_vpcs: Optional[bool] = None
     public_ips: Optional[bool] = None
+    iam_instance_profile: Optional[str] = None
     tags: Optional[Dict[str, str]] = None
     os_images: Optional[AWSOSImageConfig] = None
 
@@ -70,6 +71,7 @@ class AWSConfigInfoWithCredsPartial(CoreModel):
     vpc_ids: Optional[Dict[str, str]]
     default_vpcs: Optional[bool]
     public_ips: Optional[bool]
+    iam_instance_profile: Optional[str]
     tags: Optional[Dict[str, str]]
     os_images: Optional["AWSOSImageConfig"]
 

dstack/_internal/core/models/backends/base.py

@@ -15,6 +15,7 @@ class BackendType(str, enum.Enum):
         DATACRUNCH (BackendType): DataCrunch
         KUBERNETES (BackendType): Kubernetes
         LAMBDA (BackendType): Lambda Cloud
+        OCI (BackendType): Oracle Cloud Infrastructure
         RUNPOD (BackendType): Runpod Cloud
         TENSORDOCK (BackendType): TensorDock Marketplace
         VASTAI (BackendType): Vast.ai Marketplace

dstack/_internal/core/models/configurations.py

@@ -31,7 +31,6 @@ class RunConfigurationType(str, Enum):
 
 
 class PythonVersion(str, Enum):
-    PY38 = "3.8"  # TODO(0.19 or earlier): drop 3.8, stop building Docker images with 3.8
     PY39 = "3.9"
     PY310 = "3.10"
     PY311 = "3.11"

dstack/_internal/core/services/__init__.py

@@ -4,5 +4,9 @@ from dstack._internal.core.errors import ServerClientError
 
 
 def validate_dstack_resource_name(resource_name: str):
-    if not re.match("^[a-z][a-z0-9-]{1,40}$", resource_name):
+    if not is_valid_dstack_resource_name(resource_name):
         raise ServerClientError("Resource name should match regex '^[a-z][a-z0-9-]{1,40}$'")
+
+
+def is_valid_dstack_resource_name(resource_name: str) -> bool:
+    return re.match("^[a-z][a-z0-9-]{1,40}$", resource_name) is not None

dstack/_internal/core/services/configs/__init__.py

@@ -65,6 +65,9 @@ class ConfigManager:
         if len(self.config.projects) == 1:
             self.config.projects[0].default = True
 
+    def list_projects(self):
+        return [project.name for project in self.config.projects]
+
     def delete_project(self, name: str):
         self.config.projects = [p for p in self.config.projects if p.name != name]
 

dstack/_internal/server/background/tasks/common.py

@@ -0,0 +1,22 @@
+from datetime import timedelta
+
+from dstack._internal.core.models.backends.base import BackendType
+
+
+def get_provisioning_timeout(backend_type: BackendType, instance_type_name: str) -> timedelta:
+    """
+    This timeout is used in a few places, but roughly refers to the max time between
+    requesting instance creation and the instance becoming ready to accept jobs.
+    For container-based backends, this also includes the image pulling time.
+    """
+    if backend_type == BackendType.LAMBDA:
+        return timedelta(minutes=30)
+    if backend_type == BackendType.RUNPOD:
+        return timedelta(minutes=20)
+    if backend_type == BackendType.KUBERNETES:
+        return timedelta(minutes=20)
+    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
+        return timedelta(minutes=20)
+    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
+        return timedelta(minutes=55)
+    return timedelta(minutes=10)

dstack/_internal/server/background/tasks/process_instances.py

@@ -45,7 +45,6 @@ from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
     InstanceRuntime,
     InstanceStatus,
-    InstanceType,
     RemoteConnectionInfo,
     SSHKey,
 )
@@ -63,6 +62,7 @@ from dstack._internal.core.models.runs import (
     Retry,
 )
 from dstack._internal.core.services.profiles import get_retry
+from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
     FleetModel,
@@ -695,7 +695,8 @@ async def _check_instance(instance: InstanceModel) -> None:
 
     if instance.status == InstanceStatus.PROVISIONING and instance.started_at is not None:
         provisioning_deadline = _get_provisioning_deadline(
-            instance, job_provisioning_data.instance_type
+            instance=instance,
+            job_provisioning_data=job_provisioning_data,
         )
         if get_current_datetime() > provisioning_deadline:
             instance.status = InstanceStatus.TERMINATING
@@ -737,7 +738,8 @@ async def _wait_for_instance_provisioning_data(
         instance.name,
     )
     provisioning_deadline = _get_provisioning_deadline(
-        instance, job_provisioning_data.instance_type
+        instance=instance,
+        job_provisioning_data=job_provisioning_data,
     )
     if get_current_datetime() > provisioning_deadline:
         logger.warning(
@@ -959,24 +961,15 @@ def _get_retry_duration_deadline(instance: InstanceModel, retry: Retry) -> datet
 
 
 def _get_provisioning_deadline(
-    instance: InstanceModel, instance_type: InstanceType
+    instance: InstanceModel,
+    job_provisioning_data: JobProvisioningData,
 ) -> datetime.datetime:
-    timeout_interval = _get_instance_timeout_interval(instance.backend, instance_type.name)
+    timeout_interval = get_provisioning_timeout(
+        backend_type=job_provisioning_data.get_base_backend(),
+        instance_type_name=job_provisioning_data.instance_type.name,
+    )
     return instance.started_at.replace(tzinfo=datetime.timezone.utc) + timeout_interval
 
 
-def _get_instance_timeout_interval(
-    backend_type: BackendType, instance_type_name: str
-) -> timedelta:
-    # when changing timeouts, also consider process_running_jobs._get_runner_timeout_interval
-    if backend_type == BackendType.RUNPOD:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
-        return timedelta(seconds=3300)
-    return timedelta(seconds=600)
-
-
 def _ssh_keys_to_pkeys(ssh_keys: list[SSHKey]) -> list[PKey]:
     return [pkey_from_str(sk.private) for sk in ssh_keys if sk.private is not None]

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -1,5 +1,4 @@
 import asyncio
-from datetime import timedelta
 from typing import Dict, List, Optional
 
 from sqlalchemy import select
@@ -21,6 +20,7 @@ from dstack._internal.core.models.runs import (
     ClusterInfo,
     Job,
     JobProvisioningData,
+    JobRuntimeData,
     JobSpec,
     JobStatus,
     JobTerminationReason,
@@ -28,6 +28,7 @@ from dstack._internal.core.models.runs import (
     RunSpec,
 )
 from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, VolumeMountPoint
+from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
     InstanceModel,
@@ -148,6 +149,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
         jobs=run.jobs,
         replica_num=job.job_spec.replica_num,
         job_provisioning_data=job_provisioning_data,
+        job_runtime_data=job_submission.job_runtime_data,
     )
 
     volumes = await get_job_attached_volumes(
@@ -242,7 +244,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
 
             if not success:
                 # check timeout
-                if job_submission.age > _get_runner_timeout_interval(
+                if job_submission.age > get_provisioning_timeout(
                     backend_type=job_provisioning_data.get_base_backend(),
                     instance_type_name=job_provisioning_data.instance_type.name,
                 ):
@@ -671,6 +673,7 @@ def _get_cluster_info(
     jobs: List[Job],
     replica_num: int,
     job_provisioning_data: JobProvisioningData,
+    job_runtime_data: Optional[JobRuntimeData],
 ) -> ClusterInfo:
     job_ips = []
     for job in jobs:
@@ -681,10 +684,13 @@ def _get_cluster_info(
                 ).internal_ip
                 or ""
             )
+    gpus_per_job = len(job_provisioning_data.instance_type.resources.gpus)
+    if job_runtime_data is not None and job_runtime_data.offer is not None:
+        gpus_per_job = len(job_runtime_data.offer.instance.resources.gpus)
     cluster_info = ClusterInfo(
         job_ips=job_ips,
         master_job_ip=job_ips[0],
-        gpus_per_job=len(job_provisioning_data.instance_type.resources.gpus),
+        gpus_per_job=gpus_per_job,
     )
     return cluster_info
 
@@ -763,16 +769,3 @@ def _submit_job_to_runner(
     # do not log here, because the runner will send a new status
 
     return True
-
-
-def _get_runner_timeout_interval(backend_type: BackendType, instance_type_name: str) -> timedelta:
-    # when changing timeouts, also consider process_instances._get_instance_timeout_interval
-    if backend_type == BackendType.LAMBDA:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.KUBERNETES:
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.OCI and instance_type_name.startswith("BM."):
-        return timedelta(seconds=1200)
-    if backend_type == BackendType.VULTR and instance_type_name.startswith("vbm"):
-        return timedelta(seconds=3300)
-    return timedelta(seconds=600)

dstack/_internal/server/background/tasks/process_terminating_jobs.py

@@ -11,7 +11,6 @@ from dstack._internal.server.models import (
     JobModel,
     ProjectModel,
     VolumeAttachmentModel,
-    VolumeModel,
 )
 from dstack._internal.server.services.jobs import (
     process_terminating_job,
@@ -86,12 +85,7 @@ async def _process_job(session: AsyncSession, job_model: JobModel):
         .where(InstanceModel.id == job_model.used_instance_id)
         .options(
             joinedload(InstanceModel.project).joinedload(ProjectModel.backends),
-            joinedload(InstanceModel.volume_attachments)
-            .joinedload(VolumeAttachmentModel.volume)
-            .joinedload(VolumeModel.user),
-            joinedload(InstanceModel.volume_attachments)
-            .joinedload(VolumeAttachmentModel.volume)
-            .joinedload(VolumeModel.attachments),
+            joinedload(InstanceModel.volume_attachments).joinedload(VolumeAttachmentModel.volume),
         )
     )
     instance_model = res.unique().scalar()

dstack/_internal/server/routers/logs.py

@@ -24,4 +24,7 @@ async def poll_logs(
     user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
 ) -> JobSubmissionLogs:
     _, project = user_project
+    # The runner guarantees logs have different timestamps if throughput < 1k logs / sec.
+    # Otherwise, some logs with duplicated timestamps may be filtered out.
+    # This limitation is imposed by cloud log services that support up to millisecond timestamp resolution.
     return await logs.poll_logs_async(project=project, request=body)

dstack/_internal/server/services/backends/configurators/aws.py

@@ -2,6 +2,7 @@ import concurrent.futures
 import json
 from typing import List
 
+import botocore.exceptions
 from boto3.session import Session
 
 from dstack._internal.core.backends.aws import AWSBackend, auth, compute, resources
@@ -35,6 +36,9 @@ from dstack._internal.server.services.backends.configurators.base import (
     Configurator,
     raise_invalid_credentials_error,
 )
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
 
 REGIONS = [
     ("US East, N. Virginia", "us-east-1"),
@@ -137,7 +141,8 @@ class AWSConfigurator(Configurator):
 
     def _check_config(self, session: Session, config: AWSConfigInfoWithCredsPartial):
         self._check_tags_config(config)
-        self._check_vpc_config(session=session, config=config)
+        self._check_iam_instance_profile_config(session, config)
+        self._check_vpc_config(session, config)
 
     def _check_tags_config(self, config: AWSConfigInfoWithCredsPartial):
         if not config.tags:
@@ -151,6 +156,31 @@ class AWSConfigurator(Configurator):
         except BackendError as e:
             raise ServerClientError(e.args[0])
 
+    def _check_iam_instance_profile_config(
+        self, session: Session, config: AWSConfigInfoWithCredsPartial
+    ):
+        if config.iam_instance_profile is None:
+            return
+        try:
+            iam_client = session.client("iam")
+            iam_client.get_instance_profile(InstanceProfileName=config.iam_instance_profile)
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "NoSuchEntity":
+                raise ServerClientError(
+                    f"IAM instance profile {config.iam_instance_profile} not found"
+                )
+            logger.exception(
+                "Got botocore.exceptions.ClientError when checking iam_instance_profile"
+            )
+            raise ServerClientError(
+                f"Failed to check IAM instance profile {config.iam_instance_profile}"
+            )
+        except Exception:
+            logger.exception("Got exception when checking iam_instance_profile")
+            raise ServerClientError(
+                f"Failed to check IAM instance profile {config.iam_instance_profile}"
+            )
+
     def _check_vpc_config(self, session: Session, config: AWSConfigInfoWithCredsPartial):
         allocate_public_ip = config.public_ips if config.public_ips is not None else True
         use_default_vpcs = config.default_vpcs if config.default_vpcs is not None else True

dstack/_internal/server/services/backends/configurators/gcp.py

@@ -127,10 +127,6 @@ class GCPConfigurator(Configurator):
             _, project_id = auth.authenticate(GCPDefaultCreds())
         except BackendAuthError:
             return []
-
-        if project_id is None:
-            return []
-
         return [
             GCPConfigInfoWithCreds(
                 project_id=project_id,
@@ -152,24 +148,21 @@ class GCPConfigurator(Configurator):
         ):
             raise_invalid_credentials_error(fields=[["creds"]])
         try:
-            credentials, project_id = auth.authenticate(creds=config.creds)
-        except BackendAuthError:
+            credentials, _ = auth.authenticate(creds=config.creds, project_id=config.project_id)
+        except BackendAuthError as e:
+            details = None
+            if len(e.args) > 0:
+                details = e.args[0]
             if is_core_model_instance(config.creds, GCPServiceAccountCreds):
-                raise_invalid_credentials_error(fields=[["creds", "data"]])
+                raise_invalid_credentials_error(fields=[["creds", "data"]], details=details)
             else:
-                raise_invalid_credentials_error(fields=[["creds"]])
-        if (
-            project_id is not None
-            and config.project_id is not None
-            and config.project_id != project_id
-        ):
-            raise ServerClientError(msg="Wrong project_id", fields=[["project_id"]])
-        config_values.project_id = self._get_project_id_element(selected=project_id)
+                raise_invalid_credentials_error(fields=[["creds"]], details=details)
         config_values.regions = self._get_regions_element(
             selected=config.regions or DEFAULT_REGIONS
         )
         if config.project_id is None:
             return config_values
+        config_values.project_id = self._get_project_id_element(selected=config.project_id)
         self._check_config(config=config, credentials=credentials)
         return config_values
 

dstack/_internal/server/services/config.py

@@ -107,6 +107,16 @@ class AWSConfig(CoreModel):
             )
         ),
     ] = None
+    iam_instance_profile: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "The name of the IAM instance profile to associate with EC2 instances."
+                " You can also specify the IAM role name for roles created via the AWS console."
+                " AWS automatically creates an instance profile and gives it the same name as the role"
+            )
+        ),
+    ] = None
     tags: Annotated[
         Optional[Dict[str, str]],
         Field(description="The tags that will be assigned to resources created by `dstack`"),
@@ -251,7 +261,7 @@ class GCPConfig(CoreModel):
         ),
     ] = None
     vm_service_account: Annotated[
-        Optional[str], Field(description="The service account associated with provisioned VMs")
+        Optional[str], Field(description="The service account to associate with provisioned VMs")
     ] = None
     tags: Annotated[
         Optional[Dict[str, str]],

dstack/_internal/server/services/jobs/__init__.py

@@ -236,13 +236,14 @@ async def process_terminating_job(
         logger.debug("%s: stopping container", fmt(job_model))
         ssh_private_keys = get_instance_ssh_private_keys(instance_model)
         await stop_container(job_model, jpd, ssh_private_keys)
-        volume_models: list[VolumeModel]
         if jrd is not None and jrd.volume_names is not None:
-            volume_models = await list_project_volume_models(
-                session=session, project=instance_model.project, names=jrd.volume_names
-            )
+            volume_names = jrd.volume_names
         else:
-            volume_models = [va.volume for va in instance_model.volume_attachments]
+            # Legacy jobs before job_runtime_data/blocks were introduced
+            volume_names = [va.volume.name for va in instance_model.volume_attachments]
+        volume_models = await list_project_volume_models(
+            session=session, project=instance_model.project, names=volume_names
+        )
         if len(volume_models) > 0:
             logger.info("Detaching volumes: %s", [v.name for v in volume_models])
             all_volumes_detached = await _detach_volumes_from_job_instance(
@@ -302,11 +303,13 @@ async def process_volumes_detaching(
     jpd = get_or_error(get_job_provisioning_data(job_model))
     jrd = get_job_runtime_data(job_model)
     if jrd is not None and jrd.volume_names is not None:
-        volume_models = await list_project_volume_models(
-            session=session, project=instance_model.project, names=jrd.volume_names
-        )
+        volume_names = jrd.volume_names
     else:
-        volume_models = [va.volume for va in instance_model.volume_attachments]
+        # Legacy jobs before job_runtime_data/blocks were introduced
+        volume_names = [va.volume.name for va in instance_model.volume_attachments]
+    volume_models = await list_project_volume_models(
+        session=session, project=instance_model.project, names=volume_names
+    )
     logger.info("Detaching volumes: %s", [v.name for v in volume_models])
     all_volumes_detached = await _detach_volumes_from_job_instance(
         project=instance_model.project,

dstack/_internal/server/services/jobs/configurators/dev.py

@@ -6,8 +6,6 @@ from dstack._internal.core.models.runs import RunSpec
 from dstack._internal.server.services.jobs.configurators.base import JobConfigurator
 from dstack._internal.server.services.jobs.configurators.extensions.vscode import VSCodeDesktop
 
-DEFAULT_MAX_DURATION_SECONDS = 6 * 3600
-
 INSTALL_IPYKERNEL = (
     "(echo pip install ipykernel... && pip install -q --no-cache-dir ipykernel 2> /dev/null) || "
     'echo "no pip, ipykernel was not installed"'
@@ -44,7 +42,7 @@ class DevEnvironmentJobConfigurator(JobConfigurator):
         return False
 
     def _default_max_duration(self) -> Optional[int]:
-        return DEFAULT_MAX_DURATION_SECONDS
+        return None
 
     def _spot_policy(self) -> SpotPolicy:
         return self.run_spec.merged_profile.spot_policy or SpotPolicy.ONDEMAND