dstack 0.19.7__py3-none-any.whl → 0.19.9__py3-none-any.whl

dstack/_internal/server/migrations/versions/20166748b60c_add_jobmodel_disconnected_at.py

@@ -0,0 +1,100 @@
+"""Add JobModel.disconnected_at
+
+Revision ID: 20166748b60c
+Revises: 6c1a9d6530ee
+Create Date: 2025-05-13 16:24:32.496578
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from alembic_postgresql_enum import TableReference
+
+import dstack._internal.server.models
+
+# revision identifiers, used by Alembic.
+revision = "20166748b60c"
+down_revision = "6c1a9d6530ee"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "disconnected_at", dstack._internal.server.models.NaiveDateTime(), nullable=True
+            )
+        )
+
+    op.sync_enum_values(
+        enum_schema="public",
+        enum_name="jobterminationreason",
+        new_values=[
+            "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+            "INTERRUPTED_BY_NO_CAPACITY",
+            "INSTANCE_UNREACHABLE",
+            "WAITING_INSTANCE_LIMIT_EXCEEDED",
+            "WAITING_RUNNER_LIMIT_EXCEEDED",
+            "TERMINATED_BY_USER",
+            "VOLUME_ERROR",
+            "GATEWAY_ERROR",
+            "SCALED_DOWN",
+            "DONE_BY_RUNNER",
+            "ABORTED_BY_USER",
+            "TERMINATED_BY_SERVER",
+            "INACTIVITY_DURATION_EXCEEDED",
+            "TERMINATED_DUE_TO_UTILIZATION_POLICY",
+            "CONTAINER_EXITED_WITH_ERROR",
+            "PORTS_BINDING_FAILED",
+            "CREATING_CONTAINER_ERROR",
+            "EXECUTOR_ERROR",
+            "MAX_DURATION_EXCEEDED",
+        ],
+        affected_columns=[
+            TableReference(
+                table_schema="public", table_name="jobs", column_name="termination_reason"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.sync_enum_values(
+        enum_schema="public",
+        enum_name="jobterminationreason",
+        new_values=[
+            "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+            "INTERRUPTED_BY_NO_CAPACITY",
+            "WAITING_INSTANCE_LIMIT_EXCEEDED",
+            "WAITING_RUNNER_LIMIT_EXCEEDED",
+            "TERMINATED_BY_USER",
+            "VOLUME_ERROR",
+            "GATEWAY_ERROR",
+            "SCALED_DOWN",
+            "DONE_BY_RUNNER",
+            "ABORTED_BY_USER",
+            "TERMINATED_BY_SERVER",
+            "INACTIVITY_DURATION_EXCEEDED",
+            "TERMINATED_DUE_TO_UTILIZATION_POLICY",
+            "CONTAINER_EXITED_WITH_ERROR",
+            "PORTS_BINDING_FAILED",
+            "CREATING_CONTAINER_ERROR",
+            "EXECUTOR_ERROR",
+            "MAX_DURATION_EXCEEDED",
+        ],
+        affected_columns=[
+            TableReference(
+                table_schema="public", table_name="jobs", column_name="termination_reason"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.drop_column("disconnected_at")
+
+    # ### end Alembic commands ###

dstack/_internal/server/migrations/versions/6c1a9d6530ee_add_jobmodel_exit_status.py

@@ -0,0 +1,26 @@
+"""Add JobModel.exit_status
+
+Revision ID: 6c1a9d6530ee
+Revises: 7ba3b59d7ca6
+Create Date: 2025-05-09 10:25:19.715852
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "6c1a9d6530ee"
+down_revision = "7ba3b59d7ca6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("exit_status", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.drop_column("exit_status")

dstack/_internal/server/models.py

@@ -382,6 +382,10 @@ class JobModel(BaseModel):
         Enum(JobTerminationReason)
     )
     termination_reason_message: Mapped[Optional[str]] = mapped_column(Text)
+    # `disconnected_at` stores the first time of connectivity issues with the instance.
+    # Resets every time connectivity is restored.
+    disconnected_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
+    exit_status: Mapped[Optional[int]] = mapped_column(Integer)
     job_spec_data: Mapped[str] = mapped_column(Text)
     job_provisioning_data: Mapped[Optional[str]] = mapped_column(Text)
     runner_timestamp: Mapped[Optional[int]] = mapped_column(BigInteger)
@@ -390,7 +394,7 @@ class JobModel(BaseModel):
     remove_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
     volumes_detached_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
     # `instance_assigned` means instance assignment was done.
-    # if `instance_assigned` is True and `instance` is None, no instance was assiged.
+    # if `instance_assigned` is True and `instance` is None, no instance was assigned.
     instance_assigned: Mapped[bool] = mapped_column(Boolean, default=False)
     instance_id: Mapped[Optional[uuid.UUID]] = mapped_column(
         ForeignKey("instances.id", ondelete="CASCADE")
@@ -659,6 +663,7 @@ class PlacementGroupModel(BaseModel):
 
     fleet_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("fleets.id"))
     fleet: Mapped["FleetModel"] = relationship(foreign_keys=[fleet_id])
+    # TODO: rename `fleet_deleted` -> `to_be_deleted`
     fleet_deleted: Mapped[bool] = mapped_column(Boolean, default=False)
 
     created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime)

dstack/_internal/server/schemas/runner.py

@@ -7,7 +7,14 @@ from typing_extensions import Annotated
 
 from dstack._internal.core.models.common import CoreModel, NetworkMode
 from dstack._internal.core.models.repos.remote import RemoteRepoCreds
-from dstack._internal.core.models.runs import ClusterInfo, JobSpec, JobStatus, RunSpec
+from dstack._internal.core.models.runs import (
+    ClusterInfo,
+    JobSpec,
+    JobStatus,
+    JobSubmission,
+    Run,
+    RunSpec,
+)
 from dstack._internal.core.models.volumes import InstanceMountPoint, VolumeMountPoint
 
 
@@ -16,6 +23,7 @@ class JobStateEvent(CoreModel):
     state: JobStatus
     termination_reason: Optional[str] = None
     termination_message: Optional[str] = None
+    exit_status: Optional[int] = None
 
 
 class LogEvent(CoreModel):
@@ -38,15 +46,18 @@ class PullResponse(CoreModel):
 
 
 class SubmitBody(CoreModel):
-    run_spec: Annotated[
-        RunSpec,
+    run: Annotated[
+        Run,
         Field(
             include={
-                "run_name",
-                "repo_id",
-                "repo_data",
-                "configuration",
-                "configuration_path",
+                "id": True,
+                "run_spec": {
+                    "run_name",
+                    "repo_id",
+                    "repo_data",
+                    "configuration",
+                    "configuration_path",
+                },
             }
         ),
     ]
@@ -69,9 +80,31 @@ class SubmitBody(CoreModel):
             }
         ),
     ]
+    job_submission: Annotated[
+        JobSubmission,
+        Field(
+            include={
+                "id",
+            }
+        ),
+    ]
     cluster_info: Annotated[Optional[ClusterInfo], Field(include=True)]
     secrets: Annotated[Optional[Dict[str, str]], Field(include=True)]
     repo_credentials: Annotated[Optional[RemoteRepoCreds], Field(include=True)]
+    # run_spec is deprecated in favor of run.run_spec
+    # TODO: Remove once we no longer support instances deployed with 0.19.8 or earlier.
+    run_spec: Annotated[
+        RunSpec,
+        Field(
+            include={
+                "run_name",
+                "repo_id",
+                "repo_data",
+                "configuration",
+                "configuration_path",
+            },
+        ),
+    ]
 
 
 class HealthcheckResponse(CoreModel):

dstack/_internal/server/services/fleets.py

@@ -1,5 +1,3 @@
-import random
-import string
 import uuid
 from datetime import datetime, timezone
 from typing import List, Literal, Optional, Tuple, Union, cast
@@ -33,6 +31,7 @@ from dstack._internal.core.models.instances import (
     SSHConnectionParams,
     SSHKey,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.profiles import (
     Profile,
     SpotPolicy,
@@ -62,6 +61,7 @@ from dstack._internal.server.services.projects import (
     list_project_models,
     list_user_project_models,
 )
+from dstack._internal.server.services.resources import set_resources_defaults
 from dstack._internal.utils import random_names
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.ssh import pkey_from_str
@@ -243,6 +243,7 @@ async def get_plan(
         spec=effective_spec,
     )
     effective_spec = FleetSpec.parse_obj(effective_spec.dict())
+    _validate_fleet_spec_and_set_defaults(spec)
     current_fleet: Optional[Fleet] = None
     current_fleet_id: Optional[uuid.UUID] = None
     if effective_spec.configuration.name is not None:
@@ -282,6 +283,7 @@ async def get_create_instance_offers(
     project: ProjectModel,
     profile: Profile,
     requirements: Requirements,
+    placement_group: Optional[PlacementGroup] = None,
     fleet_spec: Optional[FleetSpec] = None,
     fleet_model: Optional[FleetModel] = None,
     blocks: Union[int, Literal["auto"]] = 1,
@@ -307,6 +309,7 @@ async def get_create_instance_offers(
         exclude_not_available=exclude_not_available,
         multinode=multinode,
         master_job_provisioning_data=master_job_provisioning_data,
+        placement_group=placement_group,
         blocks=blocks,
     )
     offers = [
@@ -345,7 +348,7 @@ async def create_fleet(
         spec=spec,
     )
     spec = FleetSpec.parse_obj(spec.dict())
-    _validate_fleet_spec(spec)
+    _validate_fleet_spec_and_set_defaults(spec)
 
     if spec.configuration.ssh_config is not None:
         _check_can_manage_ssh_fleets(user=user, project=project)
@@ -393,17 +396,12 @@ async def create_fleet(
                 )
                 fleet_model.instances.append(instances_model)
         else:
-            placement_group_name = _get_placement_group_name(
-                project=project,
-                fleet_spec=spec,
-            )
             for i in range(_get_fleet_nodes_to_provision(spec)):
                 instance_model = await create_fleet_instance_model(
                     session=session,
                     project=project,
                     user=user,
                     spec=spec,
-                    placement_group_name=placement_group_name,
                     reservation=spec.configuration.reservation,
                     instance_num=i,
                 )
@@ -417,7 +415,6 @@ async def create_fleet_instance_model(
     project: ProjectModel,
     user: UserModel,
     spec: FleetSpec,
-    placement_group_name: Optional[str],
     reservation: Optional[str],
     instance_num: int,
 ) -> InstanceModel:
@@ -431,7 +428,6 @@ async def create_fleet_instance_model(
         requirements=requirements,
         instance_name=f"{spec.configuration.name}-{instance_num}",
         instance_num=instance_num,
-        placement_group_name=placement_group_name,
         reservation=reservation,
         blocks=spec.configuration.blocks,
         tags=spec.configuration.tags,
@@ -652,7 +648,7 @@ def _remove_fleet_spec_sensitive_info(spec: FleetSpec):
                 host.ssh_key = None
 
 
-def _validate_fleet_spec(spec: FleetSpec):
+def _validate_fleet_spec_and_set_defaults(spec: FleetSpec):
     if spec.configuration.name is not None:
         validate_dstack_resource_name(spec.configuration.name)
     if spec.configuration.ssh_config is None and spec.configuration.nodes is None:
@@ -665,6 +661,8 @@ def _validate_fleet_spec(spec: FleetSpec):
             if isinstance(host, SSHHostParams) and host.ssh_key is not None:
                 _validate_ssh_key(host.ssh_key)
         _validate_internal_ips(spec.configuration.ssh_config)
+    if spec.configuration.resources is not None:
+        set_resources_defaults(spec.configuration.resources)
 
 
 def _validate_all_ssh_params_specified(ssh_config: SSHParams):
@@ -735,18 +733,3 @@ def _get_fleet_requirements(fleet_spec: FleetSpec) -> Requirements:
         reservation=fleet_spec.configuration.reservation,
     )
     return requirements
-
-
-def _get_placement_group_name(
-    project: ProjectModel,
-    fleet_spec: FleetSpec,
-) -> Optional[str]:
-    if fleet_spec.configuration.placement != InstanceGroupPlacement.CLUSTER:
-        return None
-    # A random suffix to avoid clashing with to-be-deleted placement groups left by old fleets
-    suffix = _generate_random_placement_group_suffix()
-    return f"{project.name}-{fleet_spec.configuration.name}-{suffix}-pg"
-
-
-def _generate_random_placement_group_suffix(length: int = 8) -> str:
-    return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))

dstack/_internal/server/services/instances.py

@@ -408,7 +408,6 @@ async def create_instance_model(
     requirements: Requirements,
     instance_name: str,
     instance_num: int,
-    placement_group_name: Optional[str],
     reservation: Optional[str],
     blocks: Union[Literal["auto"], int],
     tags: Optional[Dict[str, str]],
@@ -427,7 +426,6 @@ async def create_instance_model(
         user=user.name,
         ssh_keys=[project_ssh_key],
         instance_id=str(instance_id),
-        placement_group_name=placement_group_name,
         reservation=reservation,
         tags=tags,
     )

dstack/_internal/server/services/jobs/__init__.py

@@ -135,6 +135,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         status=job_model.status,
         termination_reason=job_model.termination_reason,
         termination_reason_message=job_model.termination_reason_message,
+        exit_status=job_model.exit_status,
         job_provisioning_data=job_provisioning_data,
         job_runtime_data=get_job_runtime_data(job_model),
     )

dstack/_internal/server/services/offers.py

@@ -8,12 +8,14 @@ from dstack._internal.core.backends import (
     BACKENDS_WITH_RESERVATION_SUPPORT,
 )
 from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,
     InstanceType,
     Resources,
 )
+from dstack._internal.core.models.placement import PlacementGroup
 from dstack._internal.core.models.profiles import Profile
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.core.models.volumes import Volume
@@ -31,6 +33,7 @@ async def get_offers_by_requirements(
     volumes: Optional[List[List[Volume]]] = None,
     privileged: bool = False,
     instance_mounts: bool = False,
+    placement_group: Optional[PlacementGroup] = None,
     blocks: Union[int, Literal["auto"]] = 1,
 ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]:
     backends: List[Backend] = await backends_services.get_project_backends(project=project)
@@ -116,6 +119,18 @@ async def get_offers_by_requirements(
                     new_offers.append((b, new_offer))
         offers = new_offers
 
+    if placement_group is not None:
+        new_offers = []
+        for b, o in offers:
+            for backend in backends:
+                compute = backend.compute()
+                if isinstance(
+                    compute, ComputeWithPlacementGroupSupport
+                ) and compute.is_suitable_placement_group(placement_group, o):
+                    new_offers.append((b, o))
+                    break
+        offers = new_offers
+
     if profile.instance_types is not None:
         instance_types = [i.lower() for i in profile.instance_types]
         offers = [(b, o) for b, o in offers if o.instance.name.lower() in instance_types]

dstack/_internal/server/services/placement.py

@@ -1,8 +1,9 @@
+from collections.abc import Iterable
 from typing import Optional
 from uuid import UUID
 
 from git import List
-from sqlalchemy import select
+from sqlalchemy import and_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.models.placement import (
@@ -13,15 +14,35 @@ from dstack._internal.core.models.placement import (
 from dstack._internal.server.models import PlacementGroupModel
 
 
-async def get_fleet_placement_groups(
+async def get_fleet_placement_group_models(
     session: AsyncSession,
     fleet_id: UUID,
-) -> List[PlacementGroup]:
+) -> List[PlacementGroupModel]:
     res = await session.execute(
-        select(PlacementGroupModel).where(PlacementGroupModel.fleet_id == fleet_id)
+        select(PlacementGroupModel).where(
+            and_(
+                PlacementGroupModel.fleet_id == fleet_id,
+                PlacementGroupModel.deleted == False,
+                PlacementGroupModel.fleet_deleted == False,
+            )
+        )
+    )
+    return list(res.scalars().all())
+
+
+async def schedule_fleet_placement_groups_deletion(
+    session: AsyncSession, fleet_id: UUID, except_placement_group_ids: Iterable[UUID] = ()
+) -> None:
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            and_(
+                PlacementGroupModel.fleet_id == fleet_id,
+                PlacementGroupModel.id.not_in(except_placement_group_ids),
+            )
+        )
+        .values(fleet_deleted=True)  # TODO: rename `fleet_deleted` -> `to_be_deleted`
     )
-    placement_groups = res.scalars().all()
-    return [placement_group_model_to_placement_group(pg) for pg in placement_groups]
 
 
 def placement_group_model_to_placement_group(

dstack/_internal/server/services/resources.py

@@ -0,0 +1,21 @@
+import gpuhunt
+from pydantic import parse_obj_as
+
+from dstack._internal.core.models.resources import CPUSpec, ResourcesSpec
+
+
+def set_resources_defaults(resources: ResourcesSpec) -> None:
+    # TODO: Remove in 0.20. Use resources.cpu directly
+    cpu = parse_obj_as(CPUSpec, resources.cpu)
+    if cpu.arch is None:
+        gpu = resources.gpu
+        if (
+            gpu is not None
+            and gpu.vendor in [None, gpuhunt.AcceleratorVendor.NVIDIA]
+            and gpu.name
+            and any(map(gpuhunt.is_nvidia_superchip, gpu.name))
+        ):
+            cpu.arch = gpuhunt.CPUArchitecture.ARM
+        else:
+            cpu.arch = gpuhunt.CPUArchitecture.X86
+        resources.cpu = cpu

dstack/_internal/server/services/runner/client.py

@@ -12,7 +12,7 @@ from dstack._internal.core.models.common import CoreModel, NetworkMode
 from dstack._internal.core.models.envs import Env
 from dstack._internal.core.models.repos.remote import RemoteRepoCreds
 from dstack._internal.core.models.resources import Memory
-from dstack._internal.core.models.runs import ClusterInfo, JobSpec, RunSpec
+from dstack._internal.core.models.runs import ClusterInfo, Job, Run
 from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, VolumeMountPoint
 from dstack._internal.server.schemas.runner import (
     GPUDevice,
@@ -72,8 +72,8 @@ class RunnerClient:
 
     def submit_job(
         self,
-        run_spec: RunSpec,
-        job_spec: JobSpec,
+        run: Run,
+        job: Job,
         cluster_info: ClusterInfo,
         secrets: Dict[str, str],
         repo_credentials: Optional[RemoteRepoCreds],
@@ -81,6 +81,7 @@ class RunnerClient:
     ):
         # XXX: This is a quick-and-dirty hack to deliver InstanceModel-specific environment
         # variables to the runner without runner API modification.
+        job_spec = job.job_spec
         if instance_env is not None:
             if isinstance(instance_env, Env):
                 merged_env = instance_env.as_dict()
@@ -90,11 +91,13 @@ class RunnerClient:
             job_spec = job_spec.copy(deep=True)
             job_spec.env = merged_env
         body = SubmitBody(
-            run_spec=run_spec,
+            run=run,
             job_spec=job_spec,
+            job_submission=job.job_submissions[-1],
             cluster_info=cluster_info,
             secrets=secrets,
             repo_credentials=repo_credentials,
+            run_spec=run.run_spec,
         )
         resp = requests.post(
             # use .json() to encode enums

dstack/_internal/server/services/runs.py

@@ -81,6 +81,7 @@ from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
 from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
+from dstack._internal.server.services.resources import set_resources_defaults
 from dstack._internal.server.services.users import get_user_model_by_name
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.random_names import generate_name
@@ -301,12 +302,14 @@ async def get_plan(
             project=project,
             run_name=effective_run_spec.run_name,
         )
-        if (
-            current_resource is not None
-            and not current_resource.status.is_finished()
-            and _can_update_run_spec(current_resource.run_spec, effective_run_spec)
-        ):
-            action = ApplyAction.UPDATE
+        if current_resource is not None:
+            # For backward compatibility (current_resource may has been submitted before
+            # some fields, e.g., CPUSpec.arch, were added)
+            set_resources_defaults(current_resource.run_spec.configuration.resources)
+            if not current_resource.status.is_finished() and _can_update_run_spec(
+                current_resource.run_spec, effective_run_spec
+            ):
+                action = ApplyAction.UPDATE
 
     jobs = await get_jobs_from_run_spec(effective_run_spec, replica_num=0)
 
@@ -406,6 +409,10 @@ async def apply_plan(
             project=project,
             run_spec=run_spec,
         )
+
+    # For backward compatibility (current_resource may has been submitted before
+    # some fields, e.g., CPUSpec.arch, were added)
+    set_resources_defaults(current_resource.run_spec.configuration.resources)
     try:
         _check_can_update_run_spec(current_resource.run_spec, run_spec)
     except ServerClientError:
@@ -414,6 +421,8 @@ async def apply_plan(
             raise ServerClientError("Cannot override active run. Stop the run first.")
         raise
     if not force:
+        if plan.current_resource is not None:
+            set_resources_defaults(plan.current_resource.run_spec.configuration.resources)
         if (
             plan.current_resource is None
             or plan.current_resource.id != current_resource.id
@@ -861,11 +870,12 @@ def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
     if (
         run_spec.merged_profile.utilization_policy is not None
         and run_spec.merged_profile.utilization_policy.time_window
-        > settings.SERVER_METRICS_TTL_SECONDS
+        > settings.SERVER_METRICS_RUNNING_TTL_SECONDS
     ):
         raise ServerClientError(
-            f"Maximum utilization_policy.time_window is {settings.SERVER_METRICS_TTL_SECONDS}s"
+            f"Maximum utilization_policy.time_window is {settings.SERVER_METRICS_RUNNING_TTL_SECONDS}s"
         )
+    set_resources_defaults(run_spec.configuration.resources)
 
 
 _UPDATABLE_SPEC_FIELDS = ["repo_code_hash", "configuration"]

dstack/_internal/server/settings.py

@@ -1,4 +1,5 @@
 import os
+import warnings
 from pathlib import Path
 
 DSTACK_DIR_PATH = Path("~/.dstack/").expanduser()
@@ -45,7 +46,25 @@ SERVER_CLOUDWATCH_LOG_REGION = os.getenv("DSTACK_SERVER_CLOUDWATCH_LOG_REGION")
 
 SERVER_GCP_LOGGING_PROJECT = os.getenv("DSTACK_SERVER_GCP_LOGGING_PROJECT")
 
-SERVER_METRICS_TTL_SECONDS = int(os.getenv("DSTACK_SERVER_METRICS_TTL_SECONDS", 3600))
+SERVER_METRICS_RUNNING_TTL_SECONDS: int
+_SERVER_METRICS_RUNNING_TTL_SECONDS = os.getenv("DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS")
+if _SERVER_METRICS_RUNNING_TTL_SECONDS is None:
+    _SERVER_METRICS_RUNNING_TTL_SECONDS = os.getenv("DSTACK_SERVER_METRICS_TTL_SECONDS")
+    if _SERVER_METRICS_RUNNING_TTL_SECONDS is not None:
+        warnings.warn(
+            (
+                "DSTACK_SERVER_METRICS_TTL_SECONDS is deprecated,"
+                " use DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS instead"
+            ),
+            DeprecationWarning,
+        )
+    else:
+        _SERVER_METRICS_RUNNING_TTL_SECONDS = 3600
+SERVER_METRICS_RUNNING_TTL_SECONDS = int(_SERVER_METRICS_RUNNING_TTL_SECONDS)
+del _SERVER_METRICS_RUNNING_TTL_SECONDS
+SERVER_METRICS_FINISHED_TTL_SECONDS = int(
+    os.getenv("DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS", 7 * 24 * 3600)
+)
 
 DEFAULT_PROJECT_NAME = "main"