dstack 0.19.34__py3-none-any.whl → 0.19.35__py3-none-any.whl

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -3,16 +3,22 @@ import itertools
 import math
 import uuid
 from datetime import datetime, timedelta
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from sqlalchemy import and_, func, not_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import contains_eager, joinedload, load_only, noload, selectinload
 
 from dstack._internal.core.backends.base.backend import Backend
-from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
+from dstack._internal.core.backends.base.compute import (
+    ComputeWithGroupProvisioningSupport,
+    ComputeWithVolumeSupport,
+)
+from dstack._internal.core.backends.base.models import JobConfiguration
+from dstack._internal.core.backends.features import BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT
 from dstack._internal.core.errors import BackendError, ServerClientError
 from dstack._internal.core.models.common import NetworkMode
+from dstack._internal.core.models.compute_groups import ComputeGroupProvisioningData
 from dstack._internal.core.models.fleets import (
     Fleet,
     FleetConfiguration,
@@ -42,8 +48,10 @@ from dstack._internal.core.models.runs import (
 from dstack._internal.core.models.volumes import Volume
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.server import settings
+from dstack._internal.server.background.tasks.process_compute_groups import ComputeGroupStatus
 from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
+    ComputeGroupModel,
     FleetModel,
     InstanceModel,
     JobModel,
@@ -69,6 +77,7 @@ from dstack._internal.server.services.instances import (
 from dstack._internal.server.services.jobs import (
     check_can_attach_job_volumes,
     find_job,
+    find_jobs,
     get_instances_ids_with_detaching_volumes,
     get_job_configured_volume_models,
     get_job_configured_volumes,
@@ -132,6 +141,7 @@ async def _process_next_submitted_job():
                 .join(JobModel.run)
                 .where(
                     JobModel.status == JobStatus.SUBMITTED,
+                    JobModel.waiting_master_job.is_not(True),
                     JobModel.id.not_in(lockset),
                 )
                 .options(load_only(JobModel.id))
@@ -190,6 +200,8 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
     run_spec = run.run_spec
     run_profile = run_spec.merged_profile
     job = find_job(run.jobs, job_model.replica_num, job_model.job_num)
+    replica_jobs = find_jobs(run.jobs, replica_num=job_model.replica_num)
+    replica_job_models = _get_job_models_for_jobs(run_model.jobs, replica_jobs)
     multinode = job.job_spec.jobs_per_replica > 1
 
     # Master job chooses fleet for the run.
@@ -323,6 +335,10 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
                 return
             # If no instances were locked, we can proceed in the same transaction.
 
+    # TODO: Volume attachment for compute groups is not yet supported since
+    # currently supported compute groups (e.g. Runpod) don't need explicit volume attachment.
+    need_volume_attachment = True
+
     if job_model.instance is not None:
         res = await session.execute(
             select(InstanceModel)
@@ -333,7 +349,6 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         instance = res.unique().scalar_one()
         job_model.status = JobStatus.PROVISIONING
     else:
-        # Assigned no instance, create a new one
         if run_profile.creation_policy == CreationPolicy.REUSE:
             logger.debug("%s: reuse instance failed", fmt(job_model))
             job_model.status = JobStatus.TERMINATING
@@ -342,13 +357,23 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             await session.commit()
             return
 
-        # Create a new cloud instance
-        run_job_result = await _run_job_on_new_instance(
+        jobs_to_provision = [job]
+        if (
+            multinode
+            and job.job_spec.job_num == 0
+            # job_model.waiting_master_job is not set for legacy jobs.
+            # In this case compute group provisioning not supported
+            # and jobs always provision one-by-one.
+            and job_model.waiting_master_job is not None
+        ):
+            jobs_to_provision = replica_jobs
+
+        run_job_result = await _run_jobs_on_new_instances(
             project=project,
             fleet_model=fleet_model,
             job_model=job_model,
             run=run,
-            job=job,
+            jobs=jobs_to_provision,
             project_ssh_public_key=project.ssh_public_key,
             project_ssh_private_key=project.ssh_private_key,
             master_job_provisioning_data=master_job_provisioning_data,
@@ -362,72 +387,102 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             await session.commit()
             return
 
-        logger.info("%s: now is provisioning a new instance", fmt(job_model))
-        job_provisioning_data, offer, effective_profile, _ = run_job_result
-        job_model.job_provisioning_data = job_provisioning_data.json()
-        job_model.status = JobStatus.PROVISIONING
         if fleet_model is None:
             fleet_model = await _create_fleet_model_for_job(
                 session=session,
                 project=project,
                 run=run,
             )
-        # FIXME: Fleet is not locked which may lead to duplicate instance_num.
-        # This is currently hard to fix without locking the fleet for entire provisioning duration.
-        # Processing should be done in multiple steps so that
-        # InstanceModel is created before provisioning.
-        instance_num = await _get_next_instance_num(
-            session=session,
-            fleet_model=fleet_model,
-        )
-        instance = _create_instance_model_for_job(
-            project=project,
-            fleet_model=fleet_model,
-            job_model=job_model,
-            job_provisioning_data=job_provisioning_data,
-            offer=offer,
-            instance_num=instance_num,
-            profile=effective_profile,
-        )
-        job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
-        # Both this task and process_fleets can add instances to fleets.
-        # TODO: Ensure this does not violate nodes.max when it's enforced.
-        instance.fleet_id = fleet_model.id
-        logger.info(
-            "The job %s created the new instance %s",
-            job_model.job_name,
-            instance.name,
-            extra={
-                "instance_name": instance.name,
-                "instance_status": InstanceStatus.PROVISIONING.value,
-            },
-        )
-        session.add(instance)
-        session.add(fleet_model)
-        job_model.used_instance_id = instance.id
-
-    volumes_ids = sorted([v.id for vs in volume_models for v in vs])
-    # TODO: lock instances for attaching volumes?
-    # Take lock to prevent attaching volumes that are to be deleted.
-    # If the volume was deleted before the lock, the volume will fail to attach and the job will fail.
-    await session.execute(
-        select(VolumeModel)
-        .where(VolumeModel.id.in_(volumes_ids))
-        .options(joinedload(VolumeModel.user).load_only(UserModel.name))
-        .order_by(VolumeModel.id)  # take locks in order
-        .with_for_update(key_share=True, of=VolumeModel)
-    )
-    async with get_locker(get_db().dialect_name).lock_ctx(VolumeModel.__tablename__, volumes_ids):
-        if len(volume_models) > 0:
-            await _attach_volumes(
+            session.add(fleet_model)
+
+        provisioning_data, offer, effective_profile, _ = run_job_result
+        compute_group_model = None
+        if isinstance(provisioning_data, ComputeGroupProvisioningData):
+            need_volume_attachment = False
+            provisioned_jobs = jobs_to_provision
+            jpds = provisioning_data.job_provisioning_datas
+            compute_group_model = ComputeGroupModel(
+                id=uuid.uuid4(),
+                project=project,
+                fleet=fleet_model,
+                status=ComputeGroupStatus.RUNNING,
+                provisioning_data=provisioning_data.json(),
+            )
+            session.add(compute_group_model)
+        else:
+            provisioned_jobs = [job]
+            jpds = [provisioning_data]
+            if len(jobs_to_provision) > 1:
+                # Tried provisioning multiple jobs but provisioned only one.
+                # Allow other jobs to provision one-by-one.
+                for replica_job_model in replica_job_models:
+                    replica_job_model.waiting_master_job = False
+
+        logger.info("%s: provisioned %s new instance(s)", fmt(job_model), len(provisioned_jobs))
+        provisioned_job_models = _get_job_models_for_jobs(run_model.jobs, provisioned_jobs)
+        instance = None  # Instance for attaching volumes in case of single job provisioned
+        for provisioned_job_model, jpd in zip(provisioned_job_models, jpds):
+            provisioned_job_model.job_provisioning_data = jpd.json()
+            provisioned_job_model.status = JobStatus.PROVISIONING
+            # FIXME: Fleet is not locked which may lead to duplicate instance_num.
+            # This is currently hard to fix without locking the fleet for entire provisioning duration.
+            # Processing should be done in multiple steps so that
+            # InstanceModel is created before provisioning.
+            instance_num = await _get_next_instance_num(
                 session=session,
+                fleet_model=fleet_model,
+            )
+            instance = _create_instance_model_for_job(
                 project=project,
-                job_model=job_model,
-                instance=instance,
-                volume_models=volume_models,
+                fleet_model=fleet_model,
+                compute_group_model=compute_group_model,
+                job_model=provisioned_job_model,
+                job_provisioning_data=jpd,
+                offer=offer,
+                instance_num=instance_num,
+                profile=effective_profile,
             )
-        job_model.last_processed_at = common_utils.get_current_datetime()
-        await session.commit()
+            provisioned_job_model.job_runtime_data = _prepare_job_runtime_data(
+                offer, multinode
+            ).json()
+            logger.info(
+                "Created a new instance %s for job %s",
+                instance.name,
+                provisioned_job_model.job_name,
+                extra={
+                    "instance_name": instance.name,
+                    "instance_status": InstanceStatus.PROVISIONING.value,
+                },
+            )
+            session.add(instance)
+            provisioned_job_model.used_instance_id = instance.id
+            provisioned_job_model.last_processed_at = common_utils.get_current_datetime()
+
+    volumes_ids = sorted([v.id for vs in volume_models for v in vs])
+    if need_volume_attachment:
+        # TODO: Lock instances for attaching volumes?
+        # Take lock to prevent attaching volumes that are to be deleted.
+        # If the volume was deleted before the lock, the volume will fail to attach and the job will fail.
+        await session.execute(
+            select(VolumeModel)
+            .where(VolumeModel.id.in_(volumes_ids))
+            .options(joinedload(VolumeModel.user).load_only(UserModel.name))
+            .order_by(VolumeModel.id)  # take locks in order
+            .with_for_update(key_share=True, of=VolumeModel)
+        )
+        async with get_locker(get_db().dialect_name).lock_ctx(
+            VolumeModel.__tablename__, volumes_ids
+        ):
+            if len(volume_models) > 0:
+                assert instance is not None
+                await _attach_volumes(
+                    session=session,
+                    project=project,
+                    job_model=job_model,
+                    instance=instance,
+                    volume_models=volume_models,
+                )
+            await session.commit()
 
 
 async def _select_fleet_models(
@@ -553,10 +608,9 @@ async def _find_optimal_fleet_with_offers(
         except ValueError:
             fleet_backend_offers = []
         else:
-            multinode = (
-                candidate_fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
-                or job.job_spec.jobs_per_replica > 1
-            )
+            # Handle multinode for old jobs that don't have requirements.multinode set.
+            # TODO: Drop multinode param.
+            multinode = requirements.multinode or job.job_spec.jobs_per_replica > 1
             fleet_backend_offers = await get_offers_by_requirements(
                 project=project,
                 profile=profile,
@@ -728,19 +782,33 @@ async def _assign_job_to_fleet_instance(
     return instance
 
 
-async def _run_job_on_new_instance(
+async def _run_jobs_on_new_instances(
     project: ProjectModel,
     job_model: JobModel,
     run: Run,
-    job: Job,
+    jobs: list[Job],
     project_ssh_public_key: str,
     project_ssh_private_key: str,
     master_job_provisioning_data: Optional[JobProvisioningData] = None,
-    volumes: Optional[List[List[Volume]]] = None,
+    volumes: Optional[list[list[Volume]]] = None,
     fleet_model: Optional[FleetModel] = None,
-) -> Optional[tuple[JobProvisioningData, InstanceOfferWithAvailability, Profile, Requirements]]:
+) -> Optional[
+    tuple[
+        Union[JobProvisioningData, ComputeGroupProvisioningData],
+        InstanceOfferWithAvailability,
+        Profile,
+        Requirements,
+    ]
+]:
+    """
+    Provisions an instance for a job or a compute group for multiple jobs and runs the jobs.
+    Even when multiple jobs are passes, it may still provision only one instance
+    and run only the master job in case there are no offers supporting cluster groups.
+    Other jobs should be provisioned one-by-one later.
+    """
     if volumes is None:
         volumes = []
+    job = jobs[0]
     profile = run.run_spec.merged_profile
     requirements = job.job_spec.requirements
     fleet = None
@@ -758,9 +826,7 @@ async def _run_job_on_new_instance(
             return None
         # TODO: Respect fleet provisioning properties such as tags
 
-    multinode = job.job_spec.jobs_per_replica > 1 or (
-        fleet is not None and fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
-    )
+    multinode = requirements.multinode or job.job_spec.jobs_per_replica > 1
     offers = await get_offers_by_requirements(
         project=project,
         profile=profile,
@@ -784,17 +850,31 @@ async def _run_job_on_new_instance(
             offer.price,
         )
         offer_volumes = _get_offer_volumes(volumes, offer)
+        job_configurations = [JobConfiguration(job=j, volumes=offer_volumes) for j in jobs]
+        compute = backend.compute()
         try:
-            job_provisioning_data = await common_utils.run_async(
-                backend.compute().run_job,
-                run,
-                job,
-                offer,
-                project_ssh_public_key,
-                project_ssh_private_key,
-                offer_volumes,
-            )
-            return job_provisioning_data, offer, profile, requirements
+            if len(jobs) > 1 and offer.backend in BACKENDS_WITH_GROUP_PROVISIONING_SUPPORT:
+                assert isinstance(compute, ComputeWithGroupProvisioningSupport)
+                cgpd = await common_utils.run_async(
+                    compute.run_jobs,
+                    run,
+                    job_configurations,
+                    offer,
+                    project_ssh_public_key,
+                    project_ssh_private_key,
+                )
+                return cgpd, offer, profile, requirements
+            else:
+                jpd = await common_utils.run_async(
+                    compute.run_job,
+                    run,
+                    job,
+                    offer,
+                    project_ssh_public_key,
+                    project_ssh_private_key,
+                    offer_volumes,
+                )
+                return jpd, offer, profile, requirements
         except BackendError as e:
             logger.warning(
                 "%s: %s launch in %s/%s failed: %s",
@@ -912,6 +992,7 @@ async def _get_next_instance_num(session: AsyncSession, fleet_model: FleetModel)
 def _create_instance_model_for_job(
     project: ProjectModel,
     fleet_model: FleetModel,
+    compute_group_model: Optional[ComputeGroupModel],
     job_model: JobModel,
     job_provisioning_data: JobProvisioningData,
     offer: InstanceOfferWithAvailability,
@@ -931,6 +1012,8 @@ def _create_instance_model_for_job(
         name=f"{fleet_model.name}-{instance_num}",
         instance_num=instance_num,
         project=project,
+        fleet=fleet_model,
+        compute_group=compute_group_model,
         created_at=common_utils.get_current_datetime(),
         started_at=common_utils.get_current_datetime(),
         status=InstanceStatus.PROVISIONING,
@@ -1081,3 +1164,15 @@ async def _attach_volume(
     instance.volume_attachments.append(volume_attachment_model)
 
     volume_model.last_job_processed_at = common_utils.get_current_datetime()
+
+
+def _get_job_models_for_jobs(
+    job_models: list[JobModel],
+    jobs: list[Job],
+) -> list[JobModel]:
+    """
+    Returns job models of latest submissions for a list of jobs.
+    Preserves jobs order.
+    """
+    id_to_job_model_map = {jm.id: jm for jm in job_models}
+    return [id_to_job_model_map[j.job_submissions[-1].id] for j in jobs]

dstack/_internal/server/migrations/env.py

@@ -6,7 +6,7 @@ from alembic import context
 from sqlalchemy import Connection, MetaData, text
 
 from dstack._internal.server.db import get_db
-from dstack._internal.server.models import BaseModel
+from dstack._internal.server.models import BaseModel, EnumAsString
 
 config = context.config
 
@@ -21,6 +21,14 @@ def set_target_metadata(metadata: MetaData):
     target_metadata = metadata
 
 
+def render_item(type_, obj, autogen_context):
+    """Apply custom rendering for selected items."""
+    if type_ == "type" and isinstance(obj, EnumAsString):
+        return f"sa.String(length={obj.length})"
+    # default rendering for other objects
+    return False
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
     This configures the context with just a URL
@@ -35,8 +43,8 @@ def run_migrations_offline():
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        render_item=render_item,
     )
-
     with context.begin_transaction():
         context.run_migrations()
 
@@ -61,12 +69,22 @@ def run_migrations(connection: Connection):
     # https://alembic.sqlalchemy.org/en/latest/batch.html#dealing-with-referencing-foreign-keys
     if connection.dialect.name == "sqlite":
         connection.execute(text("PRAGMA foreign_keys=OFF;"))
+    elif connection.dialect.name == "postgresql":
+        # lock_timeout is needed so that migrations that acquire locks
+        # do not wait for locks forever, blocking live queries.
+        # Better to fail and retry a deployment.
+        connection.execute(text("SET lock_timeout='10s';"))
     connection.commit()
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
         compare_type=True,
         render_as_batch=True,
+        render_item=render_item,
+        # Running each migration in a separate transaction.
+        # Running all migrations in one transaction may lead to deadlocks in HA deployments
+        # because lock ordering is not respected across all migrations.
+        transaction_per_migration=True,
     )
     with context.begin_transaction():
         context.run_migrations()

dstack/_internal/server/migrations/versions/7d1ec2b920ac_add_computegroupmodel.py

@@ -0,0 +1,93 @@
+"""Add ComputeGroupModel
+
+Revision ID: 7d1ec2b920ac
+Revises: ff1d94f65b08
+Create Date: 2025-10-21 16:01:23.739395
+
+"""
+
+import sqlalchemy as sa
+import sqlalchemy_utils
+from alembic import op
+
+import dstack._internal.server.models
+
+# revision identifiers, used by Alembic.
+revision = "7d1ec2b920ac"
+down_revision = "ff1d94f65b08"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "compute_groups",
+        sa.Column("id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False),
+        sa.Column(
+            "project_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False
+        ),
+        sa.Column("fleet_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False),
+        sa.Column("created_at", dstack._internal.server.models.NaiveDateTime(), nullable=False),
+        sa.Column("status", sa.String(length=100), nullable=False),
+        sa.Column(
+            "last_processed_at", dstack._internal.server.models.NaiveDateTime(), nullable=False
+        ),
+        sa.Column("deleted", sa.Boolean(), nullable=False),
+        sa.Column("deleted_at", dstack._internal.server.models.NaiveDateTime(), nullable=True),
+        sa.Column("provisioning_data", sa.Text(), nullable=False),
+        sa.Column(
+            "first_termination_retry_at",
+            dstack._internal.server.models.NaiveDateTime(),
+            nullable=True,
+        ),
+        sa.Column(
+            "last_termination_retry_at",
+            dstack._internal.server.models.NaiveDateTime(),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["fleet_id"], ["fleets.id"], name=op.f("fk_compute_groups_fleet_id_fleets")
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projects.id"],
+            name=op.f("fk_compute_groups_project_id_projects"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_compute_groups")),
+    )
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "compute_group_id",
+                sqlalchemy_utils.types.uuid.UUIDType(binary=False),
+                nullable=True,
+            )
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_instances_compute_group_id_compute_groups"),
+            "compute_groups",
+            ["compute_group_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("waiting_master_job", sa.Boolean(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.drop_column("waiting_master_job")
+
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_instances_compute_group_id_compute_groups"), type_="foreignkey"
+        )
+        batch_op.drop_column("compute_group_id")
+
+    op.drop_table("compute_groups")
+    # ### end Alembic commands ###

dstack/_internal/server/models.py

@@ -25,6 +25,7 @@ from sqlalchemy_utils import UUIDType
 from dstack._internal.core.errors import DstackError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreConfig, generate_dual_core_model
+from dstack._internal.core.models.compute_groups import ComputeGroupStatus
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.core.models.gateways import GatewayStatus
 from dstack._internal.core.models.health import HealthStatus
@@ -448,6 +449,12 @@ class JobModel(BaseModel):
     # Whether the replica is registered to receive service requests.
     # Always `False` for non-service runs.
     registered: Mapped[bool] = mapped_column(Boolean, server_default=false())
+    # `waiting_master_job` is `True` for non-master jobs that have to wait
+    # for master processing before they can be processed.
+    # This allows updating all replica jobs even when only master is locked,
+    # e.g. to provision instances for all jobs when processing master.
+    # If not set, all jobs should be processed only one-by-one.
+    waiting_master_job: Mapped[Optional[bool]] = mapped_column(Boolean)
 
 
 class GatewayModel(BaseModel):
@@ -592,6 +599,9 @@ class InstanceModel(BaseModel):
     fleet_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("fleets.id"))
     fleet: Mapped[Optional["FleetModel"]] = relationship(back_populates="instances")
 
+    compute_group_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("compute_groups.id"))
+    compute_group: Mapped[Optional["ComputeGroupModel"]] = relationship(back_populates="instances")
+
     status: Mapped[InstanceStatus] = mapped_column(EnumAsString(InstanceStatus, 100), index=True)
     unreachable: Mapped[bool] = mapped_column(Boolean)
 
@@ -743,6 +753,35 @@ class PlacementGroupModel(BaseModel):
     provisioning_data: Mapped[Optional[str]] = mapped_column(Text)
 
 
+class ComputeGroupModel(BaseModel):
+    __tablename__ = "compute_groups"
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        UUIDType(binary=False), primary_key=True, default=uuid.uuid4
+    )
+
+    project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"))
+    project: Mapped["ProjectModel"] = relationship(foreign_keys=[project_id])
+
+    fleet_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("fleets.id"))
+    fleet: Mapped["FleetModel"] = relationship(foreign_keys=[fleet_id])
+
+    created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime)
+    status: Mapped[ComputeGroupStatus] = mapped_column(EnumAsString(ComputeGroupStatus, 100))
+    last_processed_at: Mapped[datetime] = mapped_column(
+        NaiveDateTime, default=get_current_datetime
+    )
+    deleted: Mapped[bool] = mapped_column(Boolean, default=False)
+    deleted_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
+
+    provisioning_data: Mapped[str] = mapped_column(Text)
+
+    first_termination_retry_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
+    last_termination_retry_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
+
+    instances: Mapped[List["InstanceModel"]] = relationship(back_populates="compute_group")
+
+
 class JobMetricsPoint(BaseModel):
     __tablename__ = "job_metrics_points"
 

dstack/_internal/server/routers/runs.py

@@ -1,6 +1,6 @@
-from typing import List, Tuple
+from typing import Annotated, List, Optional, Tuple, cast
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.errors import ResourceNotExistsError
@@ -35,6 +35,11 @@ project_router = APIRouter(
 )
 
 
+def use_legacy_default_working_dir(request: Request) -> bool:
+    client_release = cast(Optional[tuple[int, ...]], request.state.client_release)
+    return client_release is not None and client_release < (0, 19, 27)
+
+
 @root_router.post(
     "/list",
     response_model=List[Run],
@@ -103,8 +108,9 @@ async def get_run(
 )
 async def get_plan(
     body: GetRunPlanRequest,
-    session: AsyncSession = Depends(get_session),
-    user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
+    session: Annotated[AsyncSession, Depends(get_session)],
+    user_project: Annotated[tuple[UserModel, ProjectModel], Depends(ProjectMember())],
+    legacy_default_working_dir: Annotated[bool, Depends(use_legacy_default_working_dir)],
 ):
     """
     Returns a run plan for the given run spec.
@@ -119,6 +125,7 @@ async def get_plan(
         user=user,
         run_spec=body.run_spec,
         max_offers=body.max_offers,
+        legacy_default_working_dir=legacy_default_working_dir,
     )
     return CustomORJSONResponse(run_plan)
 
@@ -129,8 +136,9 @@ async def get_plan(
 )
 async def apply_plan(
     body: ApplyRunPlanRequest,
-    session: AsyncSession = Depends(get_session),
-    user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
+    session: Annotated[AsyncSession, Depends(get_session)],
+    user_project: Annotated[tuple[UserModel, ProjectModel], Depends(ProjectMember())],
+    legacy_default_working_dir: Annotated[bool, Depends(use_legacy_default_working_dir)],
 ):
     """
     Creates a new run or updates an existing run.
@@ -148,6 +156,7 @@ async def apply_plan(
             project=project,
             plan=body.plan,
             force=body.force,
+            legacy_default_working_dir=legacy_default_working_dir,
         )
     )