dstack 0.19.17__py3-none-any.whl → 0.19.19__py3-none-any.whl

dstack/_internal/server/background/tasks/process_gateways.py

@@ -16,6 +16,7 @@ from dstack._internal.server.services.gateways import (
     gateway_connections_pool,
 )
 from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker
+from dstack._internal.server.services.logging import fmt
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
 
@@ -27,14 +28,14 @@ async def process_gateways_connections():
     await _process_active_connections()
 
 
-async def process_submitted_gateways():
-    lock, lockset = get_locker().get_lockset(GatewayModel.__tablename__)
+async def process_gateways():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(GatewayModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
                 select(GatewayModel)
                 .where(
-                    GatewayModel.status == GatewayStatus.SUBMITTED,
+                    GatewayModel.status.in_([GatewayStatus.SUBMITTED, GatewayStatus.PROVISIONING]),
                     GatewayModel.id.not_in(lockset),
                 )
                 .options(lazyload(GatewayModel.gateway_compute))
@@ -48,7 +49,25 @@ async def process_submitted_gateways():
             lockset.add(gateway_model.id)
         try:
             gateway_model_id = gateway_model.id
-            await _process_submitted_gateway(session=session, gateway_model=gateway_model)
+            initial_status = gateway_model.status
+            if initial_status == GatewayStatus.SUBMITTED:
+                await _process_submitted_gateway(session=session, gateway_model=gateway_model)
+            elif initial_status == GatewayStatus.PROVISIONING:
+                await _process_provisioning_gateway(session=session, gateway_model=gateway_model)
+            else:
+                logger.error(
+                    "%s: unexpected gateway status %r", fmt(gateway_model), initial_status.upper()
+                )
+            if gateway_model.status != initial_status:
+                logger.info(
+                    "%s: gateway status has changed %s -> %s%s",
+                    fmt(gateway_model),
+                    initial_status.upper(),
+                    gateway_model.status.upper(),
+                    f": {gateway_model.status_message}" if gateway_model.status_message else "",
+                )
+            gateway_model.last_processed_at = get_current_datetime()
+            await session.commit()
         finally:
             lockset.difference_update([gateway_model_id])
 
@@ -89,7 +108,7 @@ async def _process_connection(conn: GatewayConnection):
 
 
 async def _process_submitted_gateway(session: AsyncSession, gateway_model: GatewayModel):
-    logger.info("Started gateway %s provisioning", gateway_model.name)
+    logger.info("%s: started gateway provisioning", fmt(gateway_model))
     # Refetch to load related attributes.
     # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
@@ -110,8 +129,6 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew
     except BackendNotAvailable:
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Backend not available"
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
         return
 
     try:
@@ -123,53 +140,54 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew
         )
         session.add(gateway_model)
         gateway_model.status = GatewayStatus.PROVISIONING
-        await session.commit()
-        await session.refresh(gateway_model)
     except BackendError as e:
-        logger.info(
-            "Failed to create gateway compute for gateway %s: %s", gateway_model.name, repr(e)
-        )
+        logger.info("%s: failed to create gateway compute: %r", fmt(gateway_model), e)
         gateway_model.status = GatewayStatus.FAILED
         status_message = f"Backend error: {repr(e)}"
         if len(e.args) > 0:
             status_message = str(e.args[0])
         gateway_model.status_message = status_message
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
-        return
     except Exception as e:
-        logger.exception(
-            "Got exception when creating gateway compute for gateway %s", gateway_model.name
-        )
+        logger.exception("%s: got exception when creating gateway compute", fmt(gateway_model))
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = f"Unexpected error: {repr(e)}"
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
-        return
 
+
+async def _process_provisioning_gateway(
+    session: AsyncSession, gateway_model: GatewayModel
+) -> None:
+    # Refetch to load related attributes.
+    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
+    res = await session.execute(
+        select(GatewayModel)
+        .where(GatewayModel.id == gateway_model.id)
+        .execution_options(populate_existing=True)
+    )
+    gateway_model = res.unique().scalar_one()
+
+    # FIXME: problems caused by blocking on connect_to_gateway_with_retry and configure_gateway:
+    # - cannot delete the gateway before it is provisioned because the DB model is locked
+    # - connection retry counter is reset on server restart
+    # - only one server replica is processing the gateway
+    # Easy to fix by doing only one connection/configuration attempt per processing iteration. The
+    # main challenge is applying the same provisioning model to the dstack Sky gateway to avoid
+    # maintaining a different model for Sky.
     connection = await gateways_services.connect_to_gateway_with_retry(
         gateway_model.gateway_compute
     )
     if connection is None:
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Failed to connect to gateway"
-        gateway_model.last_processed_at = get_current_datetime()
         gateway_model.gateway_compute.deleted = True
-        await session.commit()
         return
-
     try:
         await gateways_services.configure_gateway(connection)
     except Exception:
-        logger.exception("Failed to configure gateway %s", gateway_model.name)
+        logger.exception("%s: failed to configure gateway", fmt(gateway_model))
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Failed to configure gateway"
-        gateway_model.last_processed_at = get_current_datetime()
         await gateway_connections_pool.remove(gateway_model.gateway_compute.ip_address)
         gateway_model.gateway_compute.active = False
-        await session.commit()
         return
 
     gateway_model.status = GatewayStatus.RUNNING
-    gateway_model.last_processed_at = get_current_datetime()
-    await session.commit()

dstack/_internal/server/background/tasks/process_idle_volumes.py

@@ -0,0 +1,139 @@
+import datetime
+from typing import List
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
+
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
+from dstack._internal.core.errors import BackendNotAvailable
+from dstack._internal.core.models.profiles import parse_duration
+from dstack._internal.core.models.volumes import VolumeStatus
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import ProjectModel, VolumeModel
+from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.volumes import (
+    get_volume_configuration,
+    volume_model_to_volume,
+)
+from dstack._internal.utils import common
+from dstack._internal.utils.common import get_current_datetime
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def process_idle_volumes():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)
+    async with get_session_ctx() as session:
+        async with lock:
+            res = await session.execute(
+                select(VolumeModel.id)
+                .where(
+                    VolumeModel.status == VolumeStatus.ACTIVE,
+                    VolumeModel.deleted == False,
+                    VolumeModel.id.not_in(lockset),
+                )
+                .order_by(VolumeModel.last_processed_at.asc())
+                .limit(10)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            volume_ids = list(res.scalars().all())
+            if not volume_ids:
+                return
+            for volume_id in volume_ids:
+                lockset.add(volume_id)
+
+        res = await session.execute(
+            select(VolumeModel)
+            .where(VolumeModel.id.in_(volume_ids))
+            .options(joinedload(VolumeModel.project).joinedload(ProjectModel.backends))
+            .options(joinedload(VolumeModel.user))
+            .options(joinedload(VolumeModel.attachments))
+            .execution_options(populate_existing=True)
+        )
+        volume_models = list(res.unique().scalars().all())
+        try:
+            volumes_to_delete = [v for v in volume_models if _should_delete_volume(v)]
+            if not volumes_to_delete:
+                return
+            await _delete_idle_volumes(session, volumes_to_delete)
+        finally:
+            lockset.difference_update(volume_ids)
+
+
+def _should_delete_volume(volume: VolumeModel) -> bool:
+    if volume.attachments:
+        return False
+
+    config = get_volume_configuration(volume)
+    if not config.auto_cleanup_duration:
+        return False
+
+    duration_seconds = parse_duration(config.auto_cleanup_duration)
+    if not duration_seconds or duration_seconds <= 0:
+        return False
+
+    idle_time = _get_idle_time(volume)
+    threshold = datetime.timedelta(seconds=duration_seconds)
+    return idle_time > threshold
+
+
+def _get_idle_time(volume: VolumeModel) -> datetime.timedelta:
+    last_used = volume.last_job_processed_at or volume.created_at
+    last_used_utc = last_used.replace(tzinfo=datetime.timezone.utc)
+    idle_time = get_current_datetime() - last_used_utc
+    return max(idle_time, datetime.timedelta(0))
+
+
+async def _delete_idle_volumes(session: AsyncSession, volumes: List[VolumeModel]):
+    # Note: Multiple volumes are deleted in the same transaction,
+    # so long deletion of one volume may block processing other volumes.
+    for volume_model in volumes:
+        logger.info("Deleting idle volume %s", volume_model.name)
+        try:
+            await _delete_idle_volume(session, volume_model)
+        except Exception:
+            logger.exception("Error when deleting idle volume %s", volume_model.name)
+
+        volume_model.deleted = True
+        volume_model.deleted_at = get_current_datetime()
+
+        logger.info("Deleted idle volume %s", volume_model.name)
+
+    await session.commit()
+
+
+async def _delete_idle_volume(session: AsyncSession, volume_model: VolumeModel):
+    volume = volume_model_to_volume(volume_model)
+
+    if volume.provisioning_data is None:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. volume.provisioning_data is None."
+        )
+        return
+
+    if volume.provisioning_data.backend is None:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None."
+        )
+        return
+
+    try:
+        backend = await backends_services.get_project_backend_by_type_or_error(
+            project=volume_model.project,
+            backend_type=volume.provisioning_data.backend,
+        )
+    except BackendNotAvailable:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available."
+        )
+        return
+
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
+    await common.run_async(
+        compute.delete_volume,
+        volume=volume,
+    )

dstack/_internal/server/background/tasks/process_instances.py

@@ -45,6 +45,7 @@ from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT
 from dstack._internal.core.errors import (
     BackendError,
     NotYetTerminated,
+    PlacementGroupNotSupportedError,
     ProvisioningError,
 )
 from dstack._internal.core.models.backends.base import BackendType
@@ -73,7 +74,7 @@ from dstack._internal.core.models.runs import (
 from dstack._internal.core.services.profiles import get_retry
 from dstack._internal.server import settings as server_settings
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     FleetModel,
     InstanceModel,
@@ -110,6 +111,8 @@ from dstack._internal.utils.ssh import (
     pkey_from_str,
 )
 
+MIN_PROCESSING_INTERVAL = timedelta(seconds=10)
+
 PENDING_JOB_RETRY_INTERVAL = timedelta(seconds=60)
 
 TERMINATION_DEADLINE_OFFSET = timedelta(minutes=20)
@@ -129,7 +132,7 @@ async def process_instances(batch_size: int = 1):
 
 
 async def _process_next_instance():
-    lock, lockset = get_locker().get_lockset(InstanceModel.__tablename__)
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(InstanceModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
@@ -145,6 +148,8 @@ async def _process_next_instance():
                         ]
                     ),
                     InstanceModel.id.not_in(lockset),
+                    InstanceModel.last_processed_at
+                    < get_current_datetime().replace(tzinfo=None) - MIN_PROCESSING_INTERVAL,
                 )
                 .options(lazyload(InstanceModel.jobs))
                 .order_by(InstanceModel.last_processed_at.asc())
@@ -1063,6 +1068,12 @@ async def _create_placement_group(
             placement_group_model_to_placement_group(placement_group_model),
             master_instance_offer,
         )
+    except PlacementGroupNotSupportedError:
+        logger.debug(
+            "Skipping offer %s because placement group not supported",
+            master_instance_offer.instance.name,
+        )
+        return None
     except BackendError as e:
         logger.warning(
             "Failed to create placement group %s in %s/%s: %r",

dstack/_internal/server/background/tasks/process_placement_groups.py

@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload
 
 from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
 from dstack._internal.core.errors import PlacementGroupInUseError
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import PlacementGroupModel, ProjectModel
 from dstack._internal.server.services import backends as backends_services
 from dstack._internal.server.services.locking import get_locker
@@ -19,7 +19,9 @@ logger = get_logger(__name__)
 
 
 async def process_placement_groups():
-    lock, lockset = get_locker().get_lockset(PlacementGroupModel.__tablename__)
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(
+        PlacementGroupModel.__tablename__
+    )
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -34,10 +34,11 @@ from dstack._internal.core.models.runs import (
     JobTerminationReason,
     Run,
     RunSpec,
+    RunStatus,
 )
 from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, VolumeMountPoint
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     InstanceModel,
     JobModel,
@@ -79,6 +80,7 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+MIN_PROCESSING_INTERVAL = timedelta(seconds=10)
 # Minimum time before terminating active job in case of connectivity issues.
 # Should be sufficient to survive most problems caused by
 # the server network flickering and providers' glitches.
@@ -93,20 +95,29 @@ async def process_running_jobs(batch_size: int = 1):
 
 
 async def _process_next_running_job():
-    lock, lockset = get_locker().get_lockset(JobModel.__tablename__)
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(JobModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
                 select(JobModel)
+                .join(JobModel.run)
                 .where(
                     JobModel.status.in_(
                         [JobStatus.PROVISIONING, JobStatus.PULLING, JobStatus.RUNNING]
                     ),
+                    RunModel.status.not_in([RunStatus.TERMINATING]),
                     JobModel.id.not_in(lockset),
+                    JobModel.last_processed_at
+                    < common_utils.get_current_datetime().replace(tzinfo=None)
+                    - MIN_PROCESSING_INTERVAL,
                 )
                 .order_by(JobModel.last_processed_at.asc())
                 .limit(1)
-                .with_for_update(skip_locked=True, key_share=True)
+                .with_for_update(
+                    skip_locked=True,
+                    key_share=True,
+                    of=JobModel,
+                )
             )
             job_model = res.unique().scalar()
             if job_model is None:

dstack/_internal/server/background/tasks/process_runs.py

@@ -19,7 +19,7 @@ from dstack._internal.core.models.runs import (
     RunStatus,
     RunTerminationReason,
 )
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import JobModel, ProjectModel, RunModel
 from dstack._internal.server.services.jobs import (
     find_job,
@@ -41,6 +41,8 @@ from dstack._internal.utils import common
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
+
+MIN_PROCESSING_INTERVAL = datetime.timedelta(seconds=5)
 ROLLING_DEPLOYMENT_MAX_SURGE = 1  # at most one extra replica during rolling deployment
 
 
@@ -52,8 +54,8 @@ async def process_runs(batch_size: int = 1):
 
 
 async def _process_next_run():
-    run_lock, run_lockset = get_locker().get_lockset(RunModel.__tablename__)
-    job_lock, job_lockset = get_locker().get_lockset(JobModel.__tablename__)
+    run_lock, run_lockset = get_locker(get_db().dialect_name).get_lockset(RunModel.__tablename__)
+    job_lock, job_lockset = get_locker(get_db().dialect_name).get_lockset(JobModel.__tablename__)
     async with get_session_ctx() as session:
         async with run_lock, job_lock:
             res = await session.execute(
@@ -61,6 +63,8 @@ async def _process_next_run():
                 .where(
                     RunModel.status.not_in(RunStatus.finished_statuses()),
                     RunModel.id.not_in(run_lockset),
+                    RunModel.last_processed_at
+                    < common.get_current_datetime().replace(tzinfo=None) - MIN_PROCESSING_INTERVAL,
                 )
                 .order_by(RunModel.last_processed_at.asc())
                 .limit(1)
@@ -337,7 +341,7 @@ async def _process_active_run(session: AsyncSession, run_model: RunModel):
                 current_time - run_model.submitted_at.replace(tzinfo=datetime.timezone.utc)
             ).total_seconds()
             logger.info(
-                "%s: run took %.2f seconds from submision to provisioning.",
+                "%s: run took %.2f seconds from submission to provisioning.",
                 fmt(run_model),
                 submit_to_provision_duration,
             )

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -1,5 +1,6 @@
 import asyncio
 import uuid
+from datetime import datetime, timedelta
 from typing import List, Optional, Tuple
 
 from sqlalchemy import select
@@ -80,15 +81,35 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+# Track when we last processed a job.
+# This is needed for a trick:
+# If no tasks were processed recently, we force batch_size 1.
+# If there are lots of runs/jobs with same offers submitted,
+# we warm up the cache instead of requesting the offers concurrently.
+# Mostly useful when runs are submitted via API without getting run plan first.
+BATCH_SIZE_RESET_TIMEOUT = timedelta(minutes=2)
+last_processed_at: Optional[datetime] = None
+
+
 async def process_submitted_jobs(batch_size: int = 1):
     tasks = []
-    for _ in range(batch_size):
+    effective_batch_size = _get_effective_batch_size(batch_size)
+    for _ in range(effective_batch_size):
         tasks.append(_process_next_submitted_job())
     await asyncio.gather(*tasks)
 
 
+def _get_effective_batch_size(batch_size: int) -> int:
+    if (
+        last_processed_at is None
+        or last_processed_at < common_utils.get_current_datetime() - BATCH_SIZE_RESET_TIMEOUT
+    ):
+        return 1
+    return batch_size
+
+
 async def _process_next_submitted_job():
-    lock, lockset = get_locker().get_lockset(JobModel.__tablename__)
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(JobModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
@@ -125,6 +146,8 @@ async def _process_next_submitted_job():
             await _process_submitted_job(session=session, job_model=job_model)
         finally:
             lockset.difference_update([job_model_id])
+        global last_processed_at
+        last_processed_at = common_utils.get_current_datetime()
 
 
 async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
@@ -214,7 +237,9 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         if get_db().dialect_name == "sqlite":
             # Start new transaction to see committed changes after lock
             await session.commit()
-        async with get_locker().lock_ctx(InstanceModel.__tablename__, instances_ids):
+        async with get_locker(get_db().dialect_name).lock_ctx(
+            InstanceModel.__tablename__, instances_ids
+        ):
             # If another job freed the instance but is still trying to detach volumes,
             # do not provision on it to prevent attaching volumes that are currently detaching.
             detaching_instances_ids = await get_instances_ids_with_detaching_volumes(session)
@@ -243,8 +268,10 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             )
             job_model.instance_assigned = True
             job_model.last_processed_at = common_utils.get_current_datetime()
-            await session.commit()
-            return
+            if len(pool_instances) > 0:
+                await session.commit()
+                return
+            # If no instances were locked, we can proceed in the same transaction.
 
     if job_model.instance is not None:
         res = await session.execute(
@@ -334,7 +361,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         .order_by(VolumeModel.id)  # take locks in order
         .with_for_update(key_share=True)
     )
-    async with get_locker().lock_ctx(VolumeModel.__tablename__, volumes_ids):
+    async with get_locker(get_db().dialect_name).lock_ctx(VolumeModel.__tablename__, volumes_ids):
         if len(volume_models) > 0:
             await _attach_volumes(
                 session=session,
@@ -527,7 +554,9 @@ async def _get_next_instance_num(session: AsyncSession, fleet_model: FleetModel)
     if len(fleet_model.instances) == 0:
         # No instances means the fleet is not in the db yet, so don't lock.
         return 0
-    async with get_locker().lock_ctx(FleetModel.__tablename__, [fleet_model.id]):
+    async with get_locker(get_db().dialect_name).lock_ctx(
+        FleetModel.__tablename__, [fleet_model.id]
+    ):
         fleet_model = (
             (
                 await session.execute(
@@ -710,3 +739,5 @@ async def _attach_volume(
         attachment_data=attachment_data.json(),
     )
     instance.volume_attachments.append(volume_attachment_model)
+
+    volume_model.last_job_processed_at = common_utils.get_current_datetime()

dstack/_internal/server/background/tasks/process_terminating_jobs.py

@@ -5,7 +5,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, lazyload
 
 from dstack._internal.core.models.runs import JobStatus
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     InstanceModel,
     JobModel,
@@ -32,8 +32,10 @@ async def process_terminating_jobs(batch_size: int = 1):
 
 
 async def _process_next_terminating_job():
-    job_lock, job_lockset = get_locker().get_lockset(JobModel.__tablename__)
-    instance_lock, instance_lockset = get_locker().get_lockset(InstanceModel.__tablename__)
+    job_lock, job_lockset = get_locker(get_db().dialect_name).get_lockset(JobModel.__tablename__)
+    instance_lock, instance_lockset = get_locker(get_db().dialect_name).get_lockset(
+        InstanceModel.__tablename__
+    )
     async with get_session_ctx() as session:
         async with job_lock, instance_lock:
             res = await session.execute(

dstack/_internal/server/background/tasks/process_volumes.py

@@ -5,7 +5,7 @@ from sqlalchemy.orm import joinedload
 from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
 from dstack._internal.core.errors import BackendError, BackendNotAvailable
 from dstack._internal.core.models.volumes import VolumeStatus
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     InstanceModel,
     ProjectModel,
@@ -22,7 +22,7 @@ logger = get_logger(__name__)
 
 
 async def process_submitted_volumes():
-    lock, lockset = get_locker().get_lockset(VolumeModel.__tablename__)
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(

dstack/_internal/server/migrations/versions/35e90e1b0d3e_add_rolling_deployment_fields.py

@@ -17,12 +17,6 @@ depends_on = None
 
 
 def upgrade() -> None:
-    with op.batch_alter_table("jobs", schema=None) as batch_op:
-        batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
-    with op.batch_alter_table("jobs", schema=None) as batch_op:
-        batch_op.execute("UPDATE jobs SET deployment_num = 0")
-        batch_op.alter_column("deployment_num", nullable=False)
-
     with op.batch_alter_table("runs", schema=None) as batch_op:
         batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
         batch_op.add_column(sa.Column("desired_replica_count", sa.Integer(), nullable=True))
@@ -32,6 +26,12 @@ def upgrade() -> None:
         batch_op.alter_column("deployment_num", nullable=False)
         batch_op.alter_column("desired_replica_count", nullable=False)
 
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.execute("UPDATE jobs SET deployment_num = 0")
+        batch_op.alter_column("deployment_num", nullable=False)
+
 
 def downgrade() -> None:
     with op.batch_alter_table("runs", schema=None) as batch_op:

dstack/_internal/server/migrations/versions/d5863798bf41_add_volumemodel_last_job_processed_at.py

@@ -0,0 +1,40 @@
+"""Add VolumeModel.last_job_processed_at
+
+Revision ID: d5863798bf41
+Revises: 644b8a114187
+Create Date: 2025-07-15 14:26:22.981687
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+import dstack._internal.server.models
+
+# revision identifiers, used by Alembic.
+revision = "d5863798bf41"
+down_revision = "644b8a114187"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("volumes", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "last_job_processed_at",
+                dstack._internal.server.models.NaiveDateTime(),
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("volumes", schema=None) as batch_op:
+        batch_op.drop_column("last_job_processed_at")
+
+    # ### end Alembic commands ###