dstack 0.19.18__py3-none-any.whl → 0.19.20__py3-none-any.whl

dstack/_internal/core/models/volumes.py

@@ -9,6 +9,7 @@ from typing_extensions import Annotated, Self
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.profiles import parse_idle_duration
 from dstack._internal.core.models.resources import Memory
 from dstack._internal.utils.common import get_or_error
 from dstack._internal.utils.tags import tags_validator
@@ -44,6 +45,16 @@ class VolumeConfiguration(CoreModel):
         Optional[str],
         Field(description="The volume ID. Must be specified when registering external volumes"),
     ] = None
+    auto_cleanup_duration: Annotated[
+        Optional[Union[str, int]],
+        Field(
+            description=(
+                "Time to wait after volume is no longer used by any job before deleting it. "
+                "Defaults to keep the volume indefinitely. "
+                "Use the value 'off' or -1 to disable auto-cleanup."
+            )
+        ),
+    ] = None
     tags: Annotated[
         Optional[Dict[str, str]],
         Field(
@@ -56,6 +67,9 @@ class VolumeConfiguration(CoreModel):
     ] = None
 
     _validate_tags = validator("tags", pre=True, allow_reuse=True)(tags_validator)
+    _validate_auto_cleanup_duration = validator(
+        "auto_cleanup_duration", pre=True, allow_reuse=True
+    )(parse_idle_duration)
 
     @property
     def size_gb(self) -> int:

dstack/_internal/core/services/diff.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional, TypedDict
+from typing import Any, Optional, TypedDict, TypeVar
 
 from pydantic import BaseModel
 
@@ -15,20 +15,19 @@ ModelDiff = dict[str, ModelFieldDiff]
 
 # TODO: calculate nested diffs
 def diff_models(
-    old: BaseModel, new: BaseModel, ignore: Optional[IncludeExcludeType] = None
+    old: BaseModel, new: BaseModel, reset: Optional[IncludeExcludeType] = None
 ) -> ModelDiff:
     """
     Returns a diff of model instances fields.
 
-    NOTE: `ignore` is implemented as `BaseModel.parse_obj(BaseModel.dict(exclude=ignore))`,
-    that is, the "ignored" fields are actually not ignored but reset to the default values
-    before comparison, meaning that 1) any field in `ignore` must have a default value,
-    2) the default value must be equal to itself (e.g. `math.nan` != `math.nan`).
+    The fields specified in the `reset` option are reset to their default values, effectively
+    excluding them from comparison (assuming that the default value is equal to itself, e.g,
+    `None == None`, `"task" == "task"`, but `math.nan != math.nan`).
 
     Args:
         old: The "old" model instance.
         new: The "new" model instance.
-        ignore: Optional fields to ignore.
+        reset: Fields to reset to their default values before comparison.
 
     Returns:
         A dict of changed fields in the form of
@@ -37,9 +36,9 @@ def diff_models(
     if type(old) is not type(new):
         raise TypeError("Both instances must be of the same Pydantic model class.")
 
-    if ignore is not None:
-        old = type(old).parse_obj(old.dict(exclude=ignore))
-        new = type(new).parse_obj(new.dict(exclude=ignore))
+    if reset is not None:
+        old = copy_model(old, reset=reset)
+        new = copy_model(new, reset=reset)
 
     changes: ModelDiff = {}
     for field in old.__fields__:
@@ -49,3 +48,24 @@ def diff_models(
             changes[field] = {"old": old_value, "new": new_value}
 
     return changes
+
+
+M = TypeVar("M", bound=BaseModel)
+
+
+def copy_model(model: M, reset: Optional[IncludeExcludeType] = None) -> M:
+    """
+    Returns a deep copy of the model instance.
+
+    Implemented as `BaseModel.parse_obj(BaseModel.dict())`, thus,
+    unlike `BaseModel.copy(deep=True)`, runs all validations.
+
+    The fields specified in the `reset` option are reset to their default values.
+
+    Args:
+        reset: Fields to reset to their default values.
+
+    Returns:
+        A deep copy of the model instance.
+    """
+    return type(model).parse_obj(model.dict(exclude=reset))

dstack/_internal/core/services/ssh/attach.py

@@ -64,6 +64,7 @@ class SSHAttach:
         run_name: str,
         dockerized: bool,
         ssh_proxy: Optional[SSHConnectionParams] = None,
+        service_port: Optional[int] = None,
         local_backend: bool = False,
         bind_address: Optional[str] = None,
     ):
@@ -90,6 +91,7 @@ class SSHAttach:
             },
         )
         self.ssh_proxy = ssh_proxy
+        self.service_port = service_port
 
         hosts: dict[str, dict[str, Union[str, int, FilePath]]] = {}
         self.hosts = hosts

dstack/_internal/server/app.py

@@ -10,7 +10,7 @@ from typing import Awaitable, Callable, List
 import sentry_sdk
 from fastapi import FastAPI, Request, Response, status
 from fastapi.datastructures import URL
-from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import Counter, Histogram
 
@@ -56,6 +56,7 @@ from dstack._internal.server.settings import (
 )
 from dstack._internal.server.utils.logging import configure_logging
 from dstack._internal.server.utils.routers import (
+    CustomORJSONResponse,
     check_client_server_compatibility,
     error_detail,
     get_server_client_error_details,
@@ -90,7 +91,10 @@ def create_app() -> FastAPI:
             profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
         )
 
-    app = FastAPI(docs_url="/api/docs", lifespan=lifespan)
+    app = FastAPI(
+        docs_url="/api/docs",
+        lifespan=lifespan,
+    )
     app.state.proxy_dependency_injector = ServerProxyDependencyInjector()
     return app
 
@@ -147,7 +151,10 @@ async def lifespan(app: FastAPI):
     )
     if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None:
         init_default_storage()
-    scheduler = start_background_tasks()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler = start_background_tasks()
+    else:
+        logger.info("Background processing is disabled")
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
@@ -157,7 +164,8 @@ async def lifespan(app: FastAPI):
     for func in _ON_STARTUP_HOOKS:
         await func(app)
     yield
-    scheduler.shutdown()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler.shutdown()
     await gateway_connections_pool.remove_all()
     service_conn_pool = await get_injector_from_app(app).get_service_connection_pool()
     await service_conn_pool.remove_all()
@@ -208,14 +216,14 @@ def register_routes(app: FastAPI, ui: bool = True):
         msg = "Access denied"
         if len(exc.args) > 0:
             msg = exc.args[0]
-        return JSONResponse(
+        return CustomORJSONResponse(
             status_code=status.HTTP_403_FORBIDDEN,
             content=error_detail(msg),
         )
 
     @app.exception_handler(ServerClientError)
     async def server_client_error_handler(request: Request, exc: ServerClientError):
-        return JSONResponse(
+        return CustomORJSONResponse(
             status_code=status.HTTP_400_BAD_REQUEST,
             content={"detail": get_server_client_error_details(exc)},
         )
@@ -223,7 +231,7 @@ def register_routes(app: FastAPI, ui: bool = True):
     @app.exception_handler(OSError)
     async def os_error_handler(request, exc: OSError):
         if exc.errno in [36, 63]:
-            return JSONResponse(
+            return CustomORJSONResponse(
                 {"detail": "Filename too long"},
                 status_code=status.HTTP_400_BAD_REQUEST,
             )
@@ -309,7 +317,7 @@ def register_routes(app: FastAPI, ui: bool = True):
 
     @app.get("/healthcheck")
     async def healthcheck():
-        return JSONResponse(content={"status": "running"})
+        return CustomORJSONResponse(content={"status": "running"})
 
     if ui and Path(__file__).parent.joinpath("statics").exists():
         app.mount(
@@ -323,7 +331,7 @@ def register_routes(app: FastAPI, ui: bool = True):
                 or _is_proxy_request(request)
                 or _is_prometheus_request(request)
             ):
-                return JSONResponse(
+                return CustomORJSONResponse(
                     {"detail": exc.detail},
                     status_code=status.HTTP_404_NOT_FOUND,
                 )

dstack/_internal/server/background/__init__.py

@@ -4,9 +4,10 @@ from apscheduler.triggers.interval import IntervalTrigger
 from dstack._internal.server import settings
 from dstack._internal.server.background.tasks.process_fleets import process_fleets
 from dstack._internal.server.background.tasks.process_gateways import (
+    process_gateways,
     process_gateways_connections,
-    process_submitted_gateways,
 )
+from dstack._internal.server.background.tasks.process_idle_volumes import process_idle_volumes
 from dstack._internal.server.background.tasks.process_instances import (
     process_instances,
 )
@@ -70,11 +71,12 @@ def start_background_tasks() -> AsyncIOScheduler:
         )
         _scheduler.add_job(delete_prometheus_metrics, IntervalTrigger(minutes=5), max_instances=1)
     _scheduler.add_job(process_gateways_connections, IntervalTrigger(seconds=15))
+    _scheduler.add_job(process_gateways, IntervalTrigger(seconds=10, jitter=2), max_instances=5)
     _scheduler.add_job(
-        process_submitted_gateways, IntervalTrigger(seconds=10, jitter=2), max_instances=5
+        process_submitted_volumes, IntervalTrigger(seconds=10, jitter=2), max_instances=5
     )
     _scheduler.add_job(
-        process_submitted_volumes, IntervalTrigger(seconds=10, jitter=2), max_instances=5
+        process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1
     )
     _scheduler.add_job(process_placement_groups, IntervalTrigger(seconds=30, jitter=5))
     for replica in range(settings.SERVER_BACKGROUND_PROCESSING_FACTOR):

dstack/_internal/server/background/tasks/process_gateways.py

@@ -16,6 +16,7 @@ from dstack._internal.server.services.gateways import (
     gateway_connections_pool,
 )
 from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker
+from dstack._internal.server.services.logging import fmt
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
 
@@ -27,14 +28,14 @@ async def process_gateways_connections():
     await _process_active_connections()
 
 
-async def process_submitted_gateways():
+async def process_gateways():
     lock, lockset = get_locker(get_db().dialect_name).get_lockset(GatewayModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
                 select(GatewayModel)
                 .where(
-                    GatewayModel.status == GatewayStatus.SUBMITTED,
+                    GatewayModel.status.in_([GatewayStatus.SUBMITTED, GatewayStatus.PROVISIONING]),
                     GatewayModel.id.not_in(lockset),
                 )
                 .options(lazyload(GatewayModel.gateway_compute))
@@ -48,7 +49,25 @@ async def process_submitted_gateways():
             lockset.add(gateway_model.id)
         try:
             gateway_model_id = gateway_model.id
-            await _process_submitted_gateway(session=session, gateway_model=gateway_model)
+            initial_status = gateway_model.status
+            if initial_status == GatewayStatus.SUBMITTED:
+                await _process_submitted_gateway(session=session, gateway_model=gateway_model)
+            elif initial_status == GatewayStatus.PROVISIONING:
+                await _process_provisioning_gateway(session=session, gateway_model=gateway_model)
+            else:
+                logger.error(
+                    "%s: unexpected gateway status %r", fmt(gateway_model), initial_status.upper()
+                )
+            if gateway_model.status != initial_status:
+                logger.info(
+                    "%s: gateway status has changed %s -> %s%s",
+                    fmt(gateway_model),
+                    initial_status.upper(),
+                    gateway_model.status.upper(),
+                    f": {gateway_model.status_message}" if gateway_model.status_message else "",
+                )
+            gateway_model.last_processed_at = get_current_datetime()
+            await session.commit()
         finally:
             lockset.difference_update([gateway_model_id])
 
@@ -89,7 +108,7 @@ async def _process_connection(conn: GatewayConnection):
 
 
 async def _process_submitted_gateway(session: AsyncSession, gateway_model: GatewayModel):
-    logger.info("Started gateway %s provisioning", gateway_model.name)
+    logger.info("%s: started gateway provisioning", fmt(gateway_model))
     # Refetch to load related attributes.
     # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
@@ -110,8 +129,6 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew
     except BackendNotAvailable:
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Backend not available"
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
         return
 
     try:
@@ -123,53 +140,54 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew
         )
         session.add(gateway_model)
         gateway_model.status = GatewayStatus.PROVISIONING
-        await session.commit()
-        await session.refresh(gateway_model)
     except BackendError as e:
-        logger.info(
-            "Failed to create gateway compute for gateway %s: %s", gateway_model.name, repr(e)
-        )
+        logger.info("%s: failed to create gateway compute: %r", fmt(gateway_model), e)
         gateway_model.status = GatewayStatus.FAILED
         status_message = f"Backend error: {repr(e)}"
         if len(e.args) > 0:
             status_message = str(e.args[0])
         gateway_model.status_message = status_message
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
-        return
     except Exception as e:
-        logger.exception(
-            "Got exception when creating gateway compute for gateway %s", gateway_model.name
-        )
+        logger.exception("%s: got exception when creating gateway compute", fmt(gateway_model))
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = f"Unexpected error: {repr(e)}"
-        gateway_model.last_processed_at = get_current_datetime()
-        await session.commit()
-        return
 
+
+async def _process_provisioning_gateway(
+    session: AsyncSession, gateway_model: GatewayModel
+) -> None:
+    # Refetch to load related attributes.
+    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
+    res = await session.execute(
+        select(GatewayModel)
+        .where(GatewayModel.id == gateway_model.id)
+        .execution_options(populate_existing=True)
+    )
+    gateway_model = res.unique().scalar_one()
+
+    # FIXME: problems caused by blocking on connect_to_gateway_with_retry and configure_gateway:
+    # - cannot delete the gateway before it is provisioned because the DB model is locked
+    # - connection retry counter is reset on server restart
+    # - only one server replica is processing the gateway
+    # Easy to fix by doing only one connection/configuration attempt per processing iteration. The
+    # main challenge is applying the same provisioning model to the dstack Sky gateway to avoid
+    # maintaining a different model for Sky.
     connection = await gateways_services.connect_to_gateway_with_retry(
         gateway_model.gateway_compute
     )
     if connection is None:
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Failed to connect to gateway"
-        gateway_model.last_processed_at = get_current_datetime()
         gateway_model.gateway_compute.deleted = True
-        await session.commit()
         return
-
     try:
         await gateways_services.configure_gateway(connection)
     except Exception:
-        logger.exception("Failed to configure gateway %s", gateway_model.name)
+        logger.exception("%s: failed to configure gateway", fmt(gateway_model))
         gateway_model.status = GatewayStatus.FAILED
         gateway_model.status_message = "Failed to configure gateway"
-        gateway_model.last_processed_at = get_current_datetime()
         await gateway_connections_pool.remove(gateway_model.gateway_compute.ip_address)
         gateway_model.gateway_compute.active = False
-        await session.commit()
         return
 
     gateway_model.status = GatewayStatus.RUNNING
-    gateway_model.last_processed_at = get_current_datetime()
-    await session.commit()

dstack/_internal/server/background/tasks/process_idle_volumes.py

@@ -0,0 +1,139 @@
+import datetime
+from typing import List
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
+
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
+from dstack._internal.core.errors import BackendNotAvailable
+from dstack._internal.core.models.profiles import parse_duration
+from dstack._internal.core.models.volumes import VolumeStatus
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import ProjectModel, VolumeModel
+from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.volumes import (
+    get_volume_configuration,
+    volume_model_to_volume,
+)
+from dstack._internal.utils import common
+from dstack._internal.utils.common import get_current_datetime
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def process_idle_volumes():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)
+    async with get_session_ctx() as session:
+        async with lock:
+            res = await session.execute(
+                select(VolumeModel.id)
+                .where(
+                    VolumeModel.status == VolumeStatus.ACTIVE,
+                    VolumeModel.deleted == False,
+                    VolumeModel.id.not_in(lockset),
+                )
+                .order_by(VolumeModel.last_processed_at.asc())
+                .limit(10)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            volume_ids = list(res.scalars().all())
+            if not volume_ids:
+                return
+            for volume_id in volume_ids:
+                lockset.add(volume_id)
+
+        res = await session.execute(
+            select(VolumeModel)
+            .where(VolumeModel.id.in_(volume_ids))
+            .options(joinedload(VolumeModel.project).joinedload(ProjectModel.backends))
+            .options(joinedload(VolumeModel.user))
+            .options(joinedload(VolumeModel.attachments))
+            .execution_options(populate_existing=True)
+        )
+        volume_models = list(res.unique().scalars().all())
+        try:
+            volumes_to_delete = [v for v in volume_models if _should_delete_volume(v)]
+            if not volumes_to_delete:
+                return
+            await _delete_idle_volumes(session, volumes_to_delete)
+        finally:
+            lockset.difference_update(volume_ids)
+
+
+def _should_delete_volume(volume: VolumeModel) -> bool:
+    if volume.attachments:
+        return False
+
+    config = get_volume_configuration(volume)
+    if not config.auto_cleanup_duration:
+        return False
+
+    duration_seconds = parse_duration(config.auto_cleanup_duration)
+    if not duration_seconds or duration_seconds <= 0:
+        return False
+
+    idle_time = _get_idle_time(volume)
+    threshold = datetime.timedelta(seconds=duration_seconds)
+    return idle_time > threshold
+
+
+def _get_idle_time(volume: VolumeModel) -> datetime.timedelta:
+    last_used = volume.last_job_processed_at or volume.created_at
+    last_used_utc = last_used.replace(tzinfo=datetime.timezone.utc)
+    idle_time = get_current_datetime() - last_used_utc
+    return max(idle_time, datetime.timedelta(0))
+
+
+async def _delete_idle_volumes(session: AsyncSession, volumes: List[VolumeModel]):
+    # Note: Multiple volumes are deleted in the same transaction,
+    # so long deletion of one volume may block processing other volumes.
+    for volume_model in volumes:
+        logger.info("Deleting idle volume %s", volume_model.name)
+        try:
+            await _delete_idle_volume(session, volume_model)
+        except Exception:
+            logger.exception("Error when deleting idle volume %s", volume_model.name)
+
+        volume_model.deleted = True
+        volume_model.deleted_at = get_current_datetime()
+
+        logger.info("Deleted idle volume %s", volume_model.name)
+
+    await session.commit()
+
+
+async def _delete_idle_volume(session: AsyncSession, volume_model: VolumeModel):
+    volume = volume_model_to_volume(volume_model)
+
+    if volume.provisioning_data is None:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. volume.provisioning_data is None."
+        )
+        return
+
+    if volume.provisioning_data.backend is None:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None."
+        )
+        return
+
+    try:
+        backend = await backends_services.get_project_backend_by_type_or_error(
+            project=volume_model.project,
+            backend_type=volume.provisioning_data.backend,
+        )
+    except BackendNotAvailable:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available."
+        )
+        return
+
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
+    await common.run_async(
+        compute.delete_volume,
+        volume=volume,
+    )

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -739,3 +739,5 @@ async def _attach_volume(
         attachment_data=attachment_data.json(),
     )
     instance.volume_attachments.append(volume_attachment_model)
+
+    volume_model.last_job_processed_at = common_utils.get_current_datetime()

dstack/_internal/server/migrations/versions/35e90e1b0d3e_add_rolling_deployment_fields.py

@@ -17,12 +17,6 @@ depends_on = None
 
 
 def upgrade() -> None:
-    with op.batch_alter_table("jobs", schema=None) as batch_op:
-        batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
-    with op.batch_alter_table("jobs", schema=None) as batch_op:
-        batch_op.execute("UPDATE jobs SET deployment_num = 0")
-        batch_op.alter_column("deployment_num", nullable=False)
-
     with op.batch_alter_table("runs", schema=None) as batch_op:
         batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
         batch_op.add_column(sa.Column("desired_replica_count", sa.Integer(), nullable=True))
@@ -32,6 +26,12 @@ def upgrade() -> None:
         batch_op.alter_column("deployment_num", nullable=False)
         batch_op.alter_column("desired_replica_count", nullable=False)
 
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("deployment_num", sa.Integer(), nullable=True))
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.execute("UPDATE jobs SET deployment_num = 0")
+        batch_op.alter_column("deployment_num", nullable=False)
+
 
 def downgrade() -> None:
     with op.batch_alter_table("runs", schema=None) as batch_op:

dstack/_internal/server/migrations/versions/d5863798bf41_add_volumemodel_last_job_processed_at.py

@@ -0,0 +1,40 @@
+"""Add VolumeModel.last_job_processed_at
+
+Revision ID: d5863798bf41
+Revises: 644b8a114187
+Create Date: 2025-07-15 14:26:22.981687
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+import dstack._internal.server.models
+
+# revision identifiers, used by Alembic.
+revision = "d5863798bf41"
+down_revision = "644b8a114187"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("volumes", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "last_job_processed_at",
+                dstack._internal.server.models.NaiveDateTime(),
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("volumes", schema=None) as batch_op:
+        batch_op.drop_column("last_job_processed_at")
+
+    # ### end Alembic commands ###

dstack/_internal/server/models.py

@@ -645,6 +645,7 @@ class VolumeModel(BaseModel):
     last_processed_at: Mapped[datetime] = mapped_column(
         NaiveDateTime, default=get_current_datetime
     )
+    last_job_processed_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
     deleted: Mapped[bool] = mapped_column(Boolean, default=False)
     deleted_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)