dstack 0.19.17__py3-none-any.whl → 0.19.19__py3-none-any.whl

dstack/_internal/core/models/runs.py

@@ -11,6 +11,7 @@ from dstack._internal.core.models.configurations import (
     DEFAULT_REPO_DIR,
     AnyRunConfiguration,
     RunConfiguration,
+    ServiceConfiguration,
 )
 from dstack._internal.core.models.files import FileArchiveMapping
 from dstack._internal.core.models.instances import (
@@ -101,6 +102,14 @@ class RunTerminationReason(str, Enum):
         }
         return mapping[self]
 
+    def to_error(self) -> Optional[str]:
+        if self == RunTerminationReason.RETRY_LIMIT_EXCEEDED:
+            return "retry limit exceeded"
+        elif self == RunTerminationReason.SERVER_ERROR:
+            return "server error"
+        else:
+            return None
+
 
 class JobTerminationReason(str, Enum):
     # Set by the server
@@ -162,6 +171,24 @@ class JobTerminationReason(str, Enum):
         default = RetryEvent.ERROR if self.to_status() == JobStatus.FAILED else None
         return mapping.get(self, default)
 
+    def to_error(self) -> Optional[str]:
+        # Should return None for values that are already
+        # handled and shown in status_message.
+        error_mapping = {
+            JobTerminationReason.INSTANCE_UNREACHABLE: "instance unreachable",
+            JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED: "waiting instance limit exceeded",
+            JobTerminationReason.VOLUME_ERROR: "volume error",
+            JobTerminationReason.GATEWAY_ERROR: "gateway error",
+            JobTerminationReason.SCALED_DOWN: "scaled down",
+            JobTerminationReason.INACTIVITY_DURATION_EXCEEDED: "inactivity duration exceeded",
+            JobTerminationReason.TERMINATED_DUE_TO_UTILIZATION_POLICY: "utilization policy",
+            JobTerminationReason.PORTS_BINDING_FAILED: "ports binding failed",
+            JobTerminationReason.CREATING_CONTAINER_ERROR: "runner error",
+            JobTerminationReason.EXECUTOR_ERROR: "executor error",
+            JobTerminationReason.MAX_DURATION_EXCEEDED: "max duration exceeded",
+        }
+        return error_mapping.get(self)
+
 
 class Requirements(CoreModel):
     # TODO: Make requirements' fields required
@@ -227,6 +254,8 @@ class JobSpec(CoreModel):
     # TODO: drop this comment when supporting jobs submitted before 0.19.17 is no longer relevant.
     repo_code_hash: Optional[str] = None
     file_archives: list[FileArchiveMapping] = []
+    # None for non-services and pre-0.19.19 services. See `get_service_port`
+    service_port: Optional[int] = None
 
 
 class JobProvisioningData(CoreModel):
@@ -305,13 +334,12 @@ class JobSubmission(CoreModel):
     finished_at: Optional[datetime]
     inactivity_secs: Optional[int]
     status: JobStatus
+    status_message: str = ""  # default for backward compatibility
     termination_reason: Optional[JobTerminationReason]
     termination_reason_message: Optional[str]
     exit_status: Optional[int]
     job_provisioning_data: Optional[JobProvisioningData]
     job_runtime_data: Optional[JobRuntimeData]
-    # TODO: make status_message and error a computed field after migrating to pydanticV2
-    status_message: Optional[str] = None
     error: Optional[str] = None
 
     @property
@@ -325,71 +353,6 @@ class JobSubmission(CoreModel):
             end_time = self.finished_at
         return end_time - self.submitted_at
 
-    @root_validator
-    def _status_message(cls, values) -> Dict:
-        try:
-            status = values["status"]
-            termination_reason = values["termination_reason"]
-            exit_code = values["exit_status"]
-        except KeyError:
-            return values
-        values["status_message"] = JobSubmission._get_status_message(
-            status=status,
-            termination_reason=termination_reason,
-            exit_status=exit_code,
-        )
-        return values
-
-    @staticmethod
-    def _get_status_message(
-        status: JobStatus,
-        termination_reason: Optional[JobTerminationReason],
-        exit_status: Optional[int],
-    ) -> str:
-        if status == JobStatus.DONE:
-            return "exited (0)"
-        elif status == JobStatus.FAILED:
-            if termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR:
-                return f"exited ({exit_status})"
-            elif termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY:
-                return "no offers"
-            elif termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
-                return "interrupted"
-            else:
-                return "error"
-        elif status == JobStatus.TERMINATED:
-            if termination_reason == JobTerminationReason.TERMINATED_BY_USER:
-                return "stopped"
-            elif termination_reason == JobTerminationReason.ABORTED_BY_USER:
-                return "aborted"
-        return status.value
-
-    @root_validator
-    def _error(cls, values) -> Dict:
-        try:
-            termination_reason = values["termination_reason"]
-        except KeyError:
-            return values
-        values["error"] = JobSubmission._get_error(termination_reason=termination_reason)
-        return values
-
-    @staticmethod
-    def _get_error(termination_reason: Optional[JobTerminationReason]) -> Optional[str]:
-        error_mapping = {
-            JobTerminationReason.INSTANCE_UNREACHABLE: "instance unreachable",
-            JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED: "waiting instance limit exceeded",
-            JobTerminationReason.VOLUME_ERROR: "volume error",
-            JobTerminationReason.GATEWAY_ERROR: "gateway error",
-            JobTerminationReason.SCALED_DOWN: "scaled down",
-            JobTerminationReason.INACTIVITY_DURATION_EXCEEDED: "inactivity duration exceeded",
-            JobTerminationReason.TERMINATED_DUE_TO_UTILIZATION_POLICY: "utilization policy",
-            JobTerminationReason.PORTS_BINDING_FAILED: "ports binding failed",
-            JobTerminationReason.CREATING_CONTAINER_ERROR: "runner error",
-            JobTerminationReason.EXECUTOR_ERROR: "executor error",
-            JobTerminationReason.MAX_DURATION_EXCEEDED: "max duration exceeded",
-        }
-        return error_mapping.get(termination_reason)
-
 
 class Job(CoreModel):
     job_spec: JobSpec
@@ -524,85 +487,17 @@ class Run(CoreModel):
     submitted_at: datetime
     last_processed_at: datetime
     status: RunStatus
-    status_message: Optional[str] = None
-    termination_reason: Optional[RunTerminationReason]
+    status_message: str = ""  # default for backward compatibility
+    termination_reason: Optional[RunTerminationReason] = None
     run_spec: RunSpec
     jobs: List[Job]
-    latest_job_submission: Optional[JobSubmission]
+    latest_job_submission: Optional[JobSubmission] = None
     cost: float = 0
     service: Optional[ServiceSpec] = None
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
-    # TODO: make error a computed field after migrating to pydanticV2
     error: Optional[str] = None
     deleted: Optional[bool] = None
 
-    @root_validator
-    def _error(cls, values) -> Dict:
-        try:
-            termination_reason = values["termination_reason"]
-        except KeyError:
-            return values
-        values["error"] = Run._get_error(termination_reason=termination_reason)
-        return values
-
-    @staticmethod
-    def _get_error(termination_reason: Optional[RunTerminationReason]) -> Optional[str]:
-        if termination_reason == RunTerminationReason.RETRY_LIMIT_EXCEEDED:
-            return "retry limit exceeded"
-        elif termination_reason == RunTerminationReason.SERVER_ERROR:
-            return "server error"
-        else:
-            return None
-
-    @root_validator
-    def _status_message(cls, values) -> Dict:
-        try:
-            status = values["status"]
-            jobs: List[Job] = values["jobs"]
-            retry_on_events = (
-                jobs[0].job_spec.retry.on_events if jobs and jobs[0].job_spec.retry else []
-            )
-            job_status = (
-                jobs[0].job_submissions[-1].status
-                if len(jobs) == 1 and jobs[0].job_submissions
-                else None
-            )
-            termination_reason = Run.get_last_termination_reason(jobs[0]) if jobs else None
-        except KeyError:
-            return values
-        values["status_message"] = Run._get_status_message(
-            status=status,
-            job_status=job_status,
-            retry_on_events=retry_on_events,
-            termination_reason=termination_reason,
-        )
-        return values
-
-    @staticmethod
-    def get_last_termination_reason(job: "Job") -> Optional[JobTerminationReason]:
-        for submission in reversed(job.job_submissions):
-            if submission.termination_reason is not None:
-                return submission.termination_reason
-        return None
-
-    @staticmethod
-    def _get_status_message(
-        status: RunStatus,
-        job_status: Optional[JobStatus],
-        retry_on_events: List[RetryEvent],
-        termination_reason: Optional[JobTerminationReason],
-    ) -> str:
-        if job_status == JobStatus.PULLING:
-            return "pulling"
-        # Currently, `retrying` is shown only for `no-capacity` events
-        if (
-            status in [RunStatus.SUBMITTED, RunStatus.PENDING]
-            and termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
-            and RetryEvent.NO_CAPACITY in retry_on_events
-        ):
-            return "retrying"
-        return status.value
-
     def is_deployment_in_progress(self) -> bool:
         return any(
             not j.job_submissions[-1].status.is_finished()
@@ -658,3 +553,11 @@ def get_policy_map(spot_policy: Optional[SpotPolicy], default: SpotPolicy) -> Op
         SpotPolicy.ONDEMAND: False,
     }
     return policy_map[spot_policy]
+
+
+def get_service_port(job_spec: JobSpec, configuration: ServiceConfiguration) -> int:
+    # Compatibility with pre-0.19.19 job specs that do not have the `service_port` property.
+    # TODO: drop when pre-0.19.19 jobs are no longer relevant.
+    if job_spec.service_port is None:
+        return configuration.port.container_port
+    return job_spec.service_port

dstack/_internal/core/models/volumes.py

@@ -9,6 +9,7 @@ from typing_extensions import Annotated, Self
 
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.profiles import parse_idle_duration
 from dstack._internal.core.models.resources import Memory
 from dstack._internal.utils.common import get_or_error
 from dstack._internal.utils.tags import tags_validator
@@ -44,6 +45,16 @@ class VolumeConfiguration(CoreModel):
         Optional[str],
         Field(description="The volume ID. Must be specified when registering external volumes"),
     ] = None
+    auto_cleanup_duration: Annotated[
+        Optional[Union[str, int]],
+        Field(
+            description=(
+                "Time to wait after volume is no longer used by any job before deleting it. "
+                "Defaults to keep the volume indefinitely. "
+                "Use the value 'off' or -1 to disable auto-cleanup."
+            )
+        ),
+    ] = None
     tags: Annotated[
         Optional[Dict[str, str]],
         Field(
@@ -56,6 +67,9 @@ class VolumeConfiguration(CoreModel):
     ] = None
 
     _validate_tags = validator("tags", pre=True, allow_reuse=True)(tags_validator)
+    _validate_auto_cleanup_duration = validator(
+        "auto_cleanup_duration", pre=True, allow_reuse=True
+    )(parse_idle_duration)
 
     @property
     def size_gb(self) -> int:

dstack/_internal/core/services/diff.py

@@ -1,14 +1,46 @@
-from typing import Any, Dict
+from typing import Any, Optional, TypedDict, TypeVar
 
 from pydantic import BaseModel
 
+from dstack._internal.core.models.common import IncludeExcludeType
+
+
+class ModelFieldDiff(TypedDict):
+    old: Any
+    new: Any
+
+
+ModelDiff = dict[str, ModelFieldDiff]
+
 
 # TODO: calculate nested diffs
-def diff_models(old: BaseModel, new: BaseModel) -> Dict[str, Any]:
+def diff_models(
+    old: BaseModel, new: BaseModel, reset: Optional[IncludeExcludeType] = None
+) -> ModelDiff:
+    """
+    Returns a diff of model instances fields.
+
+    The fields specified in the `reset` option are reset to their default values, effectively
+    excluding them from comparison (assuming that the default value is equal to itself, e.g,
+    `None == None`, `"task" == "task"`, but `math.nan != math.nan`).
+
+    Args:
+        old: The "old" model instance.
+        new: The "new" model instance.
+        reset: Fields to reset to their default values before comparison.
+
+    Returns:
+        A dict of changed fields in the form of
+        `{<field_name>: {"old": old_value, "new": new_value}}`
+    """
     if type(old) is not type(new):
         raise TypeError("Both instances must be of the same Pydantic model class.")
 
-    changes = {}
+    if reset is not None:
+        old = copy_model(old, reset=reset)
+        new = copy_model(new, reset=reset)
+
+    changes: ModelDiff = {}
     for field in old.__fields__:
         old_value = getattr(old, field)
         new_value = getattr(new, field)
@@ -16,3 +48,24 @@ def diff_models(old: BaseModel, new: BaseModel) -> Dict[str, Any]:
             changes[field] = {"old": old_value, "new": new_value}
 
     return changes
+
+
+M = TypeVar("M", bound=BaseModel)
+
+
+def copy_model(model: M, reset: Optional[IncludeExcludeType] = None) -> M:
+    """
+    Returns a deep copy of the model instance.
+
+    Implemented as `BaseModel.parse_obj(BaseModel.dict())`, thus,
+    unlike `BaseModel.copy(deep=True)`, runs all validations.
+
+    The fields specified in the `reset` option are reset to their default values.
+
+    Args:
+        reset: Fields to reset to their default values.
+
+    Returns:
+        A deep copy of the model instance.
+    """
+    return type(model).parse_obj(model.dict(exclude=reset))

dstack/_internal/core/services/ssh/attach.py

@@ -64,6 +64,7 @@ class SSHAttach:
         run_name: str,
         dockerized: bool,
         ssh_proxy: Optional[SSHConnectionParams] = None,
+        service_port: Optional[int] = None,
         local_backend: bool = False,
         bind_address: Optional[str] = None,
     ):
@@ -90,6 +91,7 @@ class SSHAttach:
             },
         )
         self.ssh_proxy = ssh_proxy
+        self.service_port = service_port
 
         hosts: dict[str, dict[str, Union[str, int, FilePath]]] = {}
         self.hosts = hosts

dstack/_internal/server/app.py

@@ -2,6 +2,7 @@ import asyncio
 import importlib.resources
 import os
 import time
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import asynccontextmanager
 from pathlib import Path
 from typing import Awaitable, Callable, List
@@ -9,7 +10,7 @@ from typing import Awaitable, Callable, List
 import sentry_sdk
 from fastapi import FastAPI, Request, Response, status
 from fastapi.datastructures import URL
-from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import Counter, Histogram
 
@@ -55,6 +56,7 @@ from dstack._internal.server.settings import (
 )
 from dstack._internal.server.utils.logging import configure_logging
 from dstack._internal.server.utils.routers import (
+    CustomORJSONResponse,
     check_client_server_compatibility,
     error_detail,
     get_server_client_error_details,
@@ -89,7 +91,10 @@ def create_app() -> FastAPI:
             profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
         )
 
-    app = FastAPI(docs_url="/api/docs", lifespan=lifespan)
+    app = FastAPI(
+        docs_url="/api/docs",
+        lifespan=lifespan,
+    )
     app.state.proxy_dependency_injector = ServerProxyDependencyInjector()
     return app
 
@@ -97,6 +102,8 @@ def create_app() -> FastAPI:
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     configure_logging()
+    server_executor = ThreadPoolExecutor(max_workers=settings.SERVER_EXECUTOR_MAX_WORKERS)
+    asyncio.get_running_loop().set_default_executor(server_executor)
     await migrate()
     _print_dstack_logo()
     if not check_required_ssh_version():
@@ -144,7 +151,10 @@ async def lifespan(app: FastAPI):
     )
     if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None:
         init_default_storage()
-    scheduler = start_background_tasks()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler = start_background_tasks()
+    else:
+        logger.info("Background processing is disabled")
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
@@ -154,7 +164,8 @@ async def lifespan(app: FastAPI):
     for func in _ON_STARTUP_HOOKS:
         await func(app)
     yield
-    scheduler.shutdown()
+    if settings.SERVER_BACKGROUND_PROCESSING_ENABLED:
+        scheduler.shutdown()
     await gateway_connections_pool.remove_all()
     service_conn_pool = await get_injector_from_app(app).get_service_connection_pool()
     await service_conn_pool.remove_all()
@@ -205,14 +216,14 @@ def register_routes(app: FastAPI, ui: bool = True):
         msg = "Access denied"
         if len(exc.args) > 0:
             msg = exc.args[0]
-        return JSONResponse(
+        return CustomORJSONResponse(
             status_code=status.HTTP_403_FORBIDDEN,
             content=error_detail(msg),
         )
 
     @app.exception_handler(ServerClientError)
     async def server_client_error_handler(request: Request, exc: ServerClientError):
-        return JSONResponse(
+        return CustomORJSONResponse(
             status_code=status.HTTP_400_BAD_REQUEST,
             content={"detail": get_server_client_error_details(exc)},
         )
@@ -220,7 +231,7 @@ def register_routes(app: FastAPI, ui: bool = True):
     @app.exception_handler(OSError)
     async def os_error_handler(request, exc: OSError):
         if exc.errno in [36, 63]:
-            return JSONResponse(
+            return CustomORJSONResponse(
                 {"detail": "Filename too long"},
                 status_code=status.HTTP_400_BAD_REQUEST,
             )
@@ -242,6 +253,23 @@ def register_routes(app: FastAPI, ui: bool = True):
         )
         return response
 
+    if settings.SERVER_PROFILING_ENABLED:
+        from pyinstrument import Profiler
+
+        @app.middleware("http")
+        async def profile_request(request: Request, call_next):
+            profiling = request.query_params.get("profile", False)
+            if profiling:
+                profiler = Profiler()
+                profiler.start()
+                respone = await call_next(request)
+                profiler.stop()
+                with open("profiling_results.html", "w+") as f:
+                    f.write(profiler.output_html())
+                return respone
+            else:
+                return await call_next(request)
+
     # this middleware must be defined after the log_request middleware
     @app.middleware("http")
     async def log_http_metrics(request: Request, call_next):
@@ -289,7 +317,7 @@ def register_routes(app: FastAPI, ui: bool = True):
 
     @app.get("/healthcheck")
     async def healthcheck():
-        return JSONResponse(content={"status": "running"})
+        return CustomORJSONResponse(content={"status": "running"})
 
     if ui and Path(__file__).parent.joinpath("statics").exists():
         app.mount(
@@ -303,7 +331,7 @@ def register_routes(app: FastAPI, ui: bool = True):
                 or _is_proxy_request(request)
                 or _is_prometheus_request(request)
             ):
-                return JSONResponse(
+                return CustomORJSONResponse(
                     {"detail": exc.detail},
                     status_code=status.HTTP_404_NOT_FOUND,
                 )

dstack/_internal/server/background/__init__.py

@@ -4,9 +4,10 @@ from apscheduler.triggers.interval import IntervalTrigger
 from dstack._internal.server import settings
 from dstack._internal.server.background.tasks.process_fleets import process_fleets
 from dstack._internal.server.background.tasks.process_gateways import (
+    process_gateways,
     process_gateways_connections,
-    process_submitted_gateways,
 )
+from dstack._internal.server.background.tasks.process_idle_volumes import process_idle_volumes
 from dstack._internal.server.background.tasks.process_instances import (
     process_instances,
 )
@@ -37,15 +38,31 @@ def get_scheduler() -> AsyncIOScheduler:
 
 
 def start_background_tasks() -> AsyncIOScheduler:
+    # We try to process as many resources as possible without exhausting DB connections.
+    #
+    # Quick tasks can process multiple resources per transaction.
+    # Potentially long tasks process one resource per transaction
+    # to avoid holding locks for all the resources if one is slow to process.
+    # Still, the next batch won't be processed unless all resources are processed,
+    # so larger batches do not increase processing rate linearly.
+    #
+    # The interval, batch_size, and max_instances determine background tasks processing rates.
+    # By default, one server replica can handle:
+    #
+    # * 150 active jobs with 2 minutes processing latency
+    # * 150 active runs with 2 minutes processing latency
+    # * 150 active instances with 2 minutes processing latency
+    #
+    # These latency numbers do not account for provisioning time,
+    # so it may be slower if a backend is slow to provision.
+    #
+    # Users can set SERVER_BACKGROUND_PROCESSING_FACTOR to process more resources per replica.
+    # They also need to increase max db connections on the client side and db side.
+    #
     # In-memory locking via locksets does not guarantee
     # that the first waiting for the lock will acquire it.
     # The jitter is needed to give all tasks a chance to acquire locks.
 
-    # The batch_size and interval determine background tasks processing rates.
-    # Currently one server replica can handle:
-    # * 150 active jobs with up to 2 minutes processing latency
-    # * 150 active runs with up to 2 minutes processing latency
-    # * 150 active instances with up to 2 minutes processing latency
     _scheduler.add_job(collect_metrics, IntervalTrigger(seconds=10), max_instances=1)
     _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1)
     if settings.ENABLE_PROMETHEUS_METRICS:
@@ -53,45 +70,54 @@ def start_background_tasks() -> AsyncIOScheduler:
             collect_prometheus_metrics, IntervalTrigger(seconds=10), max_instances=1
         )
         _scheduler.add_job(delete_prometheus_metrics, IntervalTrigger(minutes=5), max_instances=1)
-    # process_submitted_jobs and process_instances max processing rate is 75 jobs(instances) per minute.
-    _scheduler.add_job(
-        process_submitted_jobs,
-        IntervalTrigger(seconds=4, jitter=2),
-        kwargs={"batch_size": 5},
-        max_instances=2,
-    )
-    _scheduler.add_job(
-        process_running_jobs,
-        IntervalTrigger(seconds=4, jitter=2),
-        kwargs={"batch_size": 5},
-        max_instances=2,
-    )
-    _scheduler.add_job(
-        process_terminating_jobs,
-        IntervalTrigger(seconds=4, jitter=2),
-        kwargs={"batch_size": 5},
-        max_instances=2,
-    )
-    _scheduler.add_job(
-        process_runs,
-        IntervalTrigger(seconds=2, jitter=1),
-        kwargs={"batch_size": 5},
-        max_instances=2,
-    )
-    _scheduler.add_job(
-        process_instances,
-        IntervalTrigger(seconds=4, jitter=2),
-        kwargs={"batch_size": 5},
-        max_instances=2,
-    )
-    _scheduler.add_job(process_fleets, IntervalTrigger(seconds=10, jitter=2))
     _scheduler.add_job(process_gateways_connections, IntervalTrigger(seconds=15))
+    _scheduler.add_job(process_gateways, IntervalTrigger(seconds=10, jitter=2), max_instances=5)
     _scheduler.add_job(
-        process_submitted_gateways, IntervalTrigger(seconds=10, jitter=2), max_instances=5
+        process_submitted_volumes, IntervalTrigger(seconds=10, jitter=2), max_instances=5
     )
     _scheduler.add_job(
-        process_submitted_volumes, IntervalTrigger(seconds=10, jitter=2), max_instances=5
+        process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1
     )
     _scheduler.add_job(process_placement_groups, IntervalTrigger(seconds=30, jitter=5))
+    for replica in range(settings.SERVER_BACKGROUND_PROCESSING_FACTOR):
+        # Add multiple copies of tasks if requested.
+        # max_instances=1 for additional copies to avoid running too many tasks.
+        # Move other tasks here when they need per-replica scaling.
+        _scheduler.add_job(
+            process_submitted_jobs,
+            IntervalTrigger(seconds=4, jitter=2),
+            kwargs={"batch_size": 5},
+            max_instances=4 if replica == 0 else 1,
+        )
+        _scheduler.add_job(
+            process_running_jobs,
+            IntervalTrigger(seconds=4, jitter=2),
+            kwargs={"batch_size": 5},
+            max_instances=2 if replica == 0 else 1,
+        )
+        _scheduler.add_job(
+            process_terminating_jobs,
+            IntervalTrigger(seconds=4, jitter=2),
+            kwargs={"batch_size": 5},
+            max_instances=2 if replica == 0 else 1,
+        )
+        _scheduler.add_job(
+            process_runs,
+            IntervalTrigger(seconds=2, jitter=1),
+            kwargs={"batch_size": 5},
+            max_instances=2 if replica == 0 else 1,
+        )
+        _scheduler.add_job(
+            process_instances,
+            IntervalTrigger(seconds=4, jitter=2),
+            kwargs={"batch_size": 5},
+            max_instances=2 if replica == 0 else 1,
+        )
+        _scheduler.add_job(
+            process_fleets,
+            IntervalTrigger(seconds=10, jitter=2),
+            kwargs={"batch_size": 5},
+            max_instances=2 if replica == 0 else 1,
+        )
     _scheduler.start()
     return _scheduler

dstack/_internal/server/background/tasks/process_fleets.py

@@ -1,9 +1,12 @@
+import asyncio
+from datetime import timedelta
+
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
 from dstack._internal.core.models.fleets import FleetStatus
-from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import FleetModel
 from dstack._internal.server.services.fleets import (
     is_fleet_empty,
@@ -17,8 +20,18 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
-async def process_fleets():
-    lock, lockset = get_locker().get_lockset(FleetModel.__tablename__)
+MIN_PROCESSING_INTERVAL = timedelta(seconds=30)
+
+
+async def process_fleets(batch_size: int = 1):
+    tasks = []
+    for _ in range(batch_size):
+        tasks.append(_process_next_fleet())
+    await asyncio.gather(*tasks)
+
+
+async def _process_next_fleet():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(FleetModel.__tablename__)
     async with get_session_ctx() as session:
         async with lock:
             res = await session.execute(
@@ -26,6 +39,8 @@ async def process_fleets():
                 .where(
                     FleetModel.deleted == False,
                     FleetModel.id.not_in(lockset),
+                    FleetModel.last_processed_at
+                    < get_current_datetime().replace(tzinfo=None) - MIN_PROCESSING_INTERVAL,
                 )
                 .order_by(FleetModel.last_processed_at.asc())
                 .limit(1)
@@ -43,6 +58,7 @@ async def process_fleets():
 
 
 async def _process_fleet(session: AsyncSession, fleet_model: FleetModel):
+    logger.debug("Processing fleet %s", fleet_model.name)
     # Refetch to load related attributes.
     # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(