dstack 0.19.20__tar.gz → 0.19.21__tar.gz

{dstack-0.19.20 → dstack-0.19.21}/.justfile

@@ -14,4 +14,4 @@ set allow-duplicate-recipes
 
 import "runner/.justfile"
 
-# TODO: Add frontend/justfile for managing frontend development tasks
+import "frontend/.justfile"

{dstack-0.19.20 → dstack-0.19.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.20
+Version: 0.19.21
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -15,6 +15,7 @@ Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9
+Requires-Dist: apscheduler<4
 Requires-Dist: argcomplete>=3.5.0
 Requires-Dist: cachetools
 Requires-Dist: cryptography
@@ -46,7 +47,6 @@ Requires-Dist: aiorwlock; extra == 'all'
 Requires-Dist: aiosqlite; extra == 'all'
 Requires-Dist: alembic-postgresql-enum; extra == 'all'
 Requires-Dist: alembic>=1.10.2; extra == 'all'
-Requires-Dist: apscheduler<4; extra == 'all'
 Requires-Dist: asyncpg; extra == 'all'
 Requires-Dist: azure-identity>=1.12.0; extra == 'all'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'all'
@@ -92,7 +92,6 @@ Requires-Dist: aiorwlock; extra == 'aws'
 Requires-Dist: aiosqlite; extra == 'aws'
 Requires-Dist: alembic-postgresql-enum; extra == 'aws'
 Requires-Dist: alembic>=1.10.2; extra == 'aws'
-Requires-Dist: apscheduler<4; extra == 'aws'
 Requires-Dist: asyncpg; extra == 'aws'
 Requires-Dist: backports-entry-points-selectable; extra == 'aws'
 Requires-Dist: boto3>=1.38.13; extra == 'aws'
@@ -117,7 +116,6 @@ Requires-Dist: aiorwlock; extra == 'azure'
 Requires-Dist: aiosqlite; extra == 'azure'
 Requires-Dist: alembic-postgresql-enum; extra == 'azure'
 Requires-Dist: alembic>=1.10.2; extra == 'azure'
-Requires-Dist: apscheduler<4; extra == 'azure'
 Requires-Dist: asyncpg; extra == 'azure'
 Requires-Dist: azure-identity>=1.12.0; extra == 'azure'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'azure'
@@ -147,7 +145,6 @@ Requires-Dist: aiorwlock; extra == 'datacrunch'
 Requires-Dist: aiosqlite; extra == 'datacrunch'
 Requires-Dist: alembic-postgresql-enum; extra == 'datacrunch'
 Requires-Dist: alembic>=1.10.2; extra == 'datacrunch'
-Requires-Dist: apscheduler<4; extra == 'datacrunch'
 Requires-Dist: asyncpg; extra == 'datacrunch'
 Requires-Dist: backports-entry-points-selectable; extra == 'datacrunch'
 Requires-Dist: datacrunch; extra == 'datacrunch'
@@ -179,7 +176,6 @@ Requires-Dist: aiorwlock; extra == 'gcp'
 Requires-Dist: aiosqlite; extra == 'gcp'
 Requires-Dist: alembic-postgresql-enum; extra == 'gcp'
 Requires-Dist: alembic>=1.10.2; extra == 'gcp'
-Requires-Dist: apscheduler<4; extra == 'gcp'
 Requires-Dist: asyncpg; extra == 'gcp'
 Requires-Dist: backports-entry-points-selectable; extra == 'gcp'
 Requires-Dist: docker>=6.0.0; extra == 'gcp'
@@ -209,7 +205,6 @@ Requires-Dist: aiorwlock; extra == 'kubernetes'
 Requires-Dist: aiosqlite; extra == 'kubernetes'
 Requires-Dist: alembic-postgresql-enum; extra == 'kubernetes'
 Requires-Dist: alembic>=1.10.2; extra == 'kubernetes'
-Requires-Dist: apscheduler<4; extra == 'kubernetes'
 Requires-Dist: asyncpg; extra == 'kubernetes'
 Requires-Dist: backports-entry-points-selectable; extra == 'kubernetes'
 Requires-Dist: docker>=6.0.0; extra == 'kubernetes'
@@ -233,7 +228,6 @@ Requires-Dist: aiorwlock; extra == 'lambda'
 Requires-Dist: aiosqlite; extra == 'lambda'
 Requires-Dist: alembic-postgresql-enum; extra == 'lambda'
 Requires-Dist: alembic>=1.10.2; extra == 'lambda'
-Requires-Dist: apscheduler<4; extra == 'lambda'
 Requires-Dist: asyncpg; extra == 'lambda'
 Requires-Dist: backports-entry-points-selectable; extra == 'lambda'
 Requires-Dist: boto3>=1.38.13; extra == 'lambda'
@@ -258,7 +252,6 @@ Requires-Dist: aiorwlock; extra == 'nebius'
 Requires-Dist: aiosqlite; extra == 'nebius'
 Requires-Dist: alembic-postgresql-enum; extra == 'nebius'
 Requires-Dist: alembic>=1.10.2; extra == 'nebius'
-Requires-Dist: apscheduler<4; extra == 'nebius'
 Requires-Dist: asyncpg; extra == 'nebius'
 Requires-Dist: backports-entry-points-selectable; extra == 'nebius'
 Requires-Dist: docker>=6.0.0; extra == 'nebius'
@@ -282,7 +275,6 @@ Requires-Dist: aiorwlock; extra == 'oci'
 Requires-Dist: aiosqlite; extra == 'oci'
 Requires-Dist: alembic-postgresql-enum; extra == 'oci'
 Requires-Dist: alembic>=1.10.2; extra == 'oci'
-Requires-Dist: apscheduler<4; extra == 'oci'
 Requires-Dist: asyncpg; extra == 'oci'
 Requires-Dist: backports-entry-points-selectable; extra == 'oci'
 Requires-Dist: cryptography>=44.0.3; extra == 'oci'
@@ -308,7 +300,6 @@ Requires-Dist: aiorwlock; extra == 'server'
 Requires-Dist: aiosqlite; extra == 'server'
 Requires-Dist: alembic-postgresql-enum; extra == 'server'
 Requires-Dist: alembic>=1.10.2; extra == 'server'
-Requires-Dist: apscheduler<4; extra == 'server'
 Requires-Dist: asyncpg; extra == 'server'
 Requires-Dist: backports-entry-points-selectable; extra == 'server'
 Requires-Dist: docker>=6.0.0; extra == 'server'

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/concepts/dev-environments.md

@@ -471,6 +471,51 @@ utilization_policy:
 
 </div>
 
+### Schedule
+
+Specify `schedule` to start a dev environment periodically at specific UTC times using the cron syntax:
+
+<div editor-title=".dstack.yml">
+
+```yaml
+type: dev-environment
+ide: vscode
+schedule:
+  cron: "0 8 * * mon-fri" # at 8:00 UTC from Monday through Friday
+```
+
+</div>
+
+The `schedule` property can be combined with `max_duration` or `utilization_policy` to shutdown the dev environment automatically when it's not needed.
+
+??? info "Cron syntax"
+    `dstack` supports [POSIX cron syntax](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07). One exception is that days of the week are started from Monday instead of Sunday so `0` corresponds to Monday.
+    
+    The month and day of week fields accept abbreviated English month and weekday names (`jan–dec` and `mon–sun`) respectively.
+
+    A cron expression consists of five fields:
+
+    ```
+    ┌───────────── minute (0-59)
+    │ ┌───────────── hour (0-23)
+    │ │ ┌───────────── day of the month (1-31)
+    │ │ │ ┌───────────── month (1-12 or jan-dec)
+    │ │ │ │ ┌───────────── day of the week (0-6 or mon-sun)
+    │ │ │ │ │
+    │ │ │ │ │
+    │ │ │ │ │
+    * * * * *
+    ```
+
+    The following operators can be used in any of the fields:
+
+    | Operator | Description           | Example                                                                 |
+    |----------|-----------------------|-------------------------------------------------------------------------|
+    | `*`      | Any value             | `0 * * * *` runs every hour at minute 0                                 |
+    | `,`      | Value list separator  | `15,45 10 * * *` runs at 10:15 and 10:45 every day.                     |
+    | `-`      | Range of values       | `0 1-3 * * *` runs at 1:00, 2:00, and 3:00 every day.                   |
+    | `/`      | Step values           | `*/10 8-10 * * *` runs every 10 minutes during the hours 8:00 to 10:59. |
+
 ### Spot policy
 
 By default, `dstack` uses on-demand instances. However, you can change that

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/concepts/services.md

@@ -670,6 +670,64 @@ utilization_policy:
 
 </div>
 
+### Schedule
+
+Specify `schedule` to start a service periodically at specific UTC times using the cron syntax:
+
+<div editor-title=".dstack.yml">
+
+```yaml
+type: service
+name: llama-2-7b-service
+
+python: 3.12
+env:
+  - HF_TOKEN
+  - MODEL=NousResearch/Llama-2-7b-chat-hf
+commands:
+  - uv pip install vllm
+  - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000
+port: 8000
+
+resources:
+  gpu: 24GB
+
+schedule:
+  cron: "0 8 * * mon-fri" # at 8:00 UTC from Monday through Friday
+```
+
+</div>
+
+The `schedule` property can be combined with `max_duration` or `utilization_policy` to shutdown the service automatically when it's not needed.
+
+??? info "Cron syntax"
+    `dstack` supports [POSIX cron syntax](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07). One exception is that days of the week are started from Monday instead of Sunday so `0` corresponds to Monday.
+    
+    The month and day of week fields accept abbreviated English month and weekday names (`jan–dec` and `mon–sun`) respectively.
+
+    A cron expression consists of five fields:
+
+    ```
+    ┌───────────── minute (0-59)
+    │ ┌───────────── hour (0-23)
+    │ │ ┌───────────── day of the month (1-31)
+    │ │ │ ┌───────────── month (1-12 or jan-dec)
+    │ │ │ │ ┌───────────── day of the week (0-6 or mon-sun)
+    │ │ │ │ │
+    │ │ │ │ │
+    │ │ │ │ │
+    * * * * *
+    ```
+
+    The following operators can be used in any of the fields:
+
+    | Operator | Description           | Example                                                                 |
+    |----------|-----------------------|-------------------------------------------------------------------------|
+    | `*`      | Any value             | `0 * * * *` runs every hour at minute 0                                 |
+    | `,`      | Value list separator  | `15,45 10 * * *` runs at 10:15 and 10:45 every day.                     |
+    | `-`      | Range of values       | `0 1-3 * * *` runs at 1:00, 2:00, and 3:00 every day.                   |
+    | `/`      | Step values           | `*/10 8-10 * * *` runs every 10 minutes during the hours 8:00 to 10:59. |
+
 --8<-- "docs/concepts/snippets/manage-fleets.ext"
 
 !!! info "Reference"

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/concepts/tasks.md

@@ -651,6 +651,58 @@ utilization_policy:
 
 </div>
 
+### Schedule
+
+Specify `schedule` to start a task periodically at specific UTC times using the cron syntax:
+
+<div editor-title=".dstack.yml">
+
+```yaml
+type: task
+name: train
+
+python: 3.12
+commands:
+  - uv pip install -r fine-tuning/qlora/requirements.txt
+  - python fine-tuning/qlora/train.py
+
+resources:
+  gpu: H100:8
+
+schedule:
+  cron: "15 23 * * *" # everyday at 23:15 UTC
+```
+
+</div>
+
+??? info "Cron syntax"
+    `dstack` supports [POSIX cron syntax](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07). One exception is that days of the week are started from Monday instead of Sunday so `0` corresponds to Monday.
+    
+    The month and day of week fields accept abbreviated English month and weekday names (`jan–dec` and `mon–sun`) respectively.
+
+    A cron expression consists of five fields:
+
+    ```
+    ┌───────────── minute (0-59)
+    │ ┌───────────── hour (0-23)
+    │ │ ┌───────────── day of the month (1-31)
+    │ │ │ ┌───────────── month (1-12 or jan-dec)
+    │ │ │ │ ┌───────────── day of the week (0-6 or mon-sun)
+    │ │ │ │ │
+    │ │ │ │ │
+    │ │ │ │ │
+    * * * * *
+    ```
+
+    The following operators can be used in any of the fields:
+
+    | Operator | Description           | Example                                                                 |
+    |----------|-----------------------|-------------------------------------------------------------------------|
+    | `*`      | Any value             | `0 * * * *` runs every hour at minute 0                                 |
+    | `,`      | Value list separator  | `15,45 10 * * *` runs at 10:15 and 10:45 every day.                     |
+    | `-`      | Range of values       | `0 1-3 * * *` runs at 1:00, 2:00, and 3:00 every day.                   |
+    | `/`      | Step values           | `*/10 8-10 * * *` runs every 10 minutes during the hours 8:00 to 10:59. |
+
 ### Spot policy
 
 By default, `dstack` uses on-demand instances. However, you can change that

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/reference/dstack.yml/dev-environment.md

@@ -26,6 +26,14 @@ The `dev-environment` configuration type allows running [dev environments](../..
       type:
         required: true
 
+### `schedule`
+
+#SCHEMA# dstack._internal.core.models.profiles.Schedule
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
 ### `resources`
 
 #SCHEMA# dstack._internal.core.models.resources.ResourcesSpec

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/reference/dstack.yml/service.md

@@ -120,6 +120,14 @@ The `service` configuration type allows running [services](../../concepts/servic
       type:
         required: true
 
+### `schedule`
+
+#SCHEMA# dstack._internal.core.models.profiles.Schedule
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
 ### `resources`
 
 #SCHEMA# dstack._internal.core.models.resources.ResourcesSpec

{dstack-0.19.20 → dstack-0.19.21}/docs/docs/reference/dstack.yml/task.md

@@ -26,6 +26,14 @@ The `task` configuration type allows running [tasks](../../concepts/tasks.md).
       type:
         required: true
 
+### `schedule`
+
+#SCHEMA# dstack._internal.core.models.profiles.Schedule
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
 ### `resources`
 
 #SCHEMA# dstack._internal.core.models.resources.ResourcesSpec

dstack-0.19.21/frontend/.justfile

@@ -0,0 +1,20 @@
+# Justfile for building frontend
+#
+# Run `just` to see all available commands
+
+default:
+    @just --list
+
+[private]
+install-frontend:
+    #!/usr/bin/env bash
+    set -e
+    cd {{source_directory()}}
+    npm install
+
+build-frontend:
+    #!/usr/bin/env bash
+    set -e
+    cd {{source_directory()}}
+    npm run build
+    cp -r build/ ../src/dstack/_internal/server/statics/

dstack-0.19.20/src/dstack/_internal/core/backends/__init__.py → dstack-0.19.21/src/dstack/_internal/core/backends/features.py

@@ -13,6 +13,8 @@ from dstack._internal.core.backends.local.compute import LocalCompute
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.settings import LOCAL_BACKEND_ENABLED
 
+_configurator_classes = list_available_configurator_classes()
+
 
 def _get_backends_with_compute_feature(
     configurator_classes: list[type[Configurator]],
@@ -31,9 +33,6 @@ def _get_backends_with_compute_feature(
     return backend_types
 
 
-_configurator_classes = list_available_configurator_classes()
-
-
 # The following backend lists do not include unavailable backends (i.e. backends missing deps).
 BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
     configurator_classes=_configurator_classes,

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/core/compatibility/fleets.py

@@ -57,6 +57,8 @@ def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[IncludeExcludeDic
         profile_excludes.add("startup_order")
     if profile.stop_criteria is None:
         profile_excludes.add("stop_criteria")
+    if profile.schedule is None:
+        profile_excludes.add("schedule")
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes
     if profile_excludes:

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/core/compatibility/runs.py

@@ -126,6 +126,10 @@ def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType:
         configuration_excludes["files"] = True
     if not run_spec.file_archives:
         spec_excludes["file_archives"] = True
+    if configuration.schedule is None:
+        configuration_excludes["schedule"] = True
+    if profile is not None and profile.schedule is None:
+        profile_excludes.add("schedule")
 
     if configuration_excludes:
         spec_excludes["configuration"] = configuration_excludes

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/core/models/profiles.py

@@ -8,6 +8,7 @@ from typing_extensions import Annotated, Literal
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel, Duration
 from dstack._internal.utils.common import list_enum_values_for_annotation
+from dstack._internal.utils.cron import validate_cron
 from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent
 from dstack._internal.utils.tags import tags_validator
 
@@ -167,6 +168,38 @@ class UtilizationPolicy(CoreModel):
         return v
 
 
+class Schedule(CoreModel):
+    cron: Annotated[
+        Union[List[str], str],
+        Field(
+            description=(
+                "A cron expression or a list of cron expressions specifying the UTC time when the run needs to be started"
+            )
+        ),
+    ]
+
+    @validator("cron")
+    def _validate_cron(cls, v: Union[List[str], str]) -> List[str]:
+        if isinstance(v, str):
+            values = [v]
+        else:
+            values = v
+        if len(values) == 0:
+            raise ValueError("At least one cron expression must be specified")
+        for value in values:
+            validate_cron(value)
+        return values
+
+    @property
+    def crons(self) -> List[str]:
+        """
+        Access `cron` attribute as a list.
+        """
+        if isinstance(self.cron, str):
+            return [self.cron]
+        return self.cron
+
+
 class ProfileParams(CoreModel):
     backends: Annotated[
         Optional[List[BackendType]],
@@ -281,6 +314,10 @@ class ProfileParams(CoreModel):
             )
         ),
     ] = None
+    schedule: Annotated[
+        Optional[Schedule],
+        Field(description=("The schedule for starting the run at specified time")),
+    ] = None
     fleets: Annotated[
         Optional[list[str]], Field(description="The fleets considered for reuse")
     ] = None

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/server/app.py

@@ -13,6 +13,7 @@ from fastapi.datastructures import URL
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import Counter, Histogram
+from sentry_sdk.types import SamplingContext
 
 from dstack._internal.cli.utils.common import console
 from dstack._internal.core.errors import ForbiddenError, ServerClientError
@@ -81,16 +82,6 @@ REQUEST_DURATION = Histogram(
 
 
 def create_app() -> FastAPI:
-    if settings.SENTRY_DSN is not None:
-        sentry_sdk.init(
-            dsn=settings.SENTRY_DSN,
-            release=DSTACK_VERSION,
-            environment=settings.SERVER_ENVIRONMENT,
-            enable_tracing=True,
-            traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
-            profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
-        )
-
     app = FastAPI(
         docs_url="/api/docs",
         lifespan=lifespan,
@@ -102,6 +93,15 @@ def create_app() -> FastAPI:
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     configure_logging()
+    if settings.SENTRY_DSN is not None:
+        sentry_sdk.init(
+            dsn=settings.SENTRY_DSN,
+            release=DSTACK_VERSION,
+            environment=settings.SERVER_ENVIRONMENT,
+            enable_tracing=True,
+            traces_sampler=_sentry_traces_sampler,
+            profiles_sample_rate=settings.SENTRY_PROFILES_SAMPLE_RATE,
+        )
     server_executor = ThreadPoolExecutor(max_workers=settings.SERVER_EXECUTOR_MAX_WORKERS)
     asyncio.get_running_loop().set_default_executor(server_executor)
     await migrate()
@@ -379,3 +379,15 @@ def _print_dstack_logo():
 ╰━━┻━━┻╯╱╰╯╰━━┻╯
 [/]"""
     )
+
+
+def _sentry_traces_sampler(sampling_context: SamplingContext) -> float:
+    parent_sampling_decision = sampling_context["parent_sampled"]
+    if parent_sampling_decision is not None:
+        return float(parent_sampling_decision)
+    transaction_context = sampling_context["transaction_context"]
+    name = transaction_context.get("name")
+    if name is not None:
+        if name.startswith("background."):
+            return settings.SENTRY_TRACES_BACKGROUND_SAMPLE_RATE
+    return settings.SENTRY_TRACES_SAMPLE_RATE

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/server/background/__init__.py

@@ -79,6 +79,11 @@ def start_background_tasks() -> AsyncIOScheduler:
         process_idle_volumes, IntervalTrigger(seconds=60, jitter=10), max_instances=1
     )
     _scheduler.add_job(process_placement_groups, IntervalTrigger(seconds=30, jitter=5))
+    _scheduler.add_job(
+        process_fleets,
+        IntervalTrigger(seconds=10, jitter=2),
+        max_instances=1,
+    )
     for replica in range(settings.SERVER_BACKGROUND_PROCESSING_FACTOR):
         # Add multiple copies of tasks if requested.
         # max_instances=1 for additional copies to avoid running too many tasks.
@@ -113,11 +118,5 @@ def start_background_tasks() -> AsyncIOScheduler:
             kwargs={"batch_size": 5},
             max_instances=2 if replica == 0 else 1,
         )
-        _scheduler.add_job(
-            process_fleets,
-            IntervalTrigger(seconds=10, jitter=2),
-            kwargs={"batch_size": 5},
-            max_instances=2 if replica == 0 else 1,
-        )
     _scheduler.start()
     return _scheduler

dstack-0.19.21/src/dstack/_internal/server/background/tasks/process_fleets.py

@@ -0,0 +1,104 @@
+from datetime import timedelta
+from typing import List
+
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload, load_only
+
+from dstack._internal.core.models.fleets import FleetStatus
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import (
+    FleetModel,
+    InstanceModel,
+    JobModel,
+    PlacementGroupModel,
+    RunModel,
+)
+from dstack._internal.server.services.fleets import (
+    is_fleet_empty,
+    is_fleet_in_use,
+)
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.utils import sentry_utils
+from dstack._internal.utils.common import get_current_datetime
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+BATCH_SIZE = 10
+MIN_PROCESSING_INTERVAL = timedelta(seconds=30)
+
+
+@sentry_utils.instrument_background_task
+async def process_fleets():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(FleetModel.__tablename__)
+    async with get_session_ctx() as session:
+        async with lock:
+            res = await session.execute(
+                select(FleetModel)
+                .where(
+                    FleetModel.deleted == False,
+                    FleetModel.id.not_in(lockset),
+                    FleetModel.last_processed_at
+                    < get_current_datetime() - MIN_PROCESSING_INTERVAL,
+                )
+                .options(load_only(FleetModel.id))
+                .order_by(FleetModel.last_processed_at.asc())
+                .limit(BATCH_SIZE)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            fleet_models = list(res.scalars().all())
+            fleet_ids = [fm.id for fm in fleet_models]
+            for fleet_id in fleet_ids:
+                lockset.add(fleet_id)
+        try:
+            await _process_fleets(session=session, fleet_models=fleet_models)
+        finally:
+            lockset.difference_update(fleet_ids)
+
+
+async def _process_fleets(session: AsyncSession, fleet_models: List[FleetModel]):
+    fleet_ids = [fm.id for fm in fleet_models]
+    # Refetch to load related attributes.
+    res = await session.execute(
+        select(FleetModel)
+        .where(FleetModel.id.in_(fleet_ids))
+        .options(joinedload(FleetModel.instances).load_only(InstanceModel.deleted))
+        .options(
+            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id)
+        )
+        .options(joinedload(FleetModel.runs).load_only(RunModel.status))
+        .execution_options(populate_existing=True)
+    )
+    fleet_models = list(res.unique().scalars().all())
+
+    deleted_fleets_ids = []
+    now = get_current_datetime()
+    for fleet_model in fleet_models:
+        deleted = _autodelete_fleet(fleet_model)
+        if deleted:
+            deleted_fleets_ids.append(fleet_model.id)
+        fleet_model.last_processed_at = now
+
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            PlacementGroupModel.fleet_id.in_(deleted_fleets_ids),
+        )
+        .values(fleet_deleted=True)
+    )
+    await session.commit()
+
+
+def _autodelete_fleet(fleet_model: FleetModel) -> bool:
+    # Currently all empty fleets are autodeleted.
+    # TODO: If fleets with `nodes: 0..` are supported, their deletion should be skipped.
+    if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
+        return False
+
+    logger.info("Automatic cleanup of an empty fleet %s", fleet_model.name)
+    fleet_model.status = FleetStatus.TERMINATED
+    fleet_model.deleted = True
+    logger.info("Fleet %s deleted", fleet_model.name)
+    return True

{dstack-0.19.20 → dstack-0.19.21}/src/dstack/_internal/server/background/tasks/process_gateways.py

@@ -17,6 +17,7 @@ from dstack._internal.server.services.gateways import (
 )
 from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils.common import get_current_datetime
 from dstack._internal.utils.logging import get_logger
 
@@ -28,6 +29,7 @@ async def process_gateways_connections():
     await _process_active_connections()
 
 
+@sentry_utils.instrument_background_task
 async def process_gateways():
     lock, lockset = get_locker(get_db().dialect_name).get_lockset(GatewayModel.__tablename__)
     async with get_session_ctx() as session:
@@ -110,7 +112,6 @@ async def _process_connection(conn: GatewayConnection):
 async def _process_submitted_gateway(session: AsyncSession, gateway_model: GatewayModel):
     logger.info("%s: started gateway provisioning", fmt(gateway_model))
     # Refetch to load related attributes.
-    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
         select(GatewayModel)
         .where(GatewayModel.id == gateway_model.id)
@@ -157,7 +158,6 @@ async def _process_provisioning_gateway(
     session: AsyncSession, gateway_model: GatewayModel
 ) -> None:
     # Refetch to load related attributes.
-    # joinedload produces LEFT OUTER JOIN that can't be used with FOR UPDATE.
     res = await session.execute(
         select(GatewayModel)
         .where(GatewayModel.id == gateway_model.id)