dstack 0.19.20__py3-none-any.whl → 0.19.21__py3-none-any.whl

dstack/_internal/server/services/users.py

@@ -2,7 +2,6 @@ import hashlib
 import os
 import re
 import uuid
-from datetime import timezone
 from typing import Awaitable, Callable, List, Optional, Tuple
 
 from sqlalchemy import delete, select, update
@@ -195,7 +194,7 @@ def user_model_to_user(user_model: UserModel) -> User:
     return User(
         id=user_model.id,
         username=user_model.name,
-        created_at=user_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=user_model.created_at,
         global_role=user_model.global_role,
         email=user_model.email,
         active=user_model.active,
@@ -207,7 +206,7 @@ def user_model_to_user_with_creds(user_model: UserModel) -> UserWithCreds:
     return UserWithCreds(
         id=user_model.id,
         username=user_model.name,
-        created_at=user_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=user_model.created_at,
         global_role=user_model.global_role,
         email=user_model.email,
         active=user_model.active,

dstack/_internal/server/services/volumes.py

@@ -1,19 +1,18 @@
 import uuid
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from typing import List, Optional
 
 from sqlalchemy import and_, func, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, selectinload
 
-from dstack._internal.core.backends import BACKENDS_WITH_VOLUMES_SUPPORT
 from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
+from dstack._internal.core.backends.features import BACKENDS_WITH_VOLUMES_SUPPORT
 from dstack._internal.core.errors import (
     BackendNotAvailable,
     ResourceExistsError,
     ServerClientError,
 )
-from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import (
     Volume,
     VolumeAttachment,
@@ -40,7 +39,7 @@ from dstack._internal.server.services.locking import (
     string_to_lock_id,
 )
 from dstack._internal.server.services.plugins import apply_plugin_policies
-from dstack._internal.server.services.projects import list_project_models, list_user_project_models
+from dstack._internal.server.services.projects import list_user_project_models
 from dstack._internal.utils import common, random_names
 from dstack._internal.utils.logging import get_logger
 
@@ -57,10 +56,11 @@ async def list_volumes(
     limit: int,
     ascending: bool,
 ) -> List[Volume]:
-    if user.global_role == GlobalRole.ADMIN:
-        projects = await list_project_models(session=session)
-    else:
-        projects = await list_user_project_models(session=session, user=user)
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+        only_names=True,
+    )
     if project_name is not None:
         projects = [p for p in projects if p.name == project_name]
     volume_models = await list_projects_volume_models(
@@ -320,15 +320,15 @@ def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
         )
     deleted_at = None
     if volume_model.deleted_at is not None:
-        deleted_at = volume_model.deleted_at.replace(tzinfo=timezone.utc)
+        deleted_at = volume_model.deleted_at
     volume = Volume(
         name=volume_model.name,
         project_name=volume_model.project.name,
         user=volume_model.user.name,
         configuration=configuration,
         external=configuration.volume_id is not None,
-        created_at=volume_model.created_at.replace(tzinfo=timezone.utc),
-        last_processed_at=volume_model.last_processed_at.replace(tzinfo=timezone.utc),
+        created_at=volume_model.created_at,
+        last_processed_at=volume_model.last_processed_at,
         status=volume_model.status,
         status_message=volume_model.status_message,
         deleted=volume_model.deleted,

dstack/_internal/server/settings.py

@@ -93,6 +93,9 @@ DEFAULT_PROJECT_NAME = "main"
 
 SENTRY_DSN = os.getenv("DSTACK_SENTRY_DSN")
 SENTRY_TRACES_SAMPLE_RATE = float(os.getenv("DSTACK_SENTRY_TRACES_SAMPLE_RATE", 0.1))
+SENTRY_TRACES_BACKGROUND_SAMPLE_RATE = float(
+    os.getenv("DSTACK_SENTRY_TRACES_BACKGROUND_SAMPLE_RATE", 0.01)
+)
 SENTRY_PROFILES_SAMPLE_RATE = float(os.getenv("DSTACK_SENTRY_PROFILES_SAMPLE_RATE", 0))
 
 DEFAULT_CREDS_DISABLED = os.getenv("DSTACK_DEFAULT_CREDS_DISABLED") is not None

dstack/_internal/server/testing/common.py

@@ -68,6 +68,7 @@ from dstack._internal.core.models.runs import (
     Requirements,
     RunSpec,
     RunStatus,
+    RunTerminationReason,
 )
 from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import (
@@ -282,12 +283,15 @@ async def create_run(
     user: UserModel,
     run_name: str = "test-run",
     status: RunStatus = RunStatus.SUBMITTED,
+    termination_reason: Optional[RunTerminationReason] = None,
     submitted_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
     run_spec: Optional[RunSpec] = None,
     run_id: Optional[UUID] = None,
     deleted: bool = False,
     priority: int = 0,
     deployment_num: int = 0,
+    resubmission_attempt: int = 0,
+    next_triggered_at: Optional[datetime] = None,
 ) -> RunModel:
     if run_spec is None:
         run_spec = get_run_spec(
@@ -305,12 +309,15 @@ async def create_run(
         submitted_at=submitted_at,
         run_name=run_name,
         status=status,
+        termination_reason=termination_reason,
         run_spec=run_spec.json(),
         last_processed_at=submitted_at,
         jobs=[],
         priority=priority,
         deployment_num=deployment_num,
         desired_replica_count=1,
+        resubmission_attempt=resubmission_attempt,
+        next_triggered_at=next_triggered_at,
     )
     session.add(run)
     await session.commit()

dstack/_internal/server/utils/sentry_utils.py

@@ -0,0 +1,12 @@
+import functools
+
+import sentry_sdk
+
+
+def instrument_background_task(f):
+    @functools.wraps(f)
+    async def wrapper(*args, **kwargs):
+        with sentry_sdk.start_transaction(name=f"background.{f.__name__}"):
+            return await f(*args, **kwargs)
+
+    return wrapper

dstack/_internal/utils/cron.py

@@ -0,0 +1,5 @@
+from apscheduler.triggers.cron import CronTrigger
+
+
+def validate_cron(cron_expr: str):
+    CronTrigger.from_crontab(cron_expr)

dstack/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.20"
+__version__ = "0.19.21"
 __is_release__ = True
 base_image = "0.10"
 base_image_ubuntu_version = "22.04"

{dstack-0.19.20.dist-info → dstack-0.19.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.20
+Version: 0.19.21
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -15,6 +15,7 @@ Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9
+Requires-Dist: apscheduler<4
 Requires-Dist: argcomplete>=3.5.0
 Requires-Dist: cachetools
 Requires-Dist: cryptography
@@ -46,7 +47,6 @@ Requires-Dist: aiorwlock; extra == 'all'
 Requires-Dist: aiosqlite; extra == 'all'
 Requires-Dist: alembic-postgresql-enum; extra == 'all'
 Requires-Dist: alembic>=1.10.2; extra == 'all'
-Requires-Dist: apscheduler<4; extra == 'all'
 Requires-Dist: asyncpg; extra == 'all'
 Requires-Dist: azure-identity>=1.12.0; extra == 'all'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'all'
@@ -92,7 +92,6 @@ Requires-Dist: aiorwlock; extra == 'aws'
 Requires-Dist: aiosqlite; extra == 'aws'
 Requires-Dist: alembic-postgresql-enum; extra == 'aws'
 Requires-Dist: alembic>=1.10.2; extra == 'aws'
-Requires-Dist: apscheduler<4; extra == 'aws'
 Requires-Dist: asyncpg; extra == 'aws'
 Requires-Dist: backports-entry-points-selectable; extra == 'aws'
 Requires-Dist: boto3>=1.38.13; extra == 'aws'
@@ -117,7 +116,6 @@ Requires-Dist: aiorwlock; extra == 'azure'
 Requires-Dist: aiosqlite; extra == 'azure'
 Requires-Dist: alembic-postgresql-enum; extra == 'azure'
 Requires-Dist: alembic>=1.10.2; extra == 'azure'
-Requires-Dist: apscheduler<4; extra == 'azure'
 Requires-Dist: asyncpg; extra == 'azure'
 Requires-Dist: azure-identity>=1.12.0; extra == 'azure'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'azure'
@@ -147,7 +145,6 @@ Requires-Dist: aiorwlock; extra == 'datacrunch'
 Requires-Dist: aiosqlite; extra == 'datacrunch'
 Requires-Dist: alembic-postgresql-enum; extra == 'datacrunch'
 Requires-Dist: alembic>=1.10.2; extra == 'datacrunch'
-Requires-Dist: apscheduler<4; extra == 'datacrunch'
 Requires-Dist: asyncpg; extra == 'datacrunch'
 Requires-Dist: backports-entry-points-selectable; extra == 'datacrunch'
 Requires-Dist: datacrunch; extra == 'datacrunch'
@@ -179,7 +176,6 @@ Requires-Dist: aiorwlock; extra == 'gcp'
 Requires-Dist: aiosqlite; extra == 'gcp'
 Requires-Dist: alembic-postgresql-enum; extra == 'gcp'
 Requires-Dist: alembic>=1.10.2; extra == 'gcp'
-Requires-Dist: apscheduler<4; extra == 'gcp'
 Requires-Dist: asyncpg; extra == 'gcp'
 Requires-Dist: backports-entry-points-selectable; extra == 'gcp'
 Requires-Dist: docker>=6.0.0; extra == 'gcp'
@@ -209,7 +205,6 @@ Requires-Dist: aiorwlock; extra == 'kubernetes'
 Requires-Dist: aiosqlite; extra == 'kubernetes'
 Requires-Dist: alembic-postgresql-enum; extra == 'kubernetes'
 Requires-Dist: alembic>=1.10.2; extra == 'kubernetes'
-Requires-Dist: apscheduler<4; extra == 'kubernetes'
 Requires-Dist: asyncpg; extra == 'kubernetes'
 Requires-Dist: backports-entry-points-selectable; extra == 'kubernetes'
 Requires-Dist: docker>=6.0.0; extra == 'kubernetes'
@@ -233,7 +228,6 @@ Requires-Dist: aiorwlock; extra == 'lambda'
 Requires-Dist: aiosqlite; extra == 'lambda'
 Requires-Dist: alembic-postgresql-enum; extra == 'lambda'
 Requires-Dist: alembic>=1.10.2; extra == 'lambda'
-Requires-Dist: apscheduler<4; extra == 'lambda'
 Requires-Dist: asyncpg; extra == 'lambda'
 Requires-Dist: backports-entry-points-selectable; extra == 'lambda'
 Requires-Dist: boto3>=1.38.13; extra == 'lambda'
@@ -258,7 +252,6 @@ Requires-Dist: aiorwlock; extra == 'nebius'
 Requires-Dist: aiosqlite; extra == 'nebius'
 Requires-Dist: alembic-postgresql-enum; extra == 'nebius'
 Requires-Dist: alembic>=1.10.2; extra == 'nebius'
-Requires-Dist: apscheduler<4; extra == 'nebius'
 Requires-Dist: asyncpg; extra == 'nebius'
 Requires-Dist: backports-entry-points-selectable; extra == 'nebius'
 Requires-Dist: docker>=6.0.0; extra == 'nebius'
@@ -282,7 +275,6 @@ Requires-Dist: aiorwlock; extra == 'oci'
 Requires-Dist: aiosqlite; extra == 'oci'
 Requires-Dist: alembic-postgresql-enum; extra == 'oci'
 Requires-Dist: alembic>=1.10.2; extra == 'oci'
-Requires-Dist: apscheduler<4; extra == 'oci'
 Requires-Dist: asyncpg; extra == 'oci'
 Requires-Dist: backports-entry-points-selectable; extra == 'oci'
 Requires-Dist: cryptography>=44.0.3; extra == 'oci'
@@ -308,7 +300,6 @@ Requires-Dist: aiorwlock; extra == 'server'
 Requires-Dist: aiosqlite; extra == 'server'
 Requires-Dist: alembic-postgresql-enum; extra == 'server'
 Requires-Dist: alembic>=1.10.2; extra == 'server'
-Requires-Dist: apscheduler<4; extra == 'server'
 Requires-Dist: asyncpg; extra == 'server'
 Requires-Dist: backports-entry-points-selectable; extra == 'server'
 Requires-Dist: docker>=6.0.0; extra == 'server'

{dstack-0.19.20.dist-info → dstack-0.19.21.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=axeYDWu1KOu0e1Fgcergs4FNRuVFkVwRvrk2-_xOyww,102
+dstack/version.py,sha256=69F2XTd8pmd4oG2eNHMHCrCE3WDUqAhO21ECqYm3wnA,102
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=mlqH4Jwd-Qvxwt869dYXy4snQ0naUsHOwXiI0WSMojA,1246
@@ -47,8 +47,9 @@ dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14r
 dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/consts.py,sha256=c1Yd5UY6Qx7KeuYgloXWncWhMsYj6TqwlElda7NtB98,254
 dstack/_internal/core/errors.py,sha256=VTEmJ6C0zWL3fUL8ObRjB4rNL7_NxwEuWf0hZs6orWo,3377
-dstack/_internal/core/backends/__init__.py,sha256=MOZt5mNbcaAaamjLuZKUk9fJCUwk7EAMn8sbE02NwD0,2765
+dstack/_internal/core/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/configurators.py,sha256=7LFad72b6zdzzDRaFUaQrJLf6PxGRsxR3tfyn15rQbA,3976
+dstack/_internal/core/backends/features.py,sha256=cswqwmZYD8g041V_m-ovHbJRkB2_vni1DiGJpKD8M_g,2764
 dstack/_internal/core/backends/models.py,sha256=byP6YFXQHbkOyuX8MxwRvPfjtk87o9oCefR8U-lvsAI,4319
 dstack/_internal/core/backends/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/aws/auth.py,sha256=BAe3HAedoAO42Fbv7YhpODKWlyil8M0PuRJsL-_u_kw,978
@@ -162,10 +163,10 @@ dstack/_internal/core/backends/vultr/compute.py,sha256=-6Mw7ADW6HaqzK34I3T5RVhre
 dstack/_internal/core/backends/vultr/configurator.py,sha256=YdBeMveKaUDiLESTG3AzpSpQRzevUGpdnCGJx4tUR7g,2190
 dstack/_internal/core/backends/vultr/models.py,sha256=TkqK_lW6w5k_7wsoFiKDb6NXuTrHL6ypVm59FoyaSFY,928
 dstack/_internal/core/compatibility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/core/compatibility/fleets.py,sha256=TXkJ9kr8_dtPHzlVe6-JsZ1OJC2rAS92xj_cHjgGeAQ,2930
+dstack/_internal/core/compatibility/fleets.py,sha256=ijUNkEiamJ29jAszjBeT1jrsGygDFCo1zEyTa7xUuHw,3004
 dstack/_internal/core/compatibility/gateways.py,sha256=4h_lfpN9KJFyLTFexq-wlu74Rwpk0anV67v38aJ-SnI,1463
 dstack/_internal/core/compatibility/logs.py,sha256=keXt3OFKR0CjD_XMsetzRu8yQGCz7CWBwycuP267L_Q,629
-dstack/_internal/core/compatibility/runs.py,sha256=mcSjlWLr8LgxbK5LX7lNGwg6FVuRAEtzZ8ilxjdsedo,7740
+dstack/_internal/core/compatibility/runs.py,sha256=g1aIGTxLkXDh5kmnyfUzr9eQtRq65hhUhVr8PrmSc3o,7927
 dstack/_internal/core/compatibility/volumes.py,sha256=ofjpVusuc-pq285bGrIh8PAqu0QlAd6NQgU3gfJQIc0,1546
 dstack/_internal/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/models/common.py,sha256=RI03rSaxhSigIpqtWDeBZ9_yX6R0-6_wC1bMGqYsTZE,3855
@@ -179,7 +180,7 @@ dstack/_internal/core/models/instances.py,sha256=Vwh-D3WB_NBdG_1BPHBEaX60n4MtYks
 dstack/_internal/core/models/logs.py,sha256=vCMJFnebmTk5chodUQvghsyT1Funir1-InAcDHr-M68,483
 dstack/_internal/core/models/metrics.py,sha256=Xb8hCXUL-ncQ3PMsErIUAJTe9gwh5jyrQ4UQoZbibsc,269
 dstack/_internal/core/models/placement.py,sha256=WJVq5ENJykyRarQzL2EeYQag_9_jV7VSAtR_xoFvPVM,720
-dstack/_internal/core/models/profiles.py,sha256=V3O5lS4_V2H_aKTgnE_A-YVtxH9BqwnhNuMsHksjO7s,11798
+dstack/_internal/core/models/profiles.py,sha256=M9PXPaqQsOSOy32OHWwmWUwpdHR-8Ue9tEPJRSdLjKM,12866
 dstack/_internal/core/models/projects.py,sha256=HmmlrvUSSEjfGfX7X5tmGJ-CbKeyGK6shHD5OghDNks,671
 dstack/_internal/core/models/resources.py,sha256=mUqFwLazN2fUHV3ZOoi1_ZtmtjNUsIqmXYjviM_BERU,14370
 dstack/_internal/core/models/runs.py,sha256=8uvtQInQv6PbKQwCYfLk9UaQAO02Zj0gicIBrCdCvGY,20972
@@ -262,27 +263,27 @@ dstack/_internal/proxy/lib/testing/auth.py,sha256=-firWTnis9Eogoi58BURv1S-te4Hf9
 dstack/_internal/proxy/lib/testing/common.py,sha256=5e2VYboMqjBnUxkvidaWLoQ-uaBGh_hnURb_VJc38q4,1518
 dstack/_internal/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/alembic.ini,sha256=rTauBJC8Jxlw76KUygW061OFAGV7GhwAStinlaxoYGY,3166
-dstack/_internal/server/app.py,sha256=_ZtbHissUnBGw7icQWvrOCJuNmAFpN4vNpZQMYpOMog,14299
+dstack/_internal/server/app.py,sha256=W8vmHhWabKeiecHWjiHBxgyBjIka67AHC2PeZwG1cFU,14847
 dstack/_internal/server/db.py,sha256=WjuqmjG3QAZmSMCeUaJ_ynbowlHuNAvYCZO649cTPHc,3210
 dstack/_internal/server/deps.py,sha256=31e8SU_ogPJWHIDLkgl7cuC_5V91xbJoLyAj17VanfM,670
 dstack/_internal/server/main.py,sha256=kztKhCYNoHSDyJJQScWfZXE0naNleJOCQULW6dd8SGw,109
-dstack/_internal/server/models.py,sha256=iDpyj9DwgETuqoxPSPXfdL_wgpxFZhlDhcnPE2u13Zk,31368
-dstack/_internal/server/settings.py,sha256=NAvLCUsFOfI50ZLyYOCy6CRZffpdi0dk7vc7DKVMhPk,5148
-dstack/_internal/server/background/__init__.py,sha256=e-644wIdhtGjQM2ylI0jXfMjTKhDP-J6rHN0imgyj0o,5393
+dstack/_internal/server/models.py,sha256=UR0ouKSKimdYs6XwkCbZ-7ZU6r9mzI7DA5sTP561_PQ,31559
+dstack/_internal/server/settings.py,sha256=5LIHHmVHEj4CST_eOYnrjKGzMv-CTltZICSqnxu6jo8,5263
+dstack/_internal/server/background/__init__.py,sha256=ApoAca1NNtP4xu6Dv_mQkkfe2CknmWXumFIpzdhGe7I,5312
 dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
-dstack/_internal/server/background/tasks/process_fleets.py,sha256=i_94wf8rOBFnBjOF0kW2by6B623o-y2N_QtIbeTxDjQ,3505
-dstack/_internal/server/background/tasks/process_gateways.py,sha256=X04Yilmw2CHPStOyaSyhZ78D1cPVEqTc7WTEFgl-OPw,8319
-dstack/_internal/server/background/tasks/process_idle_volumes.py,sha256=Ej0NOoM0PIwDK6Q3Zd-Rl7FMCHI_LW74QQLczCrTAuc,5162
-dstack/_internal/server/background/tasks/process_instances.py,sha256=jZ5A_3BVg_15WnpKkoPWqh65IHzRoCIS1EsFgei-UmM,43347
-dstack/_internal/server/background/tasks/process_metrics.py,sha256=aJPHnmKQE19JVxJVwGFyyK7buUcbp9Smw9CkFO2DPAk,6135
-dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FylKGqz7RMBduJNlfuaGdgiPb6oneU49pdqvyinuV8Y,4167
-dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=9VoGFqdiXcVkCi_NV1VqQdqllDkB0bHIqOHKMIZK1Fg,5183
-dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=Nz9LgGTrAIkHDHpeTySy-xukfcdXtVYQAcQLrFLNRfo,43773
-dstack/_internal/server/background/tasks/process_runs.py,sha256=JisPDw0-5Uf3JD2Cb3mTXDjwbABJ5cj-EeLYgk7d74s,22608
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=Nmb7zRcoX9MQ6xw1bBTl8zRdo4smGPlWsfNnXK5HMs0,28853
-dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=j7Ph2tgCHlUYHU9-ONPSQCJ8qLdWrN2rb2l2nCYukzg,3896
-dstack/_internal/server/background/tasks/process_volumes.py,sha256=Svvi2USV8S_B6OfKg3hWSXjSanDmAfL1pQYe9EnqTNk,5134
+dstack/_internal/server/background/tasks/process_fleets.py,sha256=PInDWAcD7c-tSXGb4WzOBmJM4TIoD3Xy6C84kj6p1Zk,3703
+dstack/_internal/server/background/tasks/process_gateways.py,sha256=Ee8uGE4rKPP7U0I9OpgvIRfO3sNkpp_tdFW0Gh4XYdE,8259
+dstack/_internal/server/background/tasks/process_idle_volumes.py,sha256=mqnl8wvWaKTYvJMbgFJbOP-bZMRQG2vrhUnaNcyBldE,5223
+dstack/_internal/server/background/tasks/process_instances.py,sha256=-SKtzDK2YPM9pkTOtbU8mEeREflYt1G5lwXXRUx9Ajg,43848
+dstack/_internal/server/background/tasks/process_metrics.py,sha256=yKXe9J7m3oleK0C-oGJaYNkcPT8kqkz0nw-A7xqYbjE,6390
+dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=lgYIzjHG9EITK31yG6uQjlIcSwW5jsP9ZOBBZqW_eNs,4263
+dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=_UZm37FVV4rhdd0So7HtcKbIgrSdAr5Vx-Uen_xizec,5459
+dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=r2VoyJlvIgXzeyAt_bhh4FWbAH5WleuWuiZmbO5hepw,43784
+dstack/_internal/server/background/tasks/process_runs.py,sha256=p2YIjKobnObEjeWegdfwTNAaoe_9eYxAN7gmis6wqKI,24448
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=9jIB7yMC211yhQLFVtlmw__jGjb7w-7X1fZZ2wa4f4U,28913
+dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=4civvAaLPT1R-PopVNkcTNnbn_Na7fIhFL8szYqLHUg,4074
+dstack/_internal/server/background/tasks/process_volumes.py,sha256=7qNGX3yRoVmx2zUwJ9iTU124lGGBjKL3E8aEwKFX9eg,5208
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/migrations/env.py,sha256=RuhcN-CST86VzMatjqBMdYcG0fGGMZl_o1YgZTL-WDw,2831
 dstack/_internal/server/migrations/script.py.mako,sha256=Xi-IcoSN51tKWbAq1-nNVX7j6_89mf9AuX9DjnqUO_o,574
@@ -307,6 +308,7 @@ dstack/_internal/server/migrations/versions/3dbdce90d0e0_fix_code_uq_constraint.
 dstack/_internal/server/migrations/versions/48ad3ecbaea2_do_not_delete_projects_and_runs.py,sha256=FaVmRi9ocsygGEf5j2XDPaYbdgGl98ykycpFul1GdMw,1453
 dstack/_internal/server/migrations/versions/4ae1a5b0e7f1_add_run_list_index.py,sha256=qpzzdCpETXGCVCyhMDFoUviPBFrz585Kg5znYTffha8,862
 dstack/_internal/server/migrations/versions/4b4319398164_introduce_runs_processing.py,sha256=XUyqSYoeTYpt4wu_aFeVY-NgDSx6ZAUtTUiTHNPitwY,4938
+dstack/_internal/server/migrations/versions/50dd7ea98639_index_status_columns.py,sha256=X0gf8KqYfCfcZyq8tZj5mTSGKSeAZv3-teIemy2KlPY,1944
 dstack/_internal/server/migrations/versions/51d45659d574_add_instancemodel_blocks_fields.py,sha256=RpGbj86Omt-Rme5D3q_CIRWlAScjpILcgXS4jszceLY,1161
 dstack/_internal/server/migrations/versions/54a77e19c64c_add_manager_project_role.py,sha256=D3QgOShGrnWd-5SqIj8C6xQL2JwoRRB-b6Sm6eM_hA4,2056
 dstack/_internal/server/migrations/versions/555138b1f77f_change_instancemodel_for_asynchronous_.py,sha256=WpwO5OHPUfjNg-LV6A53wPeKXiwb_saQvOWL7PGLCu8,2782
@@ -356,6 +358,7 @@ dstack/_internal/server/migrations/versions/dfffd6a1165c_add_fields_for_gateways
 dstack/_internal/server/migrations/versions/e3b7db07727f_add_gatewaycomputemodel_app_updated_at.py,sha256=XAhYoza5xvG6ALFtNJn2H0LK8MMTDoAAV40U0opm1bg,1860
 dstack/_internal/server/migrations/versions/e6391ca6a264_separate_gateways_from_compute.py,sha256=efWiMgcv9LCrimVC52dvA_sH1nfxtNm5Ywadr_pbW7s,2612
 dstack/_internal/server/migrations/versions/ea60480f82bb_add_membermodel_member_num.py,sha256=jdUjThL8ELQAiFtODts2pxjo62fn3WEd8w50jp7-e44,825
+dstack/_internal/server/migrations/versions/ec02a26a256c_add_runmodel_next_triggered_at.py,sha256=YeEeTHYqh5i5ycO3ExZhCWc16iC--JuRtVcDRu2RJ9Q,962
 dstack/_internal/server/migrations/versions/ed0ca30e13bb_migrate_instancestatus_provisioning.py,sha256=YhXQ9ApeMB8grDjNcbnXQKHYPlIJB7QlvE1R1nM5_6k,689
 dstack/_internal/server/migrations/versions/fe72c4de8376_add_gateways.py,sha256=1-0e060NDpgYQhoKDmHWednANk1cZzMdVns7sEo5Lwo,2730
 dstack/_internal/server/migrations/versions/ffa99edd1988_add_jobterminationreason_max_duration_.py,sha256=1wBIisgN25doZfAErnqyshoSSzoDHPt4nZfC2MmSWak,2324
@@ -396,22 +399,22 @@ dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 dstack/_internal/server/services/config.py,sha256=yo8njslwfS7_blhbPPhOtzCMyg8N_mmFSw5aPvirSzw,10691
 dstack/_internal/server/services/docker.py,sha256=zAvjFHxIP03Td92NzbGEScz0piLjloE60tQ7vz0AeCA,5328
 dstack/_internal/server/services/files.py,sha256=LJYFRtIJwB-QL-FhIu1408oqVVfa2lPZZlaTuZMo6pw,2847
-dstack/_internal/server/services/fleets.py,sha256=x4zOJ2UoGJmn4LA6zRSLG05lfREEL0UtsOWD9S8rsFY,37024
-dstack/_internal/server/services/instances.py,sha256=bJlMFo5DTHHL2LM25CuzJgx9eNXKbeZ2pjX0Waorq8Q,18969
+dstack/_internal/server/services/fleets.py,sha256=ugbITHV_KRkFBFKkRIaaRDUcKFzTYEYmtrZsmoa64b8,37146
+dstack/_internal/server/services/instances.py,sha256=ox1J7OI0U-FYis8HyKgWh_DTvyxV6mmo_s6lgQwkzIk,18737
 dstack/_internal/server/services/locking.py,sha256=4y_k8U7X71_lExdaXDSSEfizhfydBXOZKe64zo-1DX8,5224
 dstack/_internal/server/services/logging.py,sha256=h5Cy22TJi5IvbKXE_kVZYXQy0XJrHK85JV4-GCuWA1o,543
 dstack/_internal/server/services/metrics.py,sha256=jKLy1jSCVR_crqVu_CmsOMbvMkucWBandI0SS1LDeRQ,6896
-dstack/_internal/server/services/offers.py,sha256=4hhqLzmo14_IY84FSDG59Sib1DrmTAOk-_q8M4aYoLU,7737
+dstack/_internal/server/services/offers.py,sha256=ACfTGW6Nx7RoUwJObsSH22lI_BDDi2ux55PXWZN6els,7746
 dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
 dstack/_internal/server/services/placement.py,sha256=U-7z4kJ6ZyrCXzkDychWoOhniz19ooFwiNI2InoTLls,2420
 dstack/_internal/server/services/plugins.py,sha256=KgBqBUqxjQirRcYNuXVetE4OQj0EPkdC7ccbrNV1WQE,3825
-dstack/_internal/server/services/projects.py,sha256=UMcqUvT4n02iTgvPSqzlPx0CoQIoi5bWsmrN78747_k,22346
+dstack/_internal/server/services/projects.py,sha256=Gk4E4srbNPDG5AR4Ec8OmajaVrmupHDsg6dk8ybb6nQ,23001
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
 dstack/_internal/server/services/resources.py,sha256=VRFOih_cMJdc0c2m9nSGsX8vWAJQV3M6N87aqS_JXfw,699
-dstack/_internal/server/services/runs.py,sha256=SN-zDWXwWpfxA9zuHEaN2aokm6_Gu2SufBS4gMmxqrQ,43851
+dstack/_internal/server/services/runs.py,sha256=Ncqun9CxD2WBwPRxOY8xIKdb9t_VqytP4iuZyNj9GV4,45373
 dstack/_internal/server/services/secrets.py,sha256=oVz8mv4pQv8onaDdPKft2qJYq3Xnh8-uskoMxBuyfpE,5567
-dstack/_internal/server/services/users.py,sha256=W-5xL7zsHNjeG7BBK54RWGvIrBOrw-FF0NcG_z9qhoE,7466
-dstack/_internal/server/services/volumes.py,sha256=67yhlPfCSHmtCJecOUy_OlubwJ3lpY8kA2Dn9DOien4,16922
+dstack/_internal/server/services/users.py,sha256=L-COF4RjtwYSfnkGQefrSiEU0haTv0o1P2OuUdvMK8E,7378
+dstack/_internal/server/services/volumes.py,sha256=DBGIUsysAO7sQ6vb4mjhzO9XLdyQvtNujucxhGnkSsU,16682
 dstack/_internal/server/services/backends/__init__.py,sha256=UzZ-JqaUEheaE3fwNfnRJ47yK2nO23mTT3BwcYDooP4,13147
 dstack/_internal/server/services/backends/handlers.py,sha256=j-MhBxrpdepoDG7f2tApjFnE23RVO5I15-hxHyOWnew,3251
 dstack/_internal/server/services/encryption/__init__.py,sha256=3kCw_cxC3-Un1OIofdW5Gqsm0ZCXXTlGz09cULBx_uc,3155
@@ -419,11 +422,11 @@ dstack/_internal/server/services/encryption/keys/__init__.py,sha256=47DEQpj8HBSa
 dstack/_internal/server/services/encryption/keys/aes.py,sha256=2He1p2_Rg6hnCeLIGJo-Yfdsij7so_338oY49RXuL3Q,2276
 dstack/_internal/server/services/encryption/keys/base.py,sha256=mqumJiidoexUPoqxhQG6J_SpC1WGYwkdjKm1MUWnXo8,352
 dstack/_internal/server/services/encryption/keys/identity.py,sha256=ryb_YSV6u4c7W1OsVfEpzJvZCrR4zZYlzLw_GpjpD2Q,741
-dstack/_internal/server/services/gateways/__init__.py,sha256=xb6U_lZEnjw0clcIyEM0m47vn25sDM8mobtMC1PkqZo,22054
+dstack/_internal/server/services/gateways/__init__.py,sha256=eI98kRhV0eJuZPwRJLtSO4M4oLfFmMMNuDIEOsmVQzg,22132
 dstack/_internal/server/services/gateways/client.py,sha256=kgkBxfIIklVQh2Djk8_qlDVHHbnoTRvoJNEwNNxTcY8,7653
 dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBWeyj57nLPcLPNm316zu3jMyeWjA,5625
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
-dstack/_internal/server/services/jobs/__init__.py,sha256=SqAVnnHYP5vWlBNambHlRXfSD2MTPD6VNn5pW_dlBxg,27685
+dstack/_internal/server/services/jobs/__init__.py,sha256=TuQpdBm62lIXPiA81ZfdjjCn3JIXMuxuOsGLFMquZhI,27569
 dstack/_internal/server/services/jobs/configurators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/jobs/configurators/base.py,sha256=RldlN_CuPt7lQy_kaks1IwAzschGlVo8aZygwwWEjco,13622
 dstack/_internal/server/services/jobs/configurators/dev.py,sha256=i58_LYvX__gxLoCx7GwjNj26M67FjgAIcEWyl3f8mOU,2435
@@ -439,8 +442,8 @@ dstack/_internal/server/services/logs/base.py,sha256=7eCB6SPF9iWviv4vZ_34SvfM_Bv
 dstack/_internal/server/services/logs/filelog.py,sha256=ZpnrMWhWVOpZHnUCZEm2pkJRZYgWBzlasdcyPCb6v90,9811
 dstack/_internal/server/services/logs/gcp.py,sha256=Rfl5x1Slr4hyjdr0YPCugfHRC3aNL7D2LCiP4CvQB1s,6917
 dstack/_internal/server/services/prometheus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/prometheus/client_metrics.py,sha256=nGhvOHEoWONGAfbNh5gC23It7B46A6hfQ8mErUQIcYI,1541
-dstack/_internal/server/services/prometheus/custom_metrics.py,sha256=rokpp552qkaXZpCLJxOexA4xEBuIIxc2mxrJFkeE9ko,13635
+dstack/_internal/server/services/prometheus/client_metrics.py,sha256=BN38NpAdjJBPpoN_q5APHhxmGBd8PghJJpZ9Vi3OaZs,1745
+dstack/_internal/server/services/prometheus/custom_metrics.py,sha256=xjBuNAbJQtfsf5zATzh7IwFCjIGjulJ-RuaMcYHKR2k,13547
 dstack/_internal/server/services/proxy/__init__.py,sha256=aklmvGaGXISztQft-nH8R98WRU6x_L0Fx3RwVDhwt3g,85
 dstack/_internal/server/services/proxy/auth.py,sha256=AVhDCnk9KvxJ7Jsd8Xmotl_29V4lNuBw8mQNBhx90Ws,485
 dstack/_internal/server/services/proxy/deps.py,sha256=u4dHHKBlTgC8Fnnl3onErgE1HGKFwyZTwvq0uT6IdQ4,846
@@ -452,7 +455,7 @@ dstack/_internal/server/services/proxy/services/service_proxy.py,sha256=4JrSxHqh
 dstack/_internal/server/services/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/runner/client.py,sha256=m2chzrQmtcRiw2JUmdfWAbcQFGVKq4CiNEFZGE21WBw,15944
 dstack/_internal/server/services/runner/ssh.py,sha256=H-X0015ZPwYq5tc31ytFF1uNaUAr9itAsABI2oPJWrk,5017
-dstack/_internal/server/services/services/__init__.py,sha256=vHJ2L9KUUbHhk0yPO2UjTZFYBAYFwGlV4EJtNZ29Ktg,11723
+dstack/_internal/server/services/services/__init__.py,sha256=X5JEwtiuxyjs0v0SEOh1pUc1Yc_r6PSbqwUjOUZVH5E,11834
 dstack/_internal/server/services/services/autoscalers.py,sha256=wg3QgfdKcaBdbtyJJlAzeMlicGKQMuRGpMPZKyemK5U,4447
 dstack/_internal/server/services/services/options.py,sha256=BEpZbRY4PKFVkNNWxLtY--QISqBL1_aPPzZYLPpRgvM,2234
 dstack/_internal/server/services/storage/__init__.py,sha256=ZjlLdCQiclM7h_dKJM9HXjMJDKCFOAf3yI4iO6K3HHk,1384
@@ -566,14 +569,16 @@ dstack/_internal/server/statics/static/media/logo.f602feeb138844eda97c8cb6414614
 dstack/_internal/server/statics/static/media/okta.12f178e6873a1100965f2a4dbd18fcec.svg,sha256=KqFI05gQM135zC1plF1WBRF2F7CyKL7km97WKsZjAHI,319
 dstack/_internal/server/statics/static/media/theme.3994c817bb7dda191c1c9640dee0bf42.svg,sha256=ZxFFBVZWuRLqmWH4zhwGLNtKjOzHj-5MGJRunFAtu1I,561
 dstack/_internal/server/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/testing/common.py,sha256=I74o8M5b3mJ5bguCYp-CKTnGQEQHTcG18Xz9j3iIn3c,34106
+dstack/_internal/server/testing/common.py,sha256=ZpqosCSolTPCel1bFwgJ_3KWr2LfyoXCiJfGf_8o124,34423
 dstack/_internal/server/testing/conf.py,sha256=-zhujfFjTHNfQDOK-hBck32By11c_kC0OeinB3esQGg,1902
 dstack/_internal/server/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/utils/common.py,sha256=PbjXtqYy1taKXpyG5ys8cIrz9MXqc9dBAsR_9D1brrk,1414
 dstack/_internal/server/utils/logging.py,sha256=bxUS2uWG5snypNRfL0d5sMLCDytyOZac81PSQlb7_rs,1907
 dstack/_internal/server/utils/routers.py,sha256=WFjcIlJETY7SAQ1eHRxaZuAAUsWYT8S_ATLHQ6a_MBs,5498
+dstack/_internal/server/utils/sentry_utils.py,sha256=3vYP_CH4Q6Ju5BZ0js2Yw4Gg_gdD8XVIjNVUpOQYwX8,277
 dstack/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/utils/common.py,sha256=m2_lmWjIPTJvI8V2XifY2FUiqT_1KIUWfnLCbAjOvEw,9119
+dstack/_internal/utils/cron.py,sha256=_axJFBCuZwMXZN04YPSk_gMvrRv6gNiGqd5hITyIg9Y,127
 dstack/_internal/utils/crypto.py,sha256=2RTSyzePuwwqc1X2HO6lwcSFyZ2kujnqluoICQ2DLJQ,1462
 dstack/_internal/utils/dxf.py,sha256=wguK9s6-69kqSDZkxd1kFEr6VlH5ixvFRJxizyOuJ8I,3229
 dstack/_internal/utils/env.py,sha256=HRbIspHpKHh05fMZeV23-hrZoV6vVMuniefD08u6ey0,357
@@ -622,8 +627,8 @@ dstack/plugins/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 dstack/plugins/builtin/rest_plugin/__init__.py,sha256=lgTsq8Z6Km2F2UhPRChVB4vDM5ZpWtdk1iB1aa20ypA,440
 dstack/plugins/builtin/rest_plugin/_models.py,sha256=9hgVuU6OGSxidar88XhQnNo9izYWeQvVH45ciErv-Es,1910
 dstack/plugins/builtin/rest_plugin/_plugin.py,sha256=iv8TmQh959iEUZjCFTzmGnFW8fHukTVpPmD4iBzOfS4,5361
-dstack-0.19.20.dist-info/METADATA,sha256=nbhx4hEoW9nhdyAR3cGIFsx48cBX9kmFFL07bfaBtbw,20952
-dstack-0.19.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dstack-0.19.20.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.20.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.20.dist-info/RECORD,,
+dstack-0.19.21.dist-info/METADATA,sha256=ANt80iZwkabsiNy_6-3bJrwON0vBN7xOmRb7oImIu-M,20506
+dstack-0.19.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dstack-0.19.21.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.21.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.21.dist-info/RECORD,,