dstack 0.18.43__py3-none-any.whl → 0.19.0__py3-none-any.whl

dstack/_internal/core/models/volumes.py

@@ -86,9 +86,7 @@ class VolumeAttachment(CoreModel):
 class Volume(CoreModel):
     id: uuid.UUID
     name: str
-    # Default user to "" for client backward compatibility (old 0.18 servers).
-    # TODO: Remove in 0.19
-    user: str = ""
+    user: str
     project_name: str
     configuration: VolumeConfiguration
     external: bool

dstack/_internal/core/services/profiles.py

@@ -12,18 +12,7 @@ from dstack._internal.core.models.runs import Retry
 def get_retry(profile: Profile) -> Optional[Retry]:
     profile_retry = profile.retry
     if profile_retry is None:
-        # Handle retry_policy before retry was introduced
-        # TODO: Remove once retry_policy no longer supported
-        profile_retry_policy = profile.retry_policy
-        if profile_retry_policy is None:
-            return None
-        if not profile_retry_policy.retry:
-            return None
-        duration = profile_retry_policy.duration or DEFAULT_RETRY_DURATION
-        return Retry(
-            on_events=[RetryEvent.NO_CAPACITY, RetryEvent.INTERRUPTION, RetryEvent.ERROR],
-            duration=duration,
-        )
+        return None
     if isinstance(profile_retry, bool):
         if profile_retry:
             return Retry(
@@ -32,6 +21,12 @@ def get_retry(profile: Profile) -> Optional[Retry]:
             )
         return None
     profile_retry = profile_retry.copy()
+    if profile_retry.on_events is None:
+        profile_retry.on_events = [
+            RetryEvent.NO_CAPACITY,
+            RetryEvent.INTERRUPTION,
+            RetryEvent.ERROR,
+        ]
     if profile_retry.duration is None:
         profile_retry.duration = DEFAULT_RETRY_DURATION
     return Retry.parse_obj(profile_retry)
@@ -42,10 +37,6 @@ def get_termination(
 ) -> Tuple[TerminationPolicy, int]:
     termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE
     termination_idle_time = default_termination_idle_time
-    if profile.termination_policy is not None:
-        termination_policy = profile.termination_policy
-    if profile.termination_idle_time is not None:
-        termination_idle_time = profile.termination_idle_time
     if profile.idle_duration is not None and int(profile.idle_duration) < 0:
         termination_policy = TerminationPolicy.DONT_DESTROY
     elif profile.idle_duration is not None:

dstack/_internal/core/services/repos.py

@@ -10,7 +10,6 @@ from git.exc import GitCommandError
 from dstack._internal.core.errors import DstackError
 from dstack._internal.core.models.config import RepoConfig
 from dstack._internal.core.models.repos import LocalRepo, RemoteRepo, RemoteRepoCreds
-from dstack._internal.core.models.repos.base import RepoProtocol
 from dstack._internal.core.models.repos.remote import GitRepoURL
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
@@ -41,7 +40,6 @@ def get_local_repo_credentials(
     r = requests.get(f"{url.as_https()}/info/refs?service=git-upload-pack", timeout=10)
     if r.status_code == 200:
         return RemoteRepoCreds(
-            protocol=RepoProtocol.HTTPS,
             clone_url=url.as_https(),
             private_key=None,
             oauth_token=None,
@@ -93,7 +91,6 @@ def check_remote_repo_credentials_https(url: GitRepoURL, oauth_token: str) -> Re
             f"Can't access `{url.as_https()}` using the `{masked}` token"
         )
     return RemoteRepoCreds(
-        protocol=RepoProtocol.HTTPS,
         clone_url=url.as_https(),
         oauth_token=oauth_token,
         private_key=None,
@@ -123,7 +120,6 @@ def check_remote_repo_credentials_ssh(url: GitRepoURL, identity_file: PathLike)
         )
 
     return RemoteRepoCreds(
-        protocol=RepoProtocol.SSH,
         clone_url=url.as_ssh(),
         private_key=private_key,
         oauth_token=None,

dstack/_internal/server/app.py

@@ -27,8 +27,8 @@ from dstack._internal.server.routers import (
     instances,
     logs,
     metrics,
-    pools,
     projects,
+    prometheus,
     repos,
     runs,
     secrets,
@@ -183,8 +183,7 @@ def register_routes(app: FastAPI, ui: bool = True):
     app.include_router(volumes.project_router)
     app.include_router(service_proxy.router, prefix="/proxy/services", tags=["service-proxy"])
     app.include_router(model_proxy.router, prefix="/proxy/models", tags=["model-proxy"])
-    app.include_router(pools.root_router)
-    app.include_router(pools.router)
+    app.include_router(prometheus.router)
 
     @app.exception_handler(ForbiddenError)
     async def forbidden_error_handler(request: Request, exc: ForbiddenError):
@@ -252,7 +251,11 @@ def register_routes(app: FastAPI, ui: bool = True):
 
         @app.exception_handler(404)
         async def custom_http_exception_handler(request, exc):
-            if request.url.path.startswith("/api") or _is_proxy_request(request):
+            if (
+                request.url.path.startswith("/api")
+                or _is_proxy_request(request)
+                or _is_prometheus_request(request)
+            ):
                 return JSONResponse(
                     {"detail": exc.detail},
                     status_code=status.HTTP_404_NOT_FOUND,
@@ -283,6 +286,10 @@ def _is_proxy_request(request: Request) -> bool:
     ) and referrer.path.startswith("/proxy")
 
 
+def _is_prometheus_request(request: Request) -> bool:
+    return request.url.path.startswith("/metrics")
+
+
 def _print_dstack_logo():
     console.print(
         """[purple]╱╱╭╮╱╱╭╮╱╱╱╱╱╱╭╮

dstack/_internal/server/background/__init__.py

@@ -1,6 +1,7 @@
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.interval import IntervalTrigger
 
+from dstack._internal.server import settings
 from dstack._internal.server.background.tasks.process_fleets import process_fleets
 from dstack._internal.server.background.tasks.process_gateways import (
     process_gateways_connections,
@@ -16,6 +17,10 @@ from dstack._internal.server.background.tasks.process_metrics import (
 from dstack._internal.server.background.tasks.process_placement_groups import (
     process_placement_groups,
 )
+from dstack._internal.server.background.tasks.process_prometheus_metrics import (
+    collect_prometheus_metrics,
+    delete_prometheus_metrics,
+)
 from dstack._internal.server.background.tasks.process_running_jobs import process_running_jobs
 from dstack._internal.server.background.tasks.process_runs import process_runs
 from dstack._internal.server.background.tasks.process_submitted_jobs import process_submitted_jobs
@@ -43,6 +48,11 @@ def start_background_tasks() -> AsyncIOScheduler:
     # * 150 active instances with up to 2 minutes processing latency
     _scheduler.add_job(collect_metrics, IntervalTrigger(seconds=10), max_instances=1)
     _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1)
+    if settings.ENABLE_PROMETHEUS_METRICS:
+        _scheduler.add_job(
+            collect_prometheus_metrics, IntervalTrigger(seconds=10), max_instances=1
+        )
+        _scheduler.add_job(delete_prometheus_metrics, IntervalTrigger(minutes=5), max_instances=1)
     # process_submitted_jobs and process_instances max processing rate is 75 jobs(instances) per minute.
     _scheduler.add_job(
         process_submitted_jobs,

dstack/_internal/server/background/tasks/process_gateways.py

@@ -54,17 +54,13 @@ async def process_submitted_gateways():
 
 
 async def _remove_inactive_connections():
-    connections = await gateway_connections_pool.all()
-    ip_addresses = [c.ip_address for c in connections]
     async with get_session_ctx() as session:
         res = await session.execute(
-            select(GatewayComputeModel).where(
-                GatewayComputeModel.ip_address.in_(ip_addresses),
-                GatewayComputeModel.active == False,
-            )
+            select(GatewayComputeModel.ip_address).where(GatewayComputeModel.active == True)
         )
-        removed_connections = res.scalars().all()
-        for conn in removed_connections:
+    active_connection_ips = set(res.scalars().all())
+    for conn in await gateway_connections_pool.all():
+        if conn.ip_address not in active_connection_ips:
             await gateway_connections_pool.remove(conn.ip_address)
 
 

dstack/_internal/server/background/tasks/process_instances.py

@@ -20,6 +20,8 @@ from dstack._internal.core.backends.base.compute import (
     DSTACK_RUNNER_BINARY_PATH,
     DSTACK_SHIM_BINARY_PATH,
     DSTACK_WORKING_DIR,
+    ComputeWithCreateInstanceSupport,
+    ComputeWithPlacementGroupSupport,
     get_shim_env,
     get_shim_pre_start_commands,
 )
@@ -76,19 +78,19 @@ from dstack._internal.server.services.fleets import (
     fleet_model_to_fleet,
     get_create_instance_offers,
 )
-from dstack._internal.server.services.locking import get_locker
-from dstack._internal.server.services.offers import is_divisible_into_blocks
-from dstack._internal.server.services.placement import (
-    get_fleet_placement_groups,
-    placement_group_model_to_placement_group,
-)
-from dstack._internal.server.services.pools import (
+from dstack._internal.server.services.instances import (
     get_instance_configuration,
     get_instance_profile,
     get_instance_provisioning_data,
     get_instance_requirements,
     get_instance_ssh_private_keys,
 )
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.offers import is_divisible_into_blocks
+from dstack._internal.server.services.placement import (
+    get_fleet_placement_groups,
+    placement_group_model_to_placement_group,
+)
 from dstack._internal.server.services.runner import client as runner_client
 from dstack._internal.server.services.runner.client import HealthStatus
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
@@ -530,12 +532,15 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
     for backend, instance_offer in offers:
         if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT:
             continue
+        compute = backend.compute()
+        assert isinstance(compute, ComputeWithCreateInstanceSupport)
         instance_offer = _get_instance_offer_for_instance(instance_offer, instance)
         if (
             instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT
             and instance.fleet
             and instance_configuration.placement_group_name
         ):
+            assert isinstance(compute, ComputeWithPlacementGroupSupport)
             placement_group_model = _create_placement_group_if_does_not_exist(
                 session=session,
                 fleet_model=instance.fleet,
@@ -546,7 +551,7 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             )
             if placement_group_model is not None:
                 placement_group = placement_group_model_to_placement_group(placement_group_model)
-                pgpd = await run_async(backend.compute().create_placement_group, placement_group)
+                pgpd = await run_async(compute.create_placement_group, placement_group)
                 placement_group_model.provisioning_data = pgpd.json()
                 session.add(placement_group_model)
                 placement_groups.append(placement_group)
@@ -559,7 +564,7 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         )
         try:
             job_provisioning_data = await run_async(
-                backend.compute().create_instance,
+                compute.create_instance,
                 instance_offer,
                 instance_configuration,
             )

dstack/_internal/server/background/tasks/process_metrics.py

@@ -11,8 +11,8 @@ from dstack._internal.server import settings
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import InstanceModel, JobMetricsPoint, JobModel
 from dstack._internal.server.schemas.runner import MetricsResponse
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
 from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
-from dstack._internal.server.services.pools import get_instance_ssh_private_keys
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
 from dstack._internal.utils.common import batched, get_current_datetime, get_or_error, run_async

dstack/_internal/server/background/tasks/process_placement_groups.py

@@ -5,6 +5,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
+from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
 from dstack._internal.core.errors import PlacementGroupInUseError
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import PlacementGroupModel, ProjectModel
@@ -28,6 +29,7 @@ async def process_placement_groups():
                     PlacementGroupModel.deleted == False,
                     PlacementGroupModel.id.not_in(lockset),
                 )
+                .order_by(PlacementGroupModel.id)  # take locks in order
                 .with_for_update(skip_locked=True)
             )
             placement_group_models = res.scalars().all()
@@ -80,8 +82,10 @@ async def _delete_placement_group(placement_group_model: PlacementGroupModel):
             "Failed to delete placement group %s. Backend not available.", placement_group.name
         )
         return
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithPlacementGroupSupport)
     try:
-        await run_async(backend.compute().delete_placement_group, placement_group)
+        await run_async(compute.delete_placement_group, placement_group)
     except PlacementGroupInUseError:
         logger.info(
             "Placement group %s is still in use. Skipping deletion for now.", placement_group.name

dstack/_internal/server/background/tasks/process_prometheus_metrics.py

@@ -0,0 +1,135 @@
+import uuid
+from datetime import datetime, timedelta
+from typing import Optional
+
+import sqlalchemy.exc
+from sqlalchemy import delete, or_, select, update
+from sqlalchemy.orm import joinedload
+
+from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT
+from dstack._internal.core.models.runs import JobStatus
+from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.models import InstanceModel, JobModel, JobPrometheusMetrics
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.server.services.runner import client
+from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
+from dstack._internal.server.utils.common import gather_map_async
+from dstack._internal.utils.common import batched, get_current_datetime, get_or_error, run_async
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+MAX_JOBS_FETCHED = 100
+BATCH_SIZE = 10
+MIN_COLLECT_INTERVAL_SECONDS = 9
+# 10 minutes should be more than enough to scrape metrics, and, in any case,
+# 10 minutes old metrics has little to no value
+METRICS_TTL_SECONDS = 600
+
+
+async def collect_prometheus_metrics():
+    now = get_current_datetime()
+    cutoff = now - timedelta(seconds=MIN_COLLECT_INTERVAL_SECONDS)
+    async with get_session_ctx() as session:
+        res = await session.execute(
+            select(JobModel)
+            .join(JobPrometheusMetrics, isouter=True)
+            .where(
+                JobModel.status.in_([JobStatus.RUNNING]),
+                or_(
+                    JobPrometheusMetrics.job_id.is_(None),
+                    JobPrometheusMetrics.collected_at < cutoff,
+                ),
+            )
+            .options(joinedload(JobModel.instance).joinedload(InstanceModel.project))
+            .order_by(JobModel.last_processed_at.asc())
+            .limit(MAX_JOBS_FETCHED)
+        )
+        job_models = res.unique().scalars().all()
+    for batch in batched(job_models, BATCH_SIZE):
+        await _collect_jobs_metrics(batch, now)
+
+
+async def delete_prometheus_metrics():
+    now = get_current_datetime()
+    cutoff = now - timedelta(seconds=METRICS_TTL_SECONDS)
+    async with get_session_ctx() as session:
+        await session.execute(
+            delete(JobPrometheusMetrics).where(JobPrometheusMetrics.collected_at < cutoff)
+        )
+        await session.commit()
+
+
+async def _collect_jobs_metrics(job_models: list[JobModel], collected_at: datetime):
+    results = await gather_map_async(job_models, _collect_job_metrics, return_exceptions=True)
+    async with get_session_ctx() as session:
+        for job_model, result in results:
+            if result is None:
+                continue
+            if isinstance(result, BaseException):
+                logger.error(
+                    "Failed to collect job %s Prometheus metrics: %r", job_model.job_name, result
+                )
+                continue
+            res = await session.execute(
+                update(JobPrometheusMetrics)
+                .where(JobPrometheusMetrics.job_id == job_model.id)
+                .values(
+                    collected_at=collected_at,
+                    text=result,
+                )
+                .returning(JobPrometheusMetrics)
+            )
+            metrics = res.scalar()
+            if metrics is None:
+                metrics = JobPrometheusMetrics(
+                    job_id=job_model.id,
+                    collected_at=collected_at,
+                    text=result,
+                )
+                try:
+                    async with session.begin_nested():
+                        session.add(metrics)
+                except sqlalchemy.exc.IntegrityError:
+                    # Concurrent server replica already committed, ignoring
+                    pass
+        await session.commit()
+
+
+async def _collect_job_metrics(job_model: JobModel) -> Optional[str]:
+    ssh_private_keys = get_instance_ssh_private_keys(get_or_error(job_model.instance))
+    jpd = get_job_provisioning_data(job_model)
+    jrd = get_job_runtime_data(job_model)
+    if jpd is None:
+        return None
+    try:
+        res = await run_async(
+            _pull_job_metrics,
+            ssh_private_keys,
+            jpd,
+            jrd,
+            job_model.id,
+        )
+    except Exception:
+        logger.exception("Failed to collect job %s Prometheus metrics", job_model.job_name)
+        return None
+
+    if isinstance(res, bool):
+        logger.warning(
+            "Failed to connect to job %s to collect Prometheus metrics", job_model.job_name
+        )
+        return None
+
+    if res is None:
+        # Either not supported by shim or exporter is not available
+        return None
+
+    return res
+
+
+@runner_ssh_tunnel(ports=[DSTACK_SHIM_HTTP_PORT], retries=1)
+def _pull_job_metrics(ports: dict[int, int], task_id: uuid.UUID) -> Optional[str]:
+    shim_client = client.ShimClient(port=ports[DSTACK_SHIM_HTTP_PORT])
+    return shim_client.get_task_metrics(task_id)

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -1,4 +1,6 @@
 import asyncio
+from collections.abc import Iterable
+from datetime import timedelta
 from typing import Dict, List, Optional
 
 from sqlalchemy import select
@@ -15,6 +17,7 @@ from dstack._internal.core.models.instances import (
     RemoteConnectionInfo,
     SSHConnectionParams,
 )
+from dstack._internal.core.models.metrics import Metric
 from dstack._internal.core.models.repos import RemoteRepoCreds
 from dstack._internal.core.models.runs import (
     ClusterInfo,
@@ -40,6 +43,7 @@ from dstack._internal.server.models import (
 from dstack._internal.server.schemas.runner import TaskStatus
 from dstack._internal.server.services import logs as logs_services
 from dstack._internal.server.services import services
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
 from dstack._internal.server.services.jobs import (
     find_job,
     get_job_attached_volumes,
@@ -48,7 +52,7 @@ from dstack._internal.server.services.jobs import (
 )
 from dstack._internal.server.services.locking import get_locker
 from dstack._internal.server.services.logging import fmt
-from dstack._internal.server.services.pools import get_instance_ssh_private_keys
+from dstack._internal.server.services.metrics import get_job_metrics
 from dstack._internal.server.services.repos import (
     get_code_model,
     get_repo_creds,
@@ -123,7 +127,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
     run_model = res.unique().scalar_one()
     repo_model = run_model.repo
     project = run_model.project
-    run = run_model_to_run(run_model)
+    run = run_model_to_run(run_model, include_sensitive=True)
     job_submission = job_model_to_job_submission(job_model)
     job_provisioning_data = job_submission.job_provisioning_data
     if job_provisioning_data is None:
@@ -343,6 +347,9 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
             job_model.status = JobStatus.TERMINATING
             job_model.termination_reason = JobTerminationReason.GATEWAY_ERROR
 
+    if job_model.status == JobStatus.RUNNING:
+        await _check_gpu_utilization(session, job_model, job)
+
     job_model.last_processed_at = common_utils.get_current_datetime()
     await session.commit()
 
@@ -646,27 +653,67 @@ def _terminate_if_inactivity_duration_exceeded(
     run_model: RunModel, job_model: JobModel, no_connections_secs: Optional[int]
 ) -> None:
     conf = RunSpec.__response__.parse_raw(run_model.run_spec).configuration
-    if is_core_model_instance(conf, DevEnvironmentConfiguration) and isinstance(
+    if not is_core_model_instance(conf, DevEnvironmentConfiguration) or not isinstance(
         conf.inactivity_duration, int
     ):
-        logger.debug("%s: no SSH connections for %s seconds", fmt(job_model), no_connections_secs)
-        job_model.inactivity_secs = no_connections_secs
-        if no_connections_secs is None:
-            # TODO(0.19 or earlier): make no_connections_secs required
-            job_model.status = JobStatus.TERMINATING
-            job_model.termination_reason = JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY
-            job_model.termination_reason_message = (
-                "The selected instance was created before dstack 0.18.41"
-                " and does not support inactivity_duration"
-            )
-        elif no_connections_secs >= conf.inactivity_duration:
-            job_model.status = JobStatus.TERMINATING
-            # TODO(0.19 or earlier): set JobTerminationReason.INACTIVITY_DURATION_EXCEEDED
-            job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
-            job_model.termination_reason_message = (
-                f"The job was inactive for {no_connections_secs} seconds,"
-                f" exceeding the inactivity_duration of {conf.inactivity_duration} seconds"
-            )
+        # reset in case inactivity_duration was disabled via in-place update
+        job_model.inactivity_secs = None
+        return
+    logger.debug("%s: no SSH connections for %s seconds", fmt(job_model), no_connections_secs)
+    job_model.inactivity_secs = no_connections_secs
+    if no_connections_secs is None:
+        # TODO(0.19 or earlier): make no_connections_secs required
+        job_model.status = JobStatus.TERMINATING
+        job_model.termination_reason = JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY
+        job_model.termination_reason_message = (
+            "The selected instance was created before dstack 0.18.41"
+            " and does not support inactivity_duration"
+        )
+    elif no_connections_secs >= conf.inactivity_duration:
+        job_model.status = JobStatus.TERMINATING
+        # TODO(0.19 or earlier): set JobTerminationReason.INACTIVITY_DURATION_EXCEEDED
+        job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
+        job_model.termination_reason_message = (
+            f"The job was inactive for {no_connections_secs} seconds,"
+            f" exceeding the inactivity_duration of {conf.inactivity_duration} seconds"
+        )
+
+
+async def _check_gpu_utilization(session: AsyncSession, job_model: JobModel, job: Job) -> None:
+    policy = job.job_spec.utilization_policy
+    if policy is None:
+        return
+    after = common_utils.get_current_datetime() - timedelta(seconds=policy.time_window)
+    job_metrics = await get_job_metrics(session, job_model, after=after)
+    gpus_util_metrics: list[Metric] = []
+    for metric in job_metrics.metrics:
+        if metric.name.startswith("gpu_util_percent_gpu"):
+            gpus_util_metrics.append(metric)
+    if not gpus_util_metrics or gpus_util_metrics[0].timestamps[-1] > after + timedelta(minutes=1):
+        # Job has started recently, not enough points collected.
+        # Assuming that metrics collection interval less than 1 minute.
+        logger.debug("%s: GPU utilization check: not enough samples", fmt(job_model))
+        return
+    if _should_terminate_due_to_low_gpu_util(
+        policy.min_gpu_utilization, [m.values for m in gpus_util_metrics]
+    ):
+        logger.info("%s: GPU utilization check: terminating", fmt(job_model))
+        job_model.status = JobStatus.TERMINATING
+        # TODO(0.19 or earlier): set JobTerminationReason.TERMINATED_DUE_TO_UTILIZATION_POLICY
+        job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
+        job_model.termination_reason_message = (
+            f"The job GPU utilization below {policy.min_gpu_utilization}%"
+            f" for {policy.time_window} seconds"
+        )
+    else:
+        logger.debug("%s: GPU utilization check: OK", fmt(job_model))
+
+
+def _should_terminate_due_to_low_gpu_util(min_util: int, gpus_util: Iterable[Iterable[int]]):
+    for gpu_util in gpus_util:
+        if all(util < min_util for util in gpu_util):
+            return True
+    return False
 
 
 def _get_cluster_info(
@@ -696,20 +743,29 @@ def _get_cluster_info(
 
 
 async def _get_job_code(
-    session: AsyncSession, project: ProjectModel, repo: RepoModel, code_hash: str
+    session: AsyncSession, project: ProjectModel, repo: RepoModel, code_hash: Optional[str]
 ) -> bytes:
+    if code_hash is None:
+        return b""
     code_model = await get_code_model(session=session, repo=repo, code_hash=code_hash)
     if code_model is None:
         return b""
-    storage = get_default_storage()
-    if storage is None or code_model.blob is not None:
+    if code_model.blob is not None:
         return code_model.blob
+    storage = get_default_storage()
+    if storage is None:
+        return b""
     blob = await common_utils.run_async(
         storage.get_code,
         project.name,
         repo.name,
         code_hash,
     )
+    if blob is None:
+        logger.error(
+            "Failed to get repo code hash %s from storage for repo %s", code_hash, repo.name
+        )
+        return b""
     return blob
 
 

dstack/_internal/server/background/tasks/process_runs.py

@@ -74,6 +74,7 @@ async def _process_next_run():
                     JobModel.run_id == run_model.id,
                     JobModel.id.not_in(job_lockset),
                 )
+                .order_by(JobModel.id)  # take locks in order
                 .with_for_update(skip_locked=True)
             )
             job_models = res.scalars().all()