dstack 0.19.20__py3-none-any.whl → 0.19.22__py3-none-any.whl

dstack/_internal/server/schemas/health/dcgm.py

@@ -0,0 +1,56 @@
+from enum import IntEnum
+
+from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.health import HealthStatus
+
+
+class DCGMHealthResult(IntEnum):
+    """
+    `dcgmHealthWatchResult_enum`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/const.go#L1020-L1026
+    """
+
+    DCGM_HEALTH_RESULT_PASS = 0
+    DCGM_HEALTH_RESULT_WARN = 10
+    DCGM_HEALTH_RESULT_FAIL = 20
+
+    def to_health_status(self) -> HealthStatus:
+        if self == self.DCGM_HEALTH_RESULT_PASS:
+            return HealthStatus.HEALTHY
+        if self == self.DCGM_HEALTH_RESULT_WARN:
+            return HealthStatus.WARNING
+        if self == self.DCGM_HEALTH_RESULT_FAIL:
+            return HealthStatus.FAILURE
+        raise AssertionError("should not reach here")
+
+
+class DCGMHealthIncident(CoreModel):
+    """
+    Flattened `dcgmIncidentInfo_t`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/health.go#L68-L73
+    """
+
+    # dcgmIncidentInfo_t
+    system: int
+    health: DCGMHealthResult
+
+    # dcgmDiagErrorDetail_t
+    error_message: str
+    error_code: int
+
+    # dcgmGroupEntityPair_t
+    entity_group_id: int  # dcgmGroupEntityPair_t
+    entity_id: int
+
+
+class DCGMHealthResponse(CoreModel):
+    """
+    `dcgmHealthResponse_v5`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/health.go#L75-L78
+    """
+
+    overall_health: DCGMHealthResult
+    incidents: list[DCGMHealthIncident]

dstack/_internal/server/schemas/instances.py

@@ -3,6 +3,8 @@ from typing import Optional
 from uuid import UUID
 
 from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.health import HealthCheck, HealthStatus
+from dstack._internal.server.schemas.runner import InstanceHealthResponse
 
 
 class ListInstancesRequest(CoreModel):
@@ -13,3 +15,33 @@ class ListInstancesRequest(CoreModel):
     prev_id: Optional[UUID] = None
     limit: int = 1000
     ascending: bool = False
+
+
+class InstanceCheck(CoreModel):
+    reachable: bool
+    message: Optional[str] = None
+    health_response: Optional[InstanceHealthResponse] = None
+
+    def get_health_status(self) -> HealthStatus:
+        if self.health_response is None:
+            return HealthStatus.HEALTHY
+        if self.health_response.dcgm is None:
+            return HealthStatus.HEALTHY
+        return self.health_response.dcgm.overall_health.to_health_status()
+
+    def has_health_checks(self) -> bool:
+        if self.health_response is None:
+            return False
+        return self.health_response.dcgm is not None
+
+
+class GetInstanceHealthChecksRequest(CoreModel):
+    fleet_name: str
+    instance_num: int
+    after: Optional[datetime] = None
+    before: Optional[datetime] = None
+    limit: Optional[int] = None
+
+
+class GetInstanceHealthChecksResponse(CoreModel):
+    health_checks: list[HealthCheck]

dstack/_internal/server/schemas/runner.py

@@ -16,6 +16,7 @@ from dstack._internal.core.models.runs import (
     RunSpec,
 )
 from dstack._internal.core.models.volumes import InstanceMountPoint, VolumeMountPoint
+from dstack._internal.server.schemas.health.dcgm import DCGMHealthResponse
 
 
 class JobStateEvent(CoreModel):
@@ -114,6 +115,10 @@ class HealthcheckResponse(CoreModel):
     version: str
 
 
+class InstanceHealthResponse(CoreModel):
+    dcgm: Optional[DCGMHealthResponse] = None
+
+
 class GPUMetrics(CoreModel):
     gpu_memory_usage_bytes: int
     gpu_util_percent: int

dstack/_internal/server/services/fleets.py

@@ -1,6 +1,6 @@
 import uuid
 from collections.abc import Callable
-from datetime import datetime, timezone
+from datetime import datetime
 from functools import wraps
 from typing import List, Literal, Optional, Tuple, TypeVar, Union, cast
 
@@ -8,8 +8,8 @@ from sqlalchemy import and_, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, selectinload
 
-from dstack._internal.core.backends import BACKENDS_WITH_CREATE_INSTANCE_SUPPORT
 from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.features import BACKENDS_WITH_CREATE_INSTANCE_SUPPORT
 from dstack._internal.core.errors import (
     ForbiddenError,
     ResourceExistsError,
@@ -49,6 +49,7 @@ from dstack._internal.server.db import get_db
 from dstack._internal.server.models import (
     FleetModel,
     InstanceModel,
+    JobModel,
     ProjectModel,
     UserModel,
 )
@@ -66,7 +67,6 @@ from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.services.projects import (
     get_member,
     get_member_permissions,
-    list_project_models,
     list_user_project_models,
 )
 from dstack._internal.server.services.resources import set_resources_defaults
@@ -87,10 +87,11 @@ async def list_fleets(
     limit: int,
     ascending: bool,
 ) -> List[Fleet]:
-    if user.global_role == GlobalRole.ADMIN:
-        projects = await list_project_models(session=session)
-    else:
-        projects = await list_user_project_models(session=session, user=user)
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+        only_names=True,
+    )
     if project_name is not None:
         projects = [p for p in projects if p.name == project_name]
     fleet_models = await list_projects_fleet_models(
@@ -398,7 +399,11 @@ async def apply_plan(
                 FleetModel.id == fleet_model.id,
                 FleetModel.deleted == False,
             )
-            .options(selectinload(FleetModel.instances))
+            .options(
+                selectinload(FleetModel.instances)
+                .joinedload(InstanceModel.jobs)
+                .load_only(JobModel.id)
+            )
             .options(selectinload(FleetModel.runs))
             .execution_options(populate_existing=True)
             .order_by(FleetModel.id)  # take locks in order
@@ -563,7 +568,11 @@ async def delete_fleets(
                 FleetModel.name.in_(names),
                 FleetModel.deleted == False,
             )
-            .options(selectinload(FleetModel.instances))
+            .options(
+                selectinload(FleetModel.instances)
+                .joinedload(InstanceModel.jobs)
+                .load_only(JobModel.id)
+            )
             .options(selectinload(FleetModel.runs))
             .execution_options(populate_existing=True)
             .order_by(FleetModel.id)  # take locks in order
@@ -600,7 +609,7 @@ def fleet_model_to_fleet(
         name=fleet_model.name,
         project_name=fleet_model.project.name,
         spec=spec,
-        created_at=fleet_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=fleet_model.created_at,
         status=fleet_model.status,
         status_message=fleet_model.status_message,
         instances=instances,

dstack/_internal/server/services/gateways/__init__.py

@@ -1,7 +1,7 @@
 import asyncio
 import datetime
 import uuid
-from datetime import timedelta, timezone
+from datetime import timedelta
 from functools import partial
 from typing import List, Optional, Sequence
 
@@ -11,16 +11,16 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 
 import dstack._internal.utils.random_names as random_names
-from dstack._internal.core.backends import (
-    BACKENDS_WITH_GATEWAY_SUPPORT,
-    BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT,
-)
 from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithGatewaySupport,
     get_dstack_gateway_wheel,
     get_dstack_runner_version,
 )
+from dstack._internal.core.backends.features import (
+    BACKENDS_WITH_GATEWAY_SUPPORT,
+    BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT,
+)
 from dstack._internal.core.errors import (
     GatewayError,
     ResourceNotExistsError,
@@ -86,15 +86,6 @@ async def get_gateway_by_name(
     return gateway_model_to_gateway(gateway)
 
 
-async def get_project_default_gateway(
-    session: AsyncSession, project: ProjectModel
-) -> Optional[Gateway]:
-    gateway: Optional[GatewayModel] = project.default_gateway
-    if gateway is None:
-        return None
-    return gateway_model_to_gateway(gateway)
-
-
 async def create_gateway_compute(
     project_name: str,
     backend_compute: Compute,
@@ -181,9 +172,9 @@ async def create_gateway(
         session.add(gateway)
         await session.commit()
 
-        if project.default_gateway is None or configuration.default:
+        default_gateway = await get_project_default_gateway_model(session=session, project=project)
+        if default_gateway is None or configuration.default:
             await set_default_gateway(session=session, project=project, name=configuration.name)
-
         return gateway_model_to_gateway(gateway)
 
 
@@ -349,6 +340,15 @@ async def get_project_gateway_model_by_name(
     return res.scalar()
 
 
+async def get_project_default_gateway_model(
+    session: AsyncSession, project: ProjectModel
+) -> Optional[GatewayModel]:
+    res = await session.execute(
+        select(GatewayModel).where(GatewayModel.id == project.default_gateway_id)
+    )
+    return res.scalar_one_or_none()
+
+
 async def generate_gateway_name(session: AsyncSession, project: ProjectModel) -> str:
     gateways = await list_project_gateway_models(session=session, project=project)
     names = {g.name for g in gateways}
@@ -557,7 +557,7 @@ def gateway_model_to_gateway(gateway_model: GatewayModel) -> Gateway:
         region=gateway_model.region,
         wildcard_domain=gateway_model.wildcard_domain,
         default=gateway_model.project.default_gateway_id == gateway_model.id,
-        created_at=gateway_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=gateway_model.created_at,
         status=gateway_model.status,
         status_message=gateway_model.status_message,
         configuration=configuration,

dstack/_internal/server/services/instances.py

@@ -1,20 +1,23 @@
+import operator
 import uuid
 from collections.abc import Container, Iterable
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Dict, List, Literal, Optional, Union
 
 import gpuhunt
 from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, load_only
 
-from dstack._internal.core.backends import BACKENDS_WITH_MULTINODE_SUPPORT
 from dstack._internal.core.backends.base.offers import (
     offer_to_catalog_item,
     requirements_to_query_filter,
 )
+from dstack._internal.core.backends.features import BACKENDS_WITH_MULTINODE_SUPPORT
+from dstack._internal.core.errors import ResourceNotExistsError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.envs import Env
+from dstack._internal.core.models.health import HealthCheck, HealthEvent, HealthStatus
 from dstack._internal.core.models.instances import (
     Instance,
     InstanceAvailability,
@@ -34,23 +37,76 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
-from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import Volume
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.server.models import (
     FleetModel,
+    InstanceHealthCheckModel,
     InstanceModel,
     ProjectModel,
     UserModel,
 )
+from dstack._internal.server.schemas.health.dcgm import DCGMHealthResponse
+from dstack._internal.server.schemas.runner import InstanceHealthResponse
 from dstack._internal.server.services.offers import generate_shared_offer
-from dstack._internal.server.services.projects import list_project_models, list_user_project_models
+from dstack._internal.server.services.projects import list_user_project_models
 from dstack._internal.utils import common as common_utils
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
 
+async def get_instance_health_checks(
+    session: AsyncSession,
+    project: ProjectModel,
+    fleet_name: str,
+    instance_num: int,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
+    limit: Optional[int] = None,
+) -> list[HealthCheck]:
+    """
+    Returns instance health checks ordered from the latest to the earliest.
+
+    Expected usage:
+        * limit=100 — get the latest 100 checks
+        * after=<now - 1 hour> — get checks for the last hour
+        * before=<earliest timestamp from the last batch>, limit=100 ­— paginate back in history
+    """
+    res = await session.execute(
+        select(InstanceModel)
+        .join(FleetModel)
+        .where(
+            ~InstanceModel.deleted,
+            InstanceModel.project_id == project.id,
+            InstanceModel.instance_num == instance_num,
+            FleetModel.name == fleet_name,
+        )
+        .options(load_only(InstanceModel.id))
+    )
+    instance = res.scalar_one_or_none()
+    if instance is None:
+        raise ResourceNotExistsError()
+
+    stmt = (
+        select(InstanceHealthCheckModel)
+        .where(InstanceHealthCheckModel.instance_id == instance.id)
+        .order_by(InstanceHealthCheckModel.collected_at.desc())
+    )
+    if after is not None:
+        stmt = stmt.where(InstanceHealthCheckModel.collected_at > after)
+    if before is not None:
+        stmt = stmt.where(InstanceHealthCheckModel.collected_at < before)
+    if limit is not None:
+        stmt = stmt.limit(limit)
+    health_checks: list[HealthCheck] = []
+    res = await session.execute(stmt)
+    for health_check_model in res.scalars():
+        health_check = instance_health_check_model_to_health_check(health_check_model)
+        health_checks.append(health_check)
+    return health_checks
+
+
 def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
     instance = Instance(
         id=instance_model.id,
@@ -61,8 +117,9 @@ def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
         instance_num=instance_model.instance_num,
         status=instance_model.status,
         unreachable=instance_model.unreachable,
+        health_status=instance_model.health,
         termination_reason=instance_model.termination_reason,
-        created=instance_model.created_at.replace(tzinfo=timezone.utc),
+        created=instance_model.created_at,
         total_blocks=instance_model.total_blocks,
         busy_blocks=instance_model.busy_blocks,
     )
@@ -82,6 +139,48 @@ def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
     return instance
 
 
+def instance_health_check_model_to_health_check(model: InstanceHealthCheckModel) -> HealthCheck:
+    collected_at = model.collected_at
+    status = HealthStatus.HEALTHY
+    events: list[HealthEvent] = []
+    instance_health_response = get_instance_health_response(model)
+    if (dcgm := instance_health_response.dcgm) is not None:
+        dcgm_health_check = dcgm_health_response_to_health_check(dcgm, collected_at)
+        status = dcgm_health_check.status
+        events.extend(dcgm_health_check.events)
+    events.sort(key=operator.attrgetter("timestamp"), reverse=True)
+    return HealthCheck(
+        collected_at=collected_at,
+        status=status,
+        events=events,
+    )
+
+
+def dcgm_health_response_to_health_check(
+    response: DCGMHealthResponse, collected_at: datetime
+) -> HealthCheck:
+    events: list[HealthEvent] = []
+    for incident in response.incidents:
+        events.append(
+            HealthEvent(
+                timestamp=collected_at,
+                status=incident.health.to_health_status(),
+                message=incident.error_message,
+            )
+        )
+    return HealthCheck(
+        collected_at=collected_at,
+        status=response.overall_health.to_health_status(),
+        events=events,
+    )
+
+
+def get_instance_health_response(
+    instance_health_check_model: InstanceHealthCheckModel,
+) -> InstanceHealthResponse:
+    return InstanceHealthResponse.__response__.parse_raw(instance_health_check_model.response)
+
+
 def get_instance_provisioning_data(instance_model: InstanceModel) -> Optional[JobProvisioningData]:
     if instance_model.job_provisioning_data is None:
         return None
@@ -195,6 +294,8 @@ def filter_pool_instances(
             continue
         if instance.unreachable:
             continue
+        if instance.health.is_failure():
+            continue
         fleet = instance.fleet
         if profile.fleets is not None and (fleet is None or fleet.name not in profile.fleets):
             continue
@@ -372,18 +473,15 @@ async def list_user_instances(
     limit: int,
     ascending: bool,
 ) -> List[Instance]:
-    if user.global_role == GlobalRole.ADMIN:
-        projects = await list_project_models(session=session)
-    else:
-        projects = await list_user_project_models(session=session, user=user)
-    if not projects:
-        return []
-
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+        only_names=True,
+    )
     if project_names is not None:
-        projects = [proj for proj in projects if proj.name in project_names]
+        projects = [p for p in projects if p.name in project_names]
         if len(projects) == 0:
             return []
-
     instance_models = await list_projects_instance_models(
         session=session,
         projects=projects,

dstack/_internal/server/services/jobs/__init__.py

@@ -1,13 +1,13 @@
 import itertools
 import json
-from datetime import timedelta, timezone
+from datetime import timedelta
 from typing import Dict, Iterable, List, Optional, Tuple
 from uuid import UUID
 
 import requests
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, load_only
 
 import dstack._internal.server.services.backends as backends_services
 from dstack._internal.core.backends.base.backend import Backend
@@ -52,6 +52,7 @@ from dstack._internal.server.services.jobs.configurators.dev import DevEnvironme
 from dstack._internal.server.services.jobs.configurators.service import ServiceJobConfigurator
 from dstack._internal.server.services.jobs.configurators.task import TaskJobConfigurator
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.services.probes import probe_model_to_probe
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
 from dstack._internal.server.services.volumes import (
@@ -115,7 +116,9 @@ async def get_run_job_model(
     return res.scalar_one_or_none()
 
 
-def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
+def job_model_to_job_submission(
+    job_model: JobModel, include_probes: bool = False
+) -> JobSubmission:
     job_provisioning_data = get_job_provisioning_data(job_model)
     if job_provisioning_data is not None:
         # TODO remove after transitioning to computed fields
@@ -130,17 +133,20 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         ):
             backend_data = json.loads(job_provisioning_data.backend_data)
             job_provisioning_data.backend = backend_data["base_backend"]
-    last_processed_at = job_model.last_processed_at.replace(tzinfo=timezone.utc)
+    last_processed_at = job_model.last_processed_at
     finished_at = None
     if job_model.status.is_finished():
         finished_at = last_processed_at
     status_message = _get_job_status_message(job_model)
     error = _get_job_error(job_model)
+    probes = []
+    if include_probes:
+        probes = [probe_model_to_probe(pm) for pm in job_model.probes]
     return JobSubmission(
         id=job_model.id,
         submission_num=job_model.submission_num,
         deployment_num=job_model.deployment_num,
-        submitted_at=job_model.submitted_at.replace(tzinfo=timezone.utc),
+        submitted_at=job_model.submitted_at,
         last_processed_at=last_processed_at,
         finished_at=finished_at,
         inactivity_secs=job_model.inactivity_secs,
@@ -152,6 +158,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         job_provisioning_data=job_provisioning_data,
         job_runtime_data=get_job_runtime_data(job_model),
         error=error,
+        probes=probes,
     )
 
 
@@ -231,10 +238,7 @@ async def process_terminating_job(
     Graceful stop should already be done by `process_terminating_run`.
     Caller must acquire the locks on the job and the job's instance.
     """
-    if (
-        job_model.remove_at is not None
-        and job_model.remove_at.replace(tzinfo=timezone.utc) > common.get_current_datetime()
-    ):
+    if job_model.remove_at is not None and job_model.remove_at > common.get_current_datetime():
         # it's too early to terminate the instance
         return
 
@@ -550,24 +554,25 @@ def _should_force_detach_volume(job_model: JobModel, stop_duration: Optional[int
     return (
         job_model.volumes_detached_at is not None
         and common.get_current_datetime()
-        > job_model.volumes_detached_at.replace(tzinfo=timezone.utc) + MIN_FORCE_DETACH_WAIT_PERIOD
+        > job_model.volumes_detached_at + MIN_FORCE_DETACH_WAIT_PERIOD
         and (
             job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER
             or stop_duration is not None
             and common.get_current_datetime()
-            > job_model.volumes_detached_at.replace(tzinfo=timezone.utc)
-            + timedelta(seconds=stop_duration)
+            > job_model.volumes_detached_at + timedelta(seconds=stop_duration)
         )
     )
 
 
 async def get_instances_ids_with_detaching_volumes(session: AsyncSession) -> List[UUID]:
     res = await session.execute(
-        select(JobModel).where(
+        select(JobModel)
+        .where(
             JobModel.status == JobStatus.TERMINATING,
             JobModel.used_instance_id.is_not(None),
             JobModel.volumes_detached_at.is_not(None),
         )
+        .options(load_only(JobModel.used_instance_id))
     )
     job_models = res.scalars().all()
     return [jm.used_instance_id for jm in job_models if jm.used_instance_id]

dstack/_internal/server/services/jobs/configurators/base.py

@@ -11,8 +11,14 @@ from dstack._internal import settings
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
 from dstack._internal.core.models.common import RegistryAuth
 from dstack._internal.core.models.configurations import (
+    DEFAULT_PROBE_INTERVAL,
+    DEFAULT_PROBE_METHOD,
+    DEFAULT_PROBE_READY_AFTER,
+    DEFAULT_PROBE_TIMEOUT,
+    DEFAULT_PROBE_URL,
     DEFAULT_REPO_DIR,
     PortMapping,
+    ProbeConfig,
     PythonVersion,
     RunConfigurationType,
     ServiceConfiguration,
@@ -26,6 +32,7 @@ from dstack._internal.core.models.runs import (
     AppSpec,
     JobSpec,
     JobSSHKey,
+    ProbeSpec,
     Requirements,
     Retry,
     RunSpec,
@@ -155,6 +162,7 @@ class JobConfigurator(ABC):
             repo_code_hash=self.run_spec.repo_code_hash,
             file_archives=self.run_spec.file_archives,
             service_port=self._service_port(),
+            probes=self._probes(),
         )
         return job_spec
 
@@ -313,6 +321,11 @@ class JobConfigurator(ABC):
             return self.run_spec.configuration.port.container_port
         return None
 
+    def _probes(self) -> list[ProbeSpec]:
+        if isinstance(self.run_spec.configuration, ServiceConfiguration):
+            return list(map(_probe_config_to_spec, self.run_spec.configuration.probes))
+        return []
+
 
 def interpolate_job_volumes(
     run_volumes: List[Union[MountPoint, str]],
@@ -353,6 +366,19 @@ def interpolate_job_volumes(
     return job_volumes
 
 
+def _probe_config_to_spec(c: ProbeConfig) -> ProbeSpec:
+    return ProbeSpec(
+        type=c.type,
+        url=c.url if c.url is not None else DEFAULT_PROBE_URL,
+        timeout=c.timeout if c.timeout is not None else DEFAULT_PROBE_TIMEOUT,
+        interval=c.interval if c.interval is not None else DEFAULT_PROBE_INTERVAL,
+        ready_after=c.ready_after if c.ready_after is not None else DEFAULT_PROBE_READY_AFTER,
+        method=c.method if c.method is not None else DEFAULT_PROBE_METHOD,
+        headers=c.headers,
+        body=c.body,
+    )
+
+
 def _join_shell_commands(commands: List[str]) -> str:
     for i, cmd in enumerate(commands):
         cmd = cmd.strip()

dstack/_internal/server/services/logging.py

@@ -1,9 +1,9 @@
 from typing import Union
 
-from dstack._internal.server.models import GatewayModel, JobModel, RunModel
+from dstack._internal.server.models import GatewayModel, JobModel, ProbeModel, RunModel
 
 
-def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
+def fmt(model: Union[RunModel, JobModel, GatewayModel, ProbeModel]) -> str:
     """Consistent string representation of a model for logging."""
     if isinstance(model, RunModel):
         return f"run({model.id.hex[:6]}){model.run_name}"
@@ -11,4 +11,6 @@ def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
         return f"job({model.id.hex[:6]}){model.job_name}"
     if isinstance(model, GatewayModel):
         return f"gateway({model.id.hex[:6]}){model.name}"
+    if isinstance(model, ProbeModel):
+        return f"probe({model.id.hex[:6]}){model.name}"
     return str(model)