dstack 0.19.21__py3-none-any.whl → 0.19.23rc1__py3-none-any.whl

dstack/_internal/server/models.py

@@ -28,6 +28,7 @@ from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import CoreModel
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.core.models.gateways import GatewayStatus
+from dstack._internal.core.models.health import HealthStatus
 from dstack._internal.core.models.instances import InstanceStatus
 from dstack._internal.core.models.profiles import (
     DEFAULT_FLEET_TERMINATION_IDLE_TIME,
@@ -427,6 +428,9 @@ class JobModel(BaseModel):
     replica_num: Mapped[int] = mapped_column(Integer)
     deployment_num: Mapped[int] = mapped_column(Integer)
     job_runtime_data: Mapped[Optional[str]] = mapped_column(Text)
+    probes: Mapped[list["ProbeModel"]] = relationship(
+        back_populates="job", order_by="ProbeModel.probe_num"
+    )
 
 
 class GatewayModel(BaseModel):
@@ -596,7 +600,11 @@ class InstanceModel(BaseModel):
     # instance termination handling
     termination_deadline: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
     termination_reason: Mapped[Optional[str]] = mapped_column(String(4000))
+    # Deprecated since 0.19.22, not used
     health_status: Mapped[Optional[str]] = mapped_column(String(4000))
+    health: Mapped[HealthStatus] = mapped_column(
+        EnumAsString(HealthStatus, 100), default=HealthStatus.HEALTHY
+    )
     first_termination_retry_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
     last_termination_retry_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
 
@@ -627,6 +635,21 @@ class InstanceModel(BaseModel):
     )
 
 
+class InstanceHealthCheckModel(BaseModel):
+    __tablename__ = "instance_health_checks"
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        UUIDType(binary=False), primary_key=True, default=uuid.uuid4
+    )
+
+    instance_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("instances.id"))
+    instance: Mapped["InstanceModel"] = relationship()
+
+    collected_at: Mapped[datetime] = mapped_column(NaiveDateTime)
+    status: Mapped[HealthStatus] = mapped_column(EnumAsString(HealthStatus, 100))
+    response: Mapped[str] = mapped_column(Text)
+
+
 class VolumeModel(BaseModel):
     __tablename__ = "volumes"
 
@@ -729,6 +752,24 @@ class JobPrometheusMetrics(BaseModel):
     text: Mapped[str] = mapped_column(Text)
 
 
+class ProbeModel(BaseModel):
+    __tablename__ = "probes"
+    __table_args__ = (UniqueConstraint("job_id", "probe_num", name="uq_probes_job_id_probe_num"),)
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        UUIDType(binary=False), primary_key=True, default=uuid.uuid4
+    )
+    name: Mapped[str] = mapped_column(String(100))
+
+    job_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("jobs.id"), primary_key=True)
+    job: Mapped["JobModel"] = relationship(back_populates="probes")
+
+    probe_num: Mapped[int] = mapped_column(Integer)  # index in JobSpec.probes
+    due: Mapped[datetime] = mapped_column(NaiveDateTime)
+    success_streak: Mapped[int] = mapped_column(BigInteger)
+    active: Mapped[bool] = mapped_column(Boolean)
+
+
 class SecretModel(BaseModel):
     __tablename__ = "secrets"
     __table_args__ = (UniqueConstraint("project_id", "name", name="uq_secrets_project_id_name"),)

dstack/_internal/server/routers/instances.py

@@ -3,12 +3,16 @@ from typing import List
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
 
-import dstack._internal.server.services.instances as instances
+import dstack._internal.server.services.instances as instances_services
 from dstack._internal.core.models.instances import Instance
 from dstack._internal.server.db import get_session
-from dstack._internal.server.models import UserModel
-from dstack._internal.server.schemas.instances import ListInstancesRequest
-from dstack._internal.server.security.permissions import Authenticated
+from dstack._internal.server.models import ProjectModel, UserModel
+from dstack._internal.server.schemas.instances import (
+    GetInstanceHealthChecksRequest,
+    GetInstanceHealthChecksResponse,
+    ListInstancesRequest,
+)
+from dstack._internal.server.security.permissions import Authenticated, ProjectMember
 from dstack._internal.server.utils.routers import (
     CustomORJSONResponse,
     get_base_api_additional_responses,
@@ -19,6 +23,11 @@ root_router = APIRouter(
     tags=["instances"],
     responses=get_base_api_additional_responses(),
 )
+project_router = APIRouter(
+    prefix="/api/project/{project_name}/instances",
+    tags=["instances"],
+    responses=get_base_api_additional_responses(),
+)
 
 
 @root_router.post("/list", response_model=List[Instance])
@@ -35,7 +44,7 @@ async def list_instances(
     the last instance from the previous page as `prev_created_at` and `prev_id`.
     """
     return CustomORJSONResponse(
-        await instances.list_user_instances(
+        await instances_services.list_user_instances(
             session=session,
             user=user,
             project_names=body.project_names,
@@ -47,3 +56,22 @@ async def list_instances(
             ascending=body.ascending,
         )
     )
+
+
+@project_router.post("/get_instance_health_checks", response_model=GetInstanceHealthChecksResponse)
+async def get_instance_health_checks(
+    body: GetInstanceHealthChecksRequest,
+    session: AsyncSession = Depends(get_session),
+    user_project: tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
+):
+    _, project = user_project
+    health_checks = await instances_services.get_instance_health_checks(
+        session=session,
+        project=project,
+        fleet_name=body.fleet_name,
+        instance_num=body.instance_num,
+        after=body.after,
+        before=body.before,
+        limit=body.limit,
+    )
+    return CustomORJSONResponse(GetInstanceHealthChecksResponse(health_checks=health_checks))

dstack/_internal/server/schemas/health/dcgm.py

@@ -0,0 +1,56 @@
+from enum import IntEnum
+
+from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.health import HealthStatus
+
+
+class DCGMHealthResult(IntEnum):
+    """
+    `dcgmHealthWatchResult_enum`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/const.go#L1020-L1026
+    """
+
+    DCGM_HEALTH_RESULT_PASS = 0
+    DCGM_HEALTH_RESULT_WARN = 10
+    DCGM_HEALTH_RESULT_FAIL = 20
+
+    def to_health_status(self) -> HealthStatus:
+        if self == self.DCGM_HEALTH_RESULT_PASS:
+            return HealthStatus.HEALTHY
+        if self == self.DCGM_HEALTH_RESULT_WARN:
+            return HealthStatus.WARNING
+        if self == self.DCGM_HEALTH_RESULT_FAIL:
+            return HealthStatus.FAILURE
+        raise AssertionError("should not reach here")
+
+
+class DCGMHealthIncident(CoreModel):
+    """
+    Flattened `dcgmIncidentInfo_t`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/health.go#L68-L73
+    """
+
+    # dcgmIncidentInfo_t
+    system: int
+    health: DCGMHealthResult
+
+    # dcgmDiagErrorDetail_t
+    error_message: str
+    error_code: int
+
+    # dcgmGroupEntityPair_t
+    entity_group_id: int  # dcgmGroupEntityPair_t
+    entity_id: int
+
+
+class DCGMHealthResponse(CoreModel):
+    """
+    `dcgmHealthResponse_v5`
+
+    See: https://github.com/NVIDIA/go-dcgm/blob/85ceb31/pkg/dcgm/health.go#L75-L78
+    """
+
+    overall_health: DCGMHealthResult
+    incidents: list[DCGMHealthIncident]

dstack/_internal/server/schemas/instances.py

@@ -3,6 +3,8 @@ from typing import Optional
 from uuid import UUID
 
 from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.health import HealthCheck, HealthStatus
+from dstack._internal.server.schemas.runner import InstanceHealthResponse
 
 
 class ListInstancesRequest(CoreModel):
@@ -13,3 +15,33 @@ class ListInstancesRequest(CoreModel):
     prev_id: Optional[UUID] = None
     limit: int = 1000
     ascending: bool = False
+
+
+class InstanceCheck(CoreModel):
+    reachable: bool
+    message: Optional[str] = None
+    health_response: Optional[InstanceHealthResponse] = None
+
+    def get_health_status(self) -> HealthStatus:
+        if self.health_response is None:
+            return HealthStatus.HEALTHY
+        if self.health_response.dcgm is None:
+            return HealthStatus.HEALTHY
+        return self.health_response.dcgm.overall_health.to_health_status()
+
+    def has_health_checks(self) -> bool:
+        if self.health_response is None:
+            return False
+        return self.health_response.dcgm is not None
+
+
+class GetInstanceHealthChecksRequest(CoreModel):
+    fleet_name: str
+    instance_num: int
+    after: Optional[datetime] = None
+    before: Optional[datetime] = None
+    limit: Optional[int] = None
+
+
+class GetInstanceHealthChecksResponse(CoreModel):
+    health_checks: list[HealthCheck]

dstack/_internal/server/schemas/runner.py

@@ -16,6 +16,7 @@ from dstack._internal.core.models.runs import (
     RunSpec,
 )
 from dstack._internal.core.models.volumes import InstanceMountPoint, VolumeMountPoint
+from dstack._internal.server.schemas.health.dcgm import DCGMHealthResponse
 
 
 class JobStateEvent(CoreModel):
@@ -114,6 +115,10 @@ class HealthcheckResponse(CoreModel):
     version: str
 
 
+class InstanceHealthResponse(CoreModel):
+    dcgm: Optional[DCGMHealthResponse] = None
+
+
 class GPUMetrics(CoreModel):
     gpu_memory_usage_bytes: int
     gpu_util_percent: int

dstack/_internal/server/services/instances.py

@@ -1,3 +1,4 @@
+import operator
 import uuid
 from collections.abc import Container, Iterable
 from datetime import datetime
@@ -6,15 +7,17 @@ from typing import Dict, List, Literal, Optional, Union
 import gpuhunt
 from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, load_only
 
 from dstack._internal.core.backends.base.offers import (
     offer_to_catalog_item,
     requirements_to_query_filter,
 )
 from dstack._internal.core.backends.features import BACKENDS_WITH_MULTINODE_SUPPORT
+from dstack._internal.core.errors import ResourceNotExistsError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.envs import Env
+from dstack._internal.core.models.health import HealthCheck, HealthEvent, HealthStatus
 from dstack._internal.core.models.instances import (
     Instance,
     InstanceAvailability,
@@ -38,10 +41,13 @@ from dstack._internal.core.models.volumes import Volume
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.server.models import (
     FleetModel,
+    InstanceHealthCheckModel,
     InstanceModel,
     ProjectModel,
     UserModel,
 )
+from dstack._internal.server.schemas.health.dcgm import DCGMHealthResponse
+from dstack._internal.server.schemas.runner import InstanceHealthResponse
 from dstack._internal.server.services.offers import generate_shared_offer
 from dstack._internal.server.services.projects import list_user_project_models
 from dstack._internal.utils import common as common_utils
@@ -50,6 +56,57 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 
+async def get_instance_health_checks(
+    session: AsyncSession,
+    project: ProjectModel,
+    fleet_name: str,
+    instance_num: int,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
+    limit: Optional[int] = None,
+) -> list[HealthCheck]:
+    """
+    Returns instance health checks ordered from the latest to the earliest.
+
+    Expected usage:
+        * limit=100 — get the latest 100 checks
+        * after=<now - 1 hour> — get checks for the last hour
+        * before=<earliest timestamp from the last batch>, limit=100 ­— paginate back in history
+    """
+    res = await session.execute(
+        select(InstanceModel)
+        .join(FleetModel)
+        .where(
+            ~InstanceModel.deleted,
+            InstanceModel.project_id == project.id,
+            InstanceModel.instance_num == instance_num,
+            FleetModel.name == fleet_name,
+        )
+        .options(load_only(InstanceModel.id))
+    )
+    instance = res.scalar_one_or_none()
+    if instance is None:
+        raise ResourceNotExistsError()
+
+    stmt = (
+        select(InstanceHealthCheckModel)
+        .where(InstanceHealthCheckModel.instance_id == instance.id)
+        .order_by(InstanceHealthCheckModel.collected_at.desc())
+    )
+    if after is not None:
+        stmt = stmt.where(InstanceHealthCheckModel.collected_at > after)
+    if before is not None:
+        stmt = stmt.where(InstanceHealthCheckModel.collected_at < before)
+    if limit is not None:
+        stmt = stmt.limit(limit)
+    health_checks: list[HealthCheck] = []
+    res = await session.execute(stmt)
+    for health_check_model in res.scalars():
+        health_check = instance_health_check_model_to_health_check(health_check_model)
+        health_checks.append(health_check)
+    return health_checks
+
+
 def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
     instance = Instance(
         id=instance_model.id,
@@ -60,6 +117,7 @@ def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
         instance_num=instance_model.instance_num,
         status=instance_model.status,
         unreachable=instance_model.unreachable,
+        health_status=instance_model.health,
         termination_reason=instance_model.termination_reason,
         created=instance_model.created_at,
         total_blocks=instance_model.total_blocks,
@@ -81,6 +139,48 @@ def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
     return instance
 
 
+def instance_health_check_model_to_health_check(model: InstanceHealthCheckModel) -> HealthCheck:
+    collected_at = model.collected_at
+    status = HealthStatus.HEALTHY
+    events: list[HealthEvent] = []
+    instance_health_response = get_instance_health_response(model)
+    if (dcgm := instance_health_response.dcgm) is not None:
+        dcgm_health_check = dcgm_health_response_to_health_check(dcgm, collected_at)
+        status = dcgm_health_check.status
+        events.extend(dcgm_health_check.events)
+    events.sort(key=operator.attrgetter("timestamp"), reverse=True)
+    return HealthCheck(
+        collected_at=collected_at,
+        status=status,
+        events=events,
+    )
+
+
+def dcgm_health_response_to_health_check(
+    response: DCGMHealthResponse, collected_at: datetime
+) -> HealthCheck:
+    events: list[HealthEvent] = []
+    for incident in response.incidents:
+        events.append(
+            HealthEvent(
+                timestamp=collected_at,
+                status=incident.health.to_health_status(),
+                message=incident.error_message,
+            )
+        )
+    return HealthCheck(
+        collected_at=collected_at,
+        status=response.overall_health.to_health_status(),
+        events=events,
+    )
+
+
+def get_instance_health_response(
+    instance_health_check_model: InstanceHealthCheckModel,
+) -> InstanceHealthResponse:
+    return InstanceHealthResponse.__response__.parse_raw(instance_health_check_model.response)
+
+
 def get_instance_provisioning_data(instance_model: InstanceModel) -> Optional[JobProvisioningData]:
     if instance_model.job_provisioning_data is None:
         return None
@@ -194,6 +294,8 @@ def filter_pool_instances(
             continue
         if instance.unreachable:
             continue
+        if instance.health.is_failure():
+            continue
         fleet = instance.fleet
         if profile.fleets is not None and (fleet is None or fleet.name not in profile.fleets):
             continue

dstack/_internal/server/services/jobs/__init__.py

@@ -52,6 +52,7 @@ from dstack._internal.server.services.jobs.configurators.dev import DevEnvironme
 from dstack._internal.server.services.jobs.configurators.service import ServiceJobConfigurator
 from dstack._internal.server.services.jobs.configurators.task import TaskJobConfigurator
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.services.probes import probe_model_to_probe
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
 from dstack._internal.server.services.volumes import (
@@ -115,7 +116,9 @@ async def get_run_job_model(
     return res.scalar_one_or_none()
 
 
-def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
+def job_model_to_job_submission(
+    job_model: JobModel, include_probes: bool = False
+) -> JobSubmission:
     job_provisioning_data = get_job_provisioning_data(job_model)
     if job_provisioning_data is not None:
         # TODO remove after transitioning to computed fields
@@ -136,6 +139,9 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         finished_at = last_processed_at
     status_message = _get_job_status_message(job_model)
     error = _get_job_error(job_model)
+    probes = []
+    if include_probes:
+        probes = [probe_model_to_probe(pm) for pm in job_model.probes]
     return JobSubmission(
         id=job_model.id,
         submission_num=job_model.submission_num,
@@ -152,6 +158,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         job_provisioning_data=job_provisioning_data,
         job_runtime_data=get_job_runtime_data(job_model),
         error=error,
+        probes=probes,
     )
 
 

dstack/_internal/server/services/jobs/configurators/base.py

@@ -11,8 +11,14 @@ from dstack._internal import settings
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
 from dstack._internal.core.models.common import RegistryAuth
 from dstack._internal.core.models.configurations import (
+    DEFAULT_PROBE_INTERVAL,
+    DEFAULT_PROBE_METHOD,
+    DEFAULT_PROBE_READY_AFTER,
+    DEFAULT_PROBE_TIMEOUT,
+    DEFAULT_PROBE_URL,
     DEFAULT_REPO_DIR,
     PortMapping,
+    ProbeConfig,
     PythonVersion,
     RunConfigurationType,
     ServiceConfiguration,
@@ -26,6 +32,7 @@ from dstack._internal.core.models.runs import (
     AppSpec,
     JobSpec,
     JobSSHKey,
+    ProbeSpec,
     Requirements,
     Retry,
     RunSpec,
@@ -155,6 +162,7 @@ class JobConfigurator(ABC):
             repo_code_hash=self.run_spec.repo_code_hash,
             file_archives=self.run_spec.file_archives,
             service_port=self._service_port(),
+            probes=self._probes(),
         )
         return job_spec
 
@@ -313,6 +321,11 @@ class JobConfigurator(ABC):
             return self.run_spec.configuration.port.container_port
         return None
 
+    def _probes(self) -> list[ProbeSpec]:
+        if isinstance(self.run_spec.configuration, ServiceConfiguration):
+            return list(map(_probe_config_to_spec, self.run_spec.configuration.probes))
+        return []
+
 
 def interpolate_job_volumes(
     run_volumes: List[Union[MountPoint, str]],
@@ -353,6 +366,19 @@ def interpolate_job_volumes(
     return job_volumes
 
 
+def _probe_config_to_spec(c: ProbeConfig) -> ProbeSpec:
+    return ProbeSpec(
+        type=c.type,
+        url=c.url if c.url is not None else DEFAULT_PROBE_URL,
+        timeout=c.timeout if c.timeout is not None else DEFAULT_PROBE_TIMEOUT,
+        interval=c.interval if c.interval is not None else DEFAULT_PROBE_INTERVAL,
+        ready_after=c.ready_after if c.ready_after is not None else DEFAULT_PROBE_READY_AFTER,
+        method=c.method if c.method is not None else DEFAULT_PROBE_METHOD,
+        headers=c.headers,
+        body=c.body,
+    )
+
+
 def _join_shell_commands(commands: List[str]) -> str:
     for i, cmd in enumerate(commands):
         cmd = cmd.strip()

dstack/_internal/server/services/logging.py

@@ -1,9 +1,9 @@
 from typing import Union
 
-from dstack._internal.server.models import GatewayModel, JobModel, RunModel
+from dstack._internal.server.models import GatewayModel, JobModel, ProbeModel, RunModel
 
 
-def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
+def fmt(model: Union[RunModel, JobModel, GatewayModel, ProbeModel]) -> str:
     """Consistent string representation of a model for logging."""
     if isinstance(model, RunModel):
         return f"run({model.id.hex[:6]}){model.run_name}"
@@ -11,4 +11,6 @@ def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
         return f"job({model.id.hex[:6]}){model.job_name}"
     if isinstance(model, GatewayModel):
         return f"gateway({model.id.hex[:6]}){model.name}"
+    if isinstance(model, ProbeModel):
+        return f"probe({model.id.hex[:6]}){model.name}"
     return str(model)

dstack/_internal/server/services/logs/aws.py

@@ -1,5 +1,7 @@
 import itertools
 import operator
+import urllib
+import urllib.parse
 from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
 from typing import Iterator, List, Optional, Set, Tuple, TypedDict
@@ -64,6 +66,7 @@ class CloudWatchLogStorage(LogStorage):
             self._client = session.client("logs")
             self._check_group_exists(group)
         self._group = group
+        self._region = self._client.meta.region_name
         # Stores names of already created streams.
         # XXX: This set acts as an unbound cache. If this becomes a problem (in case of _very_ long
         # running server and/or lots of jobs, consider replacing it with an LRU cache, e.g.,
@@ -103,7 +106,11 @@ class CloudWatchLogStorage(LogStorage):
             )
             for cw_event in cw_events
         ]
-        return JobSubmissionLogs(logs=logs, next_token=next_token)
+        return JobSubmissionLogs(
+            logs=logs,
+            external_url=self._get_stream_external_url(stream),
+            next_token=next_token,
+        )
 
     def _get_log_events_with_retry(
         self, stream: str, request: PollLogsRequest
@@ -181,6 +188,11 @@ class CloudWatchLogStorage(LogStorage):
 
         return events, next_token
 
+    def _get_stream_external_url(self, stream: str) -> str:
+        quoted_group = urllib.parse.quote(self._group, safe="")
+        quoted_stream = urllib.parse.quote(stream, safe="")
+        return f"https://console.aws.amazon.com/cloudwatch/home?region={self._region}#logsV2:log-groups/log-group/{quoted_group}/log-events/{quoted_stream}"
+
     def write_logs(
         self,
         project: ProjectModel,

dstack/_internal/server/services/logs/gcp.py

@@ -1,3 +1,4 @@
+import urllib.parse
 from typing import List
 from uuid import UUID
 
@@ -48,6 +49,7 @@ class GCPLogStorage(LogStorage):
     # (https://cloud.google.com/logging/docs/analyze/custom-index).
 
     def __init__(self, project_id: str):
+        self.project_id = project_id
         try:
             self.client = logging_v2.Client(project=project_id)
             self.logger = self.client.logger(name=self.LOG_NAME)
@@ -106,7 +108,11 @@ class GCPLogStorage(LogStorage):
                 "GCP Logging read request limit exceeded."
                 " It's recommended to increase default entries.list request quota from 60 per minute."
             )
-        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
+        return JobSubmissionLogs(
+            logs=logs,
+            external_url=self._get_stream_extrnal_url(stream_name),
+            next_token=next_token if len(logs) > 0 else None,
+        )
 
     def write_logs(
         self,
@@ -162,3 +168,12 @@ class GCPLogStorage(LogStorage):
         self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer
     ) -> str:
         return f"{project_name}-{run_name}-{job_submission_id}-{producer.value}"
+
+    def _get_stream_extrnal_url(self, stream_name: str) -> str:
+        log_name_resource_name = self._get_log_name_resource_name()
+        query = f'logName="{log_name_resource_name}" AND labels.stream="{stream_name}"'
+        quoted_query = urllib.parse.quote(query, safe="")
+        return f"https://console.cloud.google.com/logs/query;query={quoted_query}?project={self.project_id}"
+
+    def _get_log_name_resource_name(self) -> str:
+        return f"projects/{self.project_id}/logs/{self.LOG_NAME}"

dstack/_internal/server/services/probes.py

@@ -0,0 +1,6 @@
+from dstack._internal.core.models.runs import Probe
+from dstack._internal.server.models import ProbeModel
+
+
+def probe_model_to_probe(probe_model: ProbeModel) -> Probe:
+    return Probe(success_streak=probe_model.success_streak)

dstack/_internal/server/services/projects.py

@@ -197,6 +197,10 @@ async def set_project_members(
     project: ProjectModel,
     members: List[MemberSetting],
 ):
+    usernames = {m.username for m in members}
+    if len(usernames) != len(members):
+        raise ServerClientError("Cannot add same user multiple times")
+
     project = await get_project_model_by_name_or_error(
         session=session,
         project_name=project.name,
@@ -245,6 +249,10 @@ async def add_project_members(
     members: List[MemberSetting],
 ):
     """Add multiple members to a project."""
+    usernames = {m.username for m in members}
+    if len(usernames) != len(members):
+        raise ServerClientError("Cannot add same user multiple times")
+
     project = await get_project_model_by_name_or_error(
         session=session,
         project_name=project.name,
@@ -259,7 +267,10 @@ async def add_project_members(
     )
 
     if not is_self_join_to_public:
-        if requesting_user_role not in [ProjectRole.ADMIN, ProjectRole.MANAGER]:
+        if user.global_role != GlobalRole.ADMIN and requesting_user_role not in [
+            ProjectRole.ADMIN,
+            ProjectRole.MANAGER,
+        ]:
             raise ForbiddenError("Access denied: insufficient permissions to add members")
 
         if user.global_role != GlobalRole.ADMIN and requesting_user_role == ProjectRole.MANAGER:
@@ -272,8 +283,6 @@ async def add_project_members(
         if members[0].project_role != ProjectRole.USER:
             raise ForbiddenError("Access denied: can only join public projects as user role")
 
-    usernames = [member.username for member in members]
-
     res = await session.execute(
         select(UserModel).where((UserModel.name.in_(usernames)) | (UserModel.email.in_(usernames)))
     )
@@ -628,7 +637,10 @@ async def remove_project_members(
     )
 
     if not is_self_leave:
-        if requesting_user_role not in [ProjectRole.ADMIN, ProjectRole.MANAGER]:
+        if user.global_role != GlobalRole.ADMIN and requesting_user_role not in [
+            ProjectRole.ADMIN,
+            ProjectRole.MANAGER,
+        ]:
             raise ForbiddenError("Access denied: insufficient permissions to remove members")
 
     res = await session.execute(