dstack 0.19.18__py3-none-any.whl → 0.19.20__py3-none-any.whl

dstack/_internal/server/services/gateways/client.py

@@ -7,9 +7,9 @@ from pydantic import parse_obj_as
 
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
 from dstack._internal.core.errors import GatewayError
-from dstack._internal.core.models.configurations import RateLimit
+from dstack._internal.core.models.configurations import RateLimit, ServiceConfiguration
 from dstack._internal.core.models.instances import SSHConnectionParams
-from dstack._internal.core.models.runs import JobSubmission, Run
+from dstack._internal.core.models.runs import JobSpec, JobSubmission, Run, get_service_port
 from dstack._internal.proxy.gateway.schemas.stats import ServiceStats
 from dstack._internal.server import settings
 
@@ -80,13 +80,15 @@ class GatewayClient:
     async def register_replica(
         self,
         run: Run,
+        job_spec: JobSpec,
         job_submission: JobSubmission,
         ssh_head_proxy: Optional[SSHConnectionParams],
         ssh_head_proxy_private_key: Optional[str],
     ):
+        assert isinstance(run.run_spec.configuration, ServiceConfiguration)
         payload = {
             "job_id": job_submission.id.hex,
-            "app_port": run.run_spec.configuration.port.container_port,
+            "app_port": get_service_port(job_spec, run.run_spec.configuration),
             "ssh_head_proxy": ssh_head_proxy.dict() if ssh_head_proxy is not None else None,
             "ssh_head_proxy_private_key": ssh_head_proxy_private_key,
         }

dstack/_internal/server/services/instances.py

@@ -106,6 +106,14 @@ def get_instance_requirements(instance_model: InstanceModel) -> Requirements:
     return Requirements.__response__.parse_raw(instance_model.requirements)
 
 
+def get_instance_remote_connection_info(
+    instance_model: InstanceModel,
+) -> Optional[RemoteConnectionInfo]:
+    if instance_model.remote_connection_info is None:
+        return None
+    return RemoteConnectionInfo.__response__.parse_raw(instance_model.remote_connection_info)
+
+
 def get_instance_ssh_private_keys(instance_model: InstanceModel) -> tuple[str, Optional[str]]:
     """
     Returns a pair of SSH private keys: host key and optional proxy jump key.

dstack/_internal/server/services/jobs/__init__.py

@@ -134,6 +134,8 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
     finished_at = None
     if job_model.status.is_finished():
         finished_at = last_processed_at
+    status_message = _get_job_status_message(job_model)
+    error = _get_job_error(job_model)
     return JobSubmission(
         id=job_model.id,
         submission_num=job_model.submission_num,
@@ -143,11 +145,13 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         finished_at=finished_at,
         inactivity_secs=job_model.inactivity_secs,
         status=job_model.status,
+        status_message=status_message,
         termination_reason=job_model.termination_reason,
         termination_reason_message=job_model.termination_reason_message,
         exit_status=job_model.exit_status,
         job_provisioning_data=job_provisioning_data,
         job_runtime_data=get_job_runtime_data(job_model),
+        error=error,
     )
 
 
@@ -289,6 +293,19 @@ async def process_terminating_job(
     # so that stuck volumes don't prevent the instance from terminating.
     job_model.instance_id = None
     instance_model.last_job_processed_at = common.get_current_datetime()
+
+    volume_names = (
+        jrd.volume_names
+        if jrd and jrd.volume_names
+        else [va.volume.name for va in instance_model.volume_attachments]
+    )
+    if volume_names:
+        volumes = await list_project_volume_models(
+            session=session, project=instance_model.project, names=volume_names
+        )
+        for volume in volumes:
+            volume.last_job_processed_at = common.get_current_datetime()
+
     logger.info(
         "%s: instance '%s' has been released, new status is %s",
         fmt(job_model),
@@ -693,3 +710,31 @@ def _get_job_mount_point_attached_volume(
             continue
         return volume
     raise ServerClientError("Failed to find an eligible volume for the mount point")
+
+
+def _get_job_status_message(job_model: JobModel) -> str:
+    if job_model.status == JobStatus.DONE:
+        return "exited (0)"
+    elif job_model.status == JobStatus.FAILED:
+        if job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR:
+            return f"exited ({job_model.exit_status})"
+        elif (
+            job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
+        ):
+            return "no offers"
+        elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
+            return "interrupted"
+        else:
+            return "error"
+    elif job_model.status == JobStatus.TERMINATED:
+        if job_model.termination_reason == JobTerminationReason.TERMINATED_BY_USER:
+            return "stopped"
+        elif job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER:
+            return "aborted"
+    return job_model.status.value
+
+
+def _get_job_error(job_model: JobModel) -> Optional[str]:
+    if job_model.termination_reason is None:
+        return None
+    return job_model.termination_reason.to_error()

dstack/_internal/server/services/jobs/configurators/base.py

@@ -15,6 +15,7 @@ from dstack._internal.core.models.configurations import (
     PortMapping,
     PythonVersion,
     RunConfigurationType,
+    ServiceConfiguration,
 )
 from dstack._internal.core.models.profiles import (
     DEFAULT_STOP_DURATION,
@@ -153,6 +154,7 @@ class JobConfigurator(ABC):
             repo_data=self.run_spec.repo_data,
             repo_code_hash=self.run_spec.repo_code_hash,
             file_archives=self.run_spec.file_archives,
+            service_port=self._service_port(),
         )
         return job_spec
 
@@ -306,6 +308,11 @@ class JobConfigurator(ABC):
             )
         return self._job_ssh_key
 
+    def _service_port(self) -> Optional[int]:
+        if isinstance(self.run_spec.configuration, ServiceConfiguration):
+            return self.run_spec.configuration.port.container_port
+        return None
+
 
 def interpolate_job_volumes(
     run_volumes: List[Union[MountPoint, str]],

dstack/_internal/server/services/locking.py

@@ -172,7 +172,7 @@ async def _wait_to_lock_many(
     The keys must be sorted to prevent deadlock.
     """
     left_to_lock = keys.copy()
-    while len(left_to_lock) > 0:
+    while True:
         async with lock:
             locked_now_num = 0
             for key in left_to_lock:
@@ -182,4 +182,6 @@ async def _wait_to_lock_many(
                 locked.add(key)
                 locked_now_num += 1
             left_to_lock = left_to_lock[locked_now_num:]
+        if not left_to_lock:
+            return
         await asyncio.sleep(delay)

dstack/_internal/server/services/logging.py

@@ -1,12 +1,14 @@
 from typing import Union
 
-from dstack._internal.server.models import JobModel, RunModel
+from dstack._internal.server.models import GatewayModel, JobModel, RunModel
 
 
-def fmt(model: Union[RunModel, JobModel]) -> str:
+def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
     """Consistent string representation of a model for logging."""
     if isinstance(model, RunModel):
         return f"run({model.id.hex[:6]}){model.run_name}"
     if isinstance(model, JobModel):
         return f"job({model.id.hex[:6]}){model.job_name}"
+    if isinstance(model, GatewayModel):
+        return f"gateway({model.id.hex[:6]}){model.name}"
     return str(model)

dstack/_internal/server/services/logs/__init__.py

@@ -8,7 +8,11 @@ from dstack._internal.server.models import ProjectModel
 from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.aws import BOTO_AVAILABLE, CloudWatchLogStorage
-from dstack._internal.server.services.logs.base import LogStorage, LogStorageError
+from dstack._internal.server.services.logs.base import (
+    LogStorage,
+    LogStorageError,
+    b64encode_raw_message,
+)
 from dstack._internal.server.services.logs.filelog import FileLogStorage
 from dstack._internal.server.services.logs.gcp import GCP_LOGGING_AVAILABLE, GCPLogStorage
 from dstack._internal.utils.common import run_async
@@ -75,4 +79,13 @@ def write_logs(
 
 
 async def poll_logs_async(project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-    return await run_async(get_log_storage().poll_logs, project=project, request=request)
+    job_submission_logs = await run_async(
+        get_log_storage().poll_logs, project=project, request=request
+    )
+    # Logs are stored in plaintext but transmitted in base64 for API/CLI backward compatibility.
+    # Old logs stored in base64 are encoded twice for transmission and shown as base64 in CLI/UI.
+    # We live with that.
+    # TODO: Drop base64 encoding in 0.20.
+    for log_event in job_submission_logs.logs:
+        log_event.message = b64encode_raw_message(log_event.message.encode())
+    return job_submission_logs

dstack/_internal/server/services/logs/aws.py

@@ -17,7 +17,6 @@ from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
     LogStorageError,
-    b64encode_raw_message,
     datetime_to_unix_time_ms,
     unix_time_ms_to_datetime,
 )
@@ -56,6 +55,8 @@ class CloudWatchLogStorage(LogStorage):
     PAST_EVENT_MAX_DELTA = int((timedelta(days=14)).total_seconds()) * 1000 - CLOCK_DRIFT
     # "None of the log events in the batch can be more than 2 hours in the future."
     FUTURE_EVENT_MAX_DELTA = int((timedelta(hours=2)).total_seconds()) * 1000 - CLOCK_DRIFT
+    # Maximum number of retries when polling for log events to skip empty pages.
+    MAX_RETRIES = 10
 
     def __init__(self, *, group: str, region: Optional[str] = None) -> None:
         with self._wrap_boto_errors():
@@ -81,7 +82,7 @@ class CloudWatchLogStorage(LogStorage):
         next_token: Optional[str] = None
         with self._wrap_boto_errors():
             try:
-                cw_events, next_token = self._get_log_events(stream, request)
+                cw_events, next_token = self._get_log_events_with_retry(stream, request)
             except botocore.exceptions.ClientError as e:
                 if not self._is_resource_not_found_exception(e):
                     raise
@@ -102,7 +103,47 @@ class CloudWatchLogStorage(LogStorage):
             )
             for cw_event in cw_events
         ]
-        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
+
+    def _get_log_events_with_retry(
+        self, stream: str, request: PollLogsRequest
+    ) -> Tuple[List[_CloudWatchLogEvent], Optional[str]]:
+        current_request = request
+        previous_next_token = request.next_token
+
+        for attempt in range(self.MAX_RETRIES):
+            cw_events, next_token = self._get_log_events(stream, current_request)
+
+            if cw_events:
+                return cw_events, next_token
+
+            if not next_token or next_token == previous_next_token:
+                return [], None
+
+            previous_next_token = next_token
+            current_request = PollLogsRequest(
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
+                start_time=request.start_time,
+                end_time=request.end_time,
+                descending=request.descending,
+                next_token=next_token,
+                limit=request.limit,
+                diagnose=request.diagnose,
+            )
+
+        if not request.descending:
+            logger.debug(
+                "Stream %s: exhausted %d retries without finding logs, returning empty response",
+                stream,
+                self.MAX_RETRIES,
+            )
+        # Only return the next token after exhausting retries if going descending—
+        # AWS CloudWatch guarantees more logs in that case. In ascending mode,
+        # next token is always returned, even if no logs remain.
+        # So descending works reliably; ascending has limits if gaps are too large.
+        # In the future, UI/CLI should handle retries, and we can return next token for ascending too.
+        return [], next_token if request.descending else None
 
     def _get_log_events(
         self, stream: str, request: PollLogsRequest
@@ -116,7 +157,7 @@ class CloudWatchLogStorage(LogStorage):
         }
 
         if request.start_time:
-            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time) + 1
+            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time)
 
         if request.end_time:
             parameters["endTime"] = datetime_to_unix_time_ms(request.end_time)
@@ -238,8 +279,7 @@ class CloudWatchLogStorage(LogStorage):
                 skipped_future_events += 1
                 continue
             cw_event = self._runner_log_event_to_cloudwatch_event(event)
-            # as message is base64-encoded, length in bytes = length in code points.
-            message_size = len(cw_event["message"]) + self.MESSAGE_OVERHEAD_SIZE
+            message_size = len(event.message) + self.MESSAGE_OVERHEAD_SIZE
             if message_size > self.MESSAGE_MAX_SIZE:
                 # we should never hit this limit, as we use `io.Copy` to copy from pty to logs,
                 # which under the hood uses 32KiB buffer, see runner/internal/executor/executor.go,
@@ -271,7 +311,7 @@ class CloudWatchLogStorage(LogStorage):
     ) -> _CloudWatchLogEvent:
         return {
             "timestamp": runner_log_event.timestamp,
-            "message": b64encode_raw_message(runner_log_event.message),
+            "message": runner_log_event.message.decode(errors="replace"),
         }
 
     @contextmanager

dstack/_internal/server/services/logs/filelog.py

@@ -1,7 +1,9 @@
+import os
 from pathlib import Path
-from typing import List, Union
+from typing import Generator, List, Optional, Tuple, Union
 from uuid import UUID
 
+from dstack._internal.core.errors import ServerClientError
 from dstack._internal.core.models.logs import (
     JobSubmissionLogs,
     LogEvent,
@@ -14,8 +16,6 @@ from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
-    LogStorageError,
-    b64encode_raw_message,
     unix_time_ms_to_datetime,
 )
 
@@ -30,9 +30,6 @@ class FileLogStorage(LogStorage):
             self.root = Path(root)
 
     def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-        if request.descending:
-            raise LogStorageError("descending: true is not supported")
-
         log_producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
         log_file_path = self._get_log_file_path(
             project_name=project.name,
@@ -41,18 +38,17 @@ class FileLogStorage(LogStorage):
             producer=log_producer,
         )
 
+        if request.descending:
+            return self._poll_logs_descending(log_file_path, request)
+        else:
+            return self._poll_logs_ascending(log_file_path, request)
+
+    def _poll_logs_ascending(
+        self, log_file_path: Path, request: PollLogsRequest
+    ) -> JobSubmissionLogs:
         start_line = 0
         if request.next_token:
-            try:
-                start_line = int(request.next_token)
-                if start_line < 0:
-                    raise LogStorageError(
-                        f"Invalid next_token: {request.next_token}. Must be a non-negative integer."
-                    )
-            except ValueError:
-                raise LogStorageError(
-                    f"Invalid next_token: {request.next_token}. Must be a valid integer."
-                )
+            start_line = self._next_token(request)
 
         logs = []
         next_token = None
@@ -60,34 +56,140 @@ class FileLogStorage(LogStorage):
 
         try:
             with open(log_file_path) as f:
-                lines = f.readlines()
+                # Skip to start_line if needed
+                for _ in range(start_line):
+                    if f.readline() == "":
+                        # File is shorter than start_line
+                        return JobSubmissionLogs(logs=logs, next_token=next_token)
+                    current_line += 1
+
+                # Read lines one by one
+                while True:
+                    line = f.readline()
+                    if line == "":  # EOF
+                        break
 
-            for i, line in enumerate(lines):
-                if current_line < start_line:
                     current_line += 1
-                    continue
 
-                log_event = LogEvent.__response__.parse_raw(line)
-                current_line += 1
+                    try:
+                        log_event = LogEvent.__response__.parse_raw(line)
+                    except Exception:
+                        # Skip malformed lines
+                        continue
 
-                if request.start_time and log_event.timestamp <= request.start_time:
+                    if request.start_time and log_event.timestamp <= request.start_time:
+                        continue
+                    if request.end_time is not None and log_event.timestamp >= request.end_time:
+                        break
+
+                    logs.append(log_event)
+
+                    if len(logs) >= request.limit:
+                        # Check if there are more lines to read
+                        if f.readline() != "":
+                            next_token = str(current_line)
+                        break
+        except FileNotFoundError:
+            pass
+
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
+
+    def _poll_logs_descending(
+        self, log_file_path: Path, request: PollLogsRequest
+    ) -> JobSubmissionLogs:
+        start_offset = self._next_token(request)
+
+        candidate_logs = []
+
+        try:
+            line_generator = self._read_lines_reversed(log_file_path, start_offset)
+
+            for line_bytes, line_start_offset in line_generator:
+                try:
+                    line_str = line_bytes.decode("utf-8")
+                    log_event = LogEvent.__response__.parse_raw(line_str)
+                except Exception:
+                    continue  # Skip malformed lines
+
+                if request.end_time is not None and log_event.timestamp > request.end_time:
                     continue
-                if request.end_time is not None and log_event.timestamp >= request.end_time:
+                if request.start_time and log_event.timestamp <= request.start_time:
                     break
 
-                logs.append(log_event)
+                candidate_logs.append((log_event, line_start_offset))
 
-                if len(logs) >= request.limit:
-                    # Only set next_token if there are more lines to read
-                    if current_line < len(lines):
-                        next_token = str(current_line)
+                if len(candidate_logs) > request.limit:
                     break
+        except FileNotFoundError:
+            return JobSubmissionLogs(logs=[], next_token=None)
 
-        except IOError as e:
-            raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
+        logs = [log for log, offset in candidate_logs[: request.limit]]
+        next_token = None
+        if len(candidate_logs) > request.limit:
+            # We fetched one more than the limit, so there are more pages.
+            # The next token should point to the start of the last log we are returning.
+            _last_log_event, last_log_offset = candidate_logs[request.limit - 1]
+            next_token = str(last_log_offset)
 
         return JobSubmissionLogs(logs=logs, next_token=next_token)
 
+    @staticmethod
+    def _read_lines_reversed(
+        filepath: Path, start_offset: Optional[int] = None, chunk_size: int = 8192
+    ) -> Generator[Tuple[bytes, int], None, None]:
+        """
+        A generator that yields lines from a file in reverse order, along with the byte
+        offset of the start of each line. This is memory-efficient for large files.
+        """
+        with open(filepath, "rb") as f:
+            f.seek(0, os.SEEK_END)
+            file_size = f.tell()
+            cursor = file_size
+
+            # If a start_offset is provided, optimize by starting the read
+            # from a more specific location instead of the end of the file.
+            if start_offset is not None and start_offset < file_size:
+                # To get the full content of the line that straddles the offset,
+                # we need to find its end (the next newline character).
+                f.seek(start_offset)
+                chunk = f.read(chunk_size)
+                newline_pos = chunk.find(b"\n")
+                if newline_pos != -1:
+                    # Found the end of the line. The cursor for reverse reading
+                    # should start from this point to include the full line.
+                    cursor = start_offset + newline_pos + 1
+                else:
+                    # No newline found, which means the rest of the file is one line.
+                    # The default cursor pointing to file_size is correct.
+                    pass
+
+            buffer = b""
+
+            while cursor > 0:
+                seek_pos = max(0, cursor - chunk_size)
+                amount_to_read = cursor - seek_pos
+                f.seek(seek_pos)
+                chunk = f.read(amount_to_read)
+                cursor = seek_pos
+
+                buffer = chunk + buffer
+
+                while b"\n" in buffer:
+                    newline_pos = buffer.rfind(b"\n")
+                    line = buffer[newline_pos + 1 :]
+                    line_start_offset = cursor + newline_pos + 1
+
+                    # Skip lines that start at or after the start_offset
+                    if start_offset is None or line_start_offset < start_offset:
+                        yield line, line_start_offset
+
+                    buffer = buffer[:newline_pos]
+
+            # The remaining buffer is the first line of the file.
+            # Only yield it if we're not using start_offset or if it starts before start_offset
+            if buffer and (start_offset is None or 0 < start_offset):
+                yield buffer, 0
+
     def write_logs(
         self,
         project: ProjectModel,
@@ -140,5 +242,19 @@ class FileLogStorage(LogStorage):
         return LogEvent(
             timestamp=unix_time_ms_to_datetime(runner_log_event.timestamp),
             log_source=LogEventSource.STDOUT,
-            message=b64encode_raw_message(runner_log_event.message),
+            message=runner_log_event.message.decode(errors="replace"),
         )
+
+    def _next_token(self, request: PollLogsRequest) -> Optional[int]:
+        next_token = request.next_token
+        if next_token is None:
+            return None
+        try:
+            value = int(next_token)
+            if value < 0:
+                raise ValueError("Offset must be non-negative")
+            return value
+        except (ValueError, TypeError):
+            raise ServerClientError(
+                f"Invalid next_token: {next_token}. Must be a non-negative integer."
+            )

dstack/_internal/server/services/logs/gcp.py

@@ -14,7 +14,6 @@ from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
     LogStorageError,
-    b64encode_raw_message,
     unix_time_ms_to_datetime,
 )
 from dstack._internal.utils.common import batched
@@ -137,15 +136,14 @@ class GCPLogStorage(LogStorage):
         with self.logger.batch() as batcher:
             for batch in batched(logs, self.MAX_BATCH_SIZE):
                 for log in batch:
-                    message = b64encode_raw_message(log.message)
+                    message = log.message.decode(errors="replace")
                     timestamp = unix_time_ms_to_datetime(log.timestamp)
-                    # as message is base64-encoded, length in bytes = length in code points
-                    if len(message) > self.MAX_RUNNER_MESSAGE_SIZE:
+                    if len(log.message) > self.MAX_RUNNER_MESSAGE_SIZE:
                         logger.error(
                             "Stream %s: skipping event at %s, message exceeds max size: %d > %d",
                             stream_name,
                             timestamp.isoformat(),
-                            len(message),
+                            len(log.message),
                             self.MAX_RUNNER_MESSAGE_SIZE,
                         )
                         continue

dstack/_internal/server/services/prometheus/custom_metrics.py

@@ -1,4 +1,5 @@
 import itertools
+import json
 from collections import defaultdict
 from collections.abc import Generator, Iterable
 from datetime import timezone
@@ -177,6 +178,19 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
             metrics.add_sample(_JOB_CPU_TIME, labels, jmp.cpu_usage_micro / 1_000_000)
             metrics.add_sample(_JOB_MEMORY_USAGE, labels, jmp.memory_usage_bytes)
             metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
+            if gpus:
+                gpu_memory_total = gpus[0].memory_mib * 1024 * 1024
+                for gpu_num, (gpu_util, gpu_memory_usage) in enumerate(
+                    zip(
+                        json.loads(jmp.gpus_util_percent),
+                        json.loads(jmp.gpus_memory_usage_bytes),
+                    )
+                ):
+                    gpu_labels = labels.copy()
+                    gpu_labels["dstack_gpu_num"] = gpu_num
+                    metrics.add_sample(_JOB_GPU_USAGE_RATIO, gpu_labels, gpu_util / 100)
+                    metrics.add_sample(_JOB_GPU_MEMORY_TOTAL, gpu_labels, gpu_memory_total)
+                    metrics.add_sample(_JOB_GPU_MEMORY_USAGE, gpu_labels, gpu_memory_usage)
         jpm = job_prometheus_metrics.get(job.id)
         if jpm is not None:
             for metric in text_string_to_metric_families(jpm.text):
@@ -202,6 +216,9 @@ _JOB_CPU_TIME = "dstack_job_cpu_time_seconds_total"
 _JOB_MEMORY_TOTAL = "dstack_job_memory_total_bytes"
 _JOB_MEMORY_USAGE = "dstack_job_memory_usage_bytes"
 _JOB_MEMORY_WORKING_SET = "dstack_job_memory_working_set_bytes"
+_JOB_GPU_USAGE_RATIO = "dstack_job_gpu_usage_ratio"
+_JOB_GPU_MEMORY_TOTAL = "dstack_job_gpu_memory_total_bytes"
+_JOB_GPU_MEMORY_USAGE = "dstack_job_gpu_memory_usage_bytes"
 
 
 class _Metrics(dict[str, Metric]):
@@ -259,6 +276,9 @@ class _JobMetrics(_Metrics):
         (_JOB_MEMORY_TOTAL, _GAUGE, "Total memory allocated for the job, bytes"),
         (_JOB_MEMORY_USAGE, _GAUGE, "Memory used by the job (including cache), bytes"),
         (_JOB_MEMORY_WORKING_SET, _GAUGE, "Memory used by the job (not including cache), bytes"),
+        (_JOB_GPU_USAGE_RATIO, _GAUGE, "Job GPU usage, percent (as 0.0-1.0)"),
+        (_JOB_GPU_MEMORY_TOTAL, _GAUGE, "Total GPU memory allocated for the job, bytes"),
+        (_JOB_GPU_MEMORY_USAGE, _GAUGE, "GPU memory used by the job, bytes"),
     ]
 
 

dstack/_internal/server/services/proxy/repo.py

@@ -12,10 +12,12 @@ from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.instances import RemoteConnectionInfo, SSHConnectionParams
 from dstack._internal.core.models.runs import (
     JobProvisioningData,
+    JobSpec,
     JobStatus,
     RunSpec,
     RunStatus,
     ServiceSpec,
+    get_service_port,
 )
 from dstack._internal.core.models.services import AnyModel
 from dstack._internal.proxy.lib.models import (
@@ -97,9 +99,10 @@ class ServerProxyRepo(BaseProxyRepo):
                 if rci.ssh_proxy is not None:
                     ssh_head_proxy = rci.ssh_proxy
                     ssh_head_proxy_private_key = get_or_error(rci.ssh_proxy_keys)[0].private
+            job_spec: JobSpec = JobSpec.__response__.parse_raw(job.job_spec_data)
             replica = Replica(
                 id=job.id.hex,
-                app_port=run_spec.configuration.port.container_port,
+                app_port=get_service_port(job_spec, run_spec.configuration),
                 ssh_destination=ssh_destination,
                 ssh_port=ssh_port,
                 ssh_proxy=ssh_proxy,