dstack 0.0.9__py3-none-any.whl → 0.20.7__py3-none-any.whl

dstack/_internal/server/services/logs/fluentbit.py

@@ -0,0 +1,329 @@
+from datetime import datetime
+from typing import List, Optional, Protocol
+from uuid import UUID
+
+import httpx
+
+from dstack._internal.core.errors import ServerClientError
+from dstack._internal.core.models.logs import (
+    JobSubmissionLogs,
+    LogEvent,
+    LogEventSource,
+    LogProducer,
+)
+from dstack._internal.server.models import ProjectModel
+from dstack._internal.server.schemas.logs import PollLogsRequest
+from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
+from dstack._internal.server.services.logs.base import (
+    LogStorage,
+    LogStorageError,
+    unix_time_ms_to_datetime,
+)
+from dstack._internal.utils.common import batched
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+ELASTICSEARCH_AVAILABLE = True
+try:
+    from elasticsearch import Elasticsearch
+    from elasticsearch.exceptions import ApiError, TransportError
+except ImportError:
+    ELASTICSEARCH_AVAILABLE = False
+else:
+    ElasticsearchError: tuple = (ApiError, TransportError)  # type: ignore[misc]
+
+    class ElasticsearchReader:
+        """Reads logs from Elasticsearch or OpenSearch."""
+
+        def __init__(
+            self,
+            host: str,
+            index: str,
+            api_key: Optional[str] = None,
+        ) -> None:
+            if api_key:
+                self._client = Elasticsearch(hosts=[host], api_key=api_key)
+            else:
+                self._client = Elasticsearch(hosts=[host])
+            self._index = index
+            # Verify connection
+            try:
+                self._client.info()
+            except ElasticsearchError as e:
+                raise LogStorageError(f"Failed to connect to Elasticsearch/OpenSearch: {e}") from e
+
+        def read(
+            self,
+            stream_name: str,
+            request: PollLogsRequest,
+        ) -> JobSubmissionLogs:
+            sort_order = "desc" if request.descending else "asc"
+
+            query: dict = {
+                "bool": {
+                    "must": [
+                        {"term": {"stream.keyword": stream_name}},
+                    ]
+                }
+            }
+
+            if request.start_time:
+                query["bool"].setdefault("filter", []).append(
+                    {"range": {"@timestamp": {"gt": request.start_time.isoformat()}}}
+                )
+            if request.end_time:
+                query["bool"].setdefault("filter", []).append(
+                    {"range": {"@timestamp": {"lt": request.end_time.isoformat()}}}
+                )
+
+            search_params: dict = {
+                "index": self._index,
+                "query": query,
+                "sort": [
+                    {"@timestamp": {"order": sort_order}},
+                    {"_id": {"order": sort_order}},
+                ],
+                "size": request.limit,
+            }
+
+            if request.next_token:
+                parts = request.next_token.split(":", 1)
+                if len(parts) != 2 or not parts[0] or not parts[1]:
+                    raise ServerClientError(
+                        f"Invalid next_token: {request.next_token}. "
+                        "Must be in format 'timestamp:document_id'."
+                    )
+                search_params["search_after"] = [parts[0], parts[1]]
+
+            try:
+                response = self._client.search(**search_params)
+            except ElasticsearchError as e:
+                raise LogStorageError(f"Elasticsearch/OpenSearch error: {e}") from e
+
+            hits = response.get("hits", {}).get("hits", [])
+            logs = []
+            last_sort_values = None
+
+            for hit in hits:
+                source = hit.get("_source", {})
+                timestamp_str = source.get("@timestamp")
+                message = source.get("message", "")
+
+                if timestamp_str:
+                    try:
+                        timestamp = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
+                    except ValueError:
+                        continue
+                else:
+                    continue
+
+                logs.append(
+                    LogEvent(
+                        timestamp=timestamp,
+                        log_source=LogEventSource.STDOUT,
+                        message=message,
+                    )
+                )
+
+                sort_values = hit.get("sort")
+                if sort_values and len(sort_values) >= 2:
+                    last_sort_values = sort_values
+
+            next_token = None
+            if len(logs) == request.limit and last_sort_values is not None:
+                next_token = f"{last_sort_values[0]}:{last_sort_values[1]}"
+
+            return JobSubmissionLogs(
+                logs=logs,
+                next_token=next_token,
+            )
+
+        def close(self) -> None:
+            self._client.close()
+
+
+FLUENTBIT_AVAILABLE = True
+try:
+    from fluent import sender as fluent_sender
+except ImportError:
+    FLUENTBIT_AVAILABLE = False
+else:
+
+    class FluentBitWriter(Protocol):
+        def write(self, tag: str, records: List[dict]) -> None: ...
+        def close(self) -> None: ...
+
+    class LogReader(Protocol):
+        def read(self, stream_name: str, request: PollLogsRequest) -> JobSubmissionLogs: ...
+        def close(self) -> None: ...
+
+    class HTTPFluentBitWriter:
+        """Writes logs to Fluent-bit via HTTP POST."""
+
+        def __init__(self, host: str, port: int, tag_prefix: str) -> None:
+            self._endpoint = f"http://{host}:{port}"
+            self._client = httpx.Client(timeout=30.0)
+            self._tag_prefix = tag_prefix
+
+        def write(self, tag: str, records: List[dict]) -> None:
+            prefixed_tag = f"{self._tag_prefix}.{tag}" if self._tag_prefix else tag
+            for record in records:
+                try:
+                    response = self._client.post(
+                        f"{self._endpoint}/{prefixed_tag}",
+                        json=record,
+                        headers={"Content-Type": "application/json"},
+                    )
+                    response.raise_for_status()
+                except httpx.HTTPStatusError as e:
+                    raise LogStorageError(
+                        f"Fluent-bit HTTP error: status {e.response.status_code}"
+                    ) from e
+                except httpx.HTTPError as e:
+                    raise LogStorageError(f"Fluent-bit HTTP error: {e}") from e
+
+        def close(self) -> None:
+            self._client.close()
+
+    class ForwardFluentBitWriter:
+        """Writes logs to Fluent-bit using Forward protocol."""
+
+        def __init__(self, host: str, port: int, tag_prefix: str) -> None:
+            self._sender = fluent_sender.FluentSender(tag_prefix, host=host, port=port)
+            self._tag_prefix = tag_prefix
+
+        def write(self, tag: str, records: List[dict]) -> None:
+            for record in records:
+                if not self._sender.emit(tag, record):
+                    error = self._sender.last_error
+                    self._sender.clear_last_error()
+                    raise LogStorageError(f"Fluent-bit Forward error: {error}")
+
+        def close(self) -> None:
+            self._sender.close()
+
+    class NullLogReader:
+        """
+        Null reader for ship-only mode (no Elasticsearch/OpenSearch configured).
+
+        Returns empty logs. Useful when logs are shipped to an external system
+        that is accessed directly rather than through dstack.
+        """
+
+        def read(self, stream_name: str, request: PollLogsRequest) -> JobSubmissionLogs:
+            return JobSubmissionLogs(logs=[], next_token=None)
+
+        def close(self) -> None:
+            pass
+
+    class FluentBitLogStorage(LogStorage):
+        """
+        Log storage using Fluent-bit for writing and optionally Elasticsearch/OpenSearch for reading.
+
+        Supports two modes:
+        - Full mode: Writes to Fluent-bit and reads from Elasticsearch/OpenSearch
+        - Ship-only mode: Writes to Fluent-bit only (no reading, returns empty logs)
+        """
+
+        MAX_BATCH_SIZE = 100
+
+        def __init__(
+            self,
+            host: str,
+            port: int,
+            protocol: str,
+            tag_prefix: str,
+            es_host: Optional[str] = None,
+            es_index: str = "dstack-logs",
+            es_api_key: Optional[str] = None,
+        ) -> None:
+            self._tag_prefix = tag_prefix
+
+            if protocol == "http":
+                self._writer: FluentBitWriter = HTTPFluentBitWriter(
+                    host=host, port=port, tag_prefix=tag_prefix
+                )
+            elif protocol == "forward":
+                self._writer = ForwardFluentBitWriter(host=host, port=port, tag_prefix=tag_prefix)
+            else:
+                raise LogStorageError(f"Unsupported Fluent-bit protocol: {protocol}")
+
+            self._reader: LogReader
+            if es_host:
+                if not ELASTICSEARCH_AVAILABLE:
+                    raise LogStorageError(
+                        "Elasticsearch/OpenSearch host configured but elasticsearch package "
+                        "is not installed. Install with: pip install elasticsearch"
+                    )
+                self._reader = ElasticsearchReader(
+                    host=es_host,
+                    index=es_index,
+                    api_key=es_api_key,
+                )
+                logger.info(
+                    "Fluent-bit log storage initialized with Elasticsearch/OpenSearch reader"
+                )
+            else:
+                self._reader = NullLogReader()
+                logger.info(
+                    "Fluent-bit log storage initialized in ship-only mode "
+                    "(no Elasticsearch/OpenSearch configured for reading)"
+                )
+
+        def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
+            producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
+            stream_name = self._get_stream_name(
+                project_name=project.name,
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
+                producer=producer,
+            )
+            return self._reader.read(stream_name=stream_name, request=request)
+
+        def write_logs(
+            self,
+            project: ProjectModel,
+            run_name: str,
+            job_submission_id: UUID,
+            runner_logs: List[RunnerLogEvent],
+            job_logs: List[RunnerLogEvent],
+        ) -> None:
+            producers_with_logs = [(LogProducer.RUNNER, runner_logs), (LogProducer.JOB, job_logs)]
+            for producer, producer_logs in producers_with_logs:
+                if not producer_logs:
+                    continue
+                stream_name = self._get_stream_name(
+                    project_name=project.name,
+                    run_name=run_name,
+                    job_submission_id=job_submission_id,
+                    producer=producer,
+                )
+                self._write_logs_to_stream(stream_name=stream_name, logs=producer_logs)
+
+        def _write_logs_to_stream(self, stream_name: str, logs: List[RunnerLogEvent]) -> None:
+            for batch in batched(logs, self.MAX_BATCH_SIZE):
+                records = []
+                for log in batch:
+                    message = log.message.decode(errors="replace")
+                    timestamp = unix_time_ms_to_datetime(log.timestamp)
+                    records.append(
+                        {
+                            "message": message,
+                            "@timestamp": timestamp.isoformat(),
+                            "stream": stream_name,
+                        }
+                    )
+                self._writer.write(tag=stream_name, records=records)
+
+        def close(self) -> None:
+            try:
+                self._writer.close()
+            finally:
+                self._reader.close()
+
+        def _get_stream_name(
+            self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer
+        ) -> str:
+            return f"{project_name}/{run_name}/{job_submission_id}/{producer.value}"

dstack/_internal/server/services/logs/gcp.py

@@ -0,0 +1,181 @@
+import urllib.parse
+from typing import List
+from uuid import UUID
+
+from dstack._internal.core.errors import ServerClientError
+from dstack._internal.core.models.logs import (
+    JobSubmissionLogs,
+    LogEvent,
+    LogEventSource,
+    LogProducer,
+)
+from dstack._internal.server.models import ProjectModel
+from dstack._internal.server.schemas.logs import PollLogsRequest
+from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
+from dstack._internal.server.services.logs.base import (
+    LogStorage,
+    LogStorageError,
+    unix_time_ms_to_datetime,
+)
+from dstack._internal.utils.common import batched
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+GCP_LOGGING_AVAILABLE = True
+try:
+    import google.api_core.exceptions
+    import google.auth.exceptions
+    from google.cloud import logging_v2
+    from google.cloud.logging_v2.types import ListLogEntriesRequest
+except ImportError:
+    GCP_LOGGING_AVAILABLE = False
+else:
+
+    class GCPLogStorage(LogStorage):
+        # Max expected message size from runner is 32KB.
+        # Max expected LogEntry size is 32KB + metadata < 50KB < 256KB limit.
+        # With MAX_BATCH_SIZE = 100, max write request size < 5MB < 10 MB limit.
+        # See: https://cloud.google.com/logging/quotas.
+        MAX_RUNNER_MESSAGE_SIZE = 32 * 1024
+        MAX_BATCH_SIZE = 100
+
+        # Use the same log name for all run logs so that it's easy to manage all dstack-related logs.
+        LOG_NAME = "dstack-run-logs"
+        # Logs from different jobs belong to different "streams".
+        # GCP Logging has no built-in concepts of streams, so we implement them with labels.
+        # It should be fast to filter by labels since labels are indexed by default
+        # (https://cloud.google.com/logging/docs/analyze/custom-index).
+
+        def __init__(self, project_id: str):
+            self.project_id = project_id
+            try:
+                self.client = logging_v2.Client(project=project_id)
+                self.logger = self.client.logger(name=self.LOG_NAME)
+                self.logger.list_entries(max_results=1)
+                # Python client doesn't seem to support dry_run,
+                # so emit an empty log to check permissions.
+                self.logger.log_empty()
+            except google.auth.exceptions.DefaultCredentialsError:
+                raise LogStorageError("Default credentials not found")
+            except google.api_core.exceptions.NotFound:
+                raise LogStorageError(f"Project {project_id} not found")
+            except google.api_core.exceptions.PermissionDenied:
+                raise LogStorageError("Insufficient permissions")
+
+        def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
+            # TODO: GCP may return logs in random order when events have the same timestamp.
+            producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
+            stream_name = self._get_stream_name(
+                project_name=project.name,
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
+                producer=producer,
+            )
+            log_filters = [f'labels.stream = "{stream_name}"']
+            if request.start_time:
+                log_filters.append(f'timestamp > "{request.start_time.isoformat()}"')
+            if request.end_time:
+                log_filters.append(f'timestamp < "{request.end_time.isoformat()}"')
+            log_filter = " AND ".join(log_filters)
+
+            order_by = logging_v2.DESCENDING if request.descending else logging_v2.ASCENDING
+            try:
+                # Use low-level API to get access to next_page_token
+                request_obj = ListLogEntriesRequest(
+                    resource_names=[f"projects/{self.client.project}"],
+                    filter=log_filter,
+                    order_by=order_by,
+                    page_size=request.limit,
+                    page_token=request.next_token,
+                )
+                response = self.client._logging_api._gapic_api.list_log_entries(  # type: ignore[attr-defined]
+                    request=request_obj
+                )
+
+                logs = [
+                    LogEvent(
+                        timestamp=entry.timestamp,
+                        message=entry.json_payload.get("message"),
+                        log_source=LogEventSource.STDOUT,
+                    )
+                    for entry in response.entries
+                ]
+                next_token = response.next_page_token or None
+            except google.api_core.exceptions.ResourceExhausted as e:
+                logger.warning("GCP Logging exception: %s", repr(e))
+                # GCP Logging has severely low quota of 60 reads/min for entries.list
+                raise ServerClientError(
+                    "GCP Logging read request limit exceeded."
+                    " It's recommended to increase default entries.list request quota from 60 per minute."
+                )
+            return JobSubmissionLogs(
+                logs=logs,
+                external_url=self._get_stream_extrnal_url(stream_name),
+                next_token=next_token if len(logs) > 0 else None,
+            )
+
+        def write_logs(
+            self,
+            project: ProjectModel,
+            run_name: str,
+            job_submission_id: UUID,
+            runner_logs: List[RunnerLogEvent],
+            job_logs: List[RunnerLogEvent],
+        ):
+            producers_with_logs = [(LogProducer.RUNNER, runner_logs), (LogProducer.JOB, job_logs)]
+            for producer, producer_logs in producers_with_logs:
+                stream_name = self._get_stream_name(
+                    project_name=project.name,
+                    run_name=run_name,
+                    job_submission_id=job_submission_id,
+                    producer=producer,
+                )
+                self._write_logs_to_stream(
+                    stream_name=stream_name,
+                    logs=producer_logs,
+                )
+
+        def close(self):
+            self.client.close()
+
+        def _write_logs_to_stream(self, stream_name: str, logs: List[RunnerLogEvent]):
+            with self.logger.batch() as batcher:
+                for batch in batched(logs, self.MAX_BATCH_SIZE):
+                    for log in batch:
+                        message = log.message.decode(errors="replace")
+                        timestamp = unix_time_ms_to_datetime(log.timestamp)
+                        if len(log.message) > self.MAX_RUNNER_MESSAGE_SIZE:
+                            logger.error(
+                                "Stream %s: skipping event at %s, message exceeds max size: %d > %d",
+                                stream_name,
+                                timestamp.isoformat(),
+                                len(log.message),
+                                self.MAX_RUNNER_MESSAGE_SIZE,
+                            )
+                            continue
+                        batcher.log_struct(
+                            {
+                                "message": message,
+                            },
+                            labels={
+                                "stream": stream_name,
+                            },
+                            timestamp=timestamp,
+                        )
+                    batcher.commit()
+
+        def _get_stream_name(
+            self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer
+        ) -> str:
+            return f"{project_name}-{run_name}-{job_submission_id}-{producer.value}"
+
+        def _get_stream_extrnal_url(self, stream_name: str) -> str:
+            log_name_resource_name = self._get_log_name_resource_name()
+            query = f'logName="{log_name_resource_name}" AND labels.stream="{stream_name}"'
+            quoted_query = urllib.parse.quote(query, safe="")
+            return f"https://console.cloud.google.com/logs/query;query={quoted_query}?project={self.project_id}"
+
+        def _get_log_name_resource_name(self) -> str:
+            return f"projects/{self.project_id}/logs/{self.LOG_NAME}"

dstack/_internal/server/services/metrics.py

@@ -0,0 +1,172 @@
+import json
+from collections import defaultdict
+from collections.abc import Sequence
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.core.models.instances import Resources
+from dstack._internal.core.models.metrics import JobMetrics, Metric
+from dstack._internal.server.models import JobMetricsPoint, JobModel
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_or_error
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def get_job_metrics(
+    session: AsyncSession,
+    job_model: JobModel,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
+    limit: Optional[int] = None,
+) -> JobMetrics:
+    """
+    Returns metrics ordered from the latest to the earliest.
+
+    Expected usage:
+        * limit=100 — get the latest 100 points
+        * after=<now - 1 hour> — get points for the last one hour
+        * before=<earliest timestamp from the last batch>, limit=100 ­— paginate back in history
+    """
+    stmt = (
+        select(JobMetricsPoint)
+        .where(JobMetricsPoint.job_id == job_model.id)
+        .order_by(JobMetricsPoint.timestamp_micro.desc())
+    )
+    if after is not None:
+        # we need +1 point for cpu_usage_percent, thus >=
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro >= _datetime_to_unix_time_micro(after))
+    if before is not None:
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro < _datetime_to_unix_time_micro(before))
+    if limit is not None:
+        # +1 for cpu_usage_percent
+        stmt = stmt.limit(limit + 1)
+    res = await session.execute(stmt)
+    points = res.scalars().all()
+    # we need at least 2 points to calculate cpu_usage_percent
+    if len(points) < 2:
+        return JobMetrics(metrics=[])
+    return _calculate_job_metrics(job_model, points)
+
+
+def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint]) -> JobMetrics:
+    timestamps: list[datetime] = []
+    cpu_usage_points: list[int] = []
+    memory_usage_points: list[int] = []
+    memory_working_set_points: list[int] = []
+    gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list)
+    gpus_util_points: defaultdict[int, list[int]] = defaultdict(list)
+
+    cpus_detected_num: Optional[int] = None
+    memory_total: Optional[int] = None
+    gpu_memory_total: Optional[int] = None
+    resources: Optional[Resources] = None
+    jrd = get_job_runtime_data(job_model)
+    if jrd is not None and jrd.offer is not None:
+        resources = jrd.offer.instance.resources
+    else:
+        jpd = get_job_provisioning_data(job_model)
+        if jpd is not None:
+            resources = jpd.instance_type.resources
+    if resources is not None:
+        cpus_detected_num = resources.cpus
+        memory_total = resources.memory_mib * 1024 * 1024
+        if len(resources.gpus) > 0:
+            gpu_memory_total = resources.gpus[0].memory_mib * 1024 * 1024
+
+    gpus_detected_num: Optional[int] = None
+    gpus_detected_num_mismatch: bool = False
+    for point, prev_point in zip(points, points[1:]):
+        timestamps.append(_unix_time_micro_to_datetime(point.timestamp_micro))
+        cpu_usage_points.append(_get_cpu_usage(point, prev_point))
+        memory_usage_points.append(point.memory_usage_bytes)
+        memory_working_set_points.append(point.memory_working_set_bytes)
+        gpus_memory_usage = json.loads(point.gpus_memory_usage_bytes)
+        gpus_util = json.loads(point.gpus_util_percent)
+        if gpus_detected_num is None:
+            gpus_detected_num = len(gpus_memory_usage)
+        if len(gpus_memory_usage) != gpus_detected_num or len(gpus_util) != gpus_detected_num:
+            gpus_detected_num_mismatch = True
+        if not gpus_detected_num_mismatch:
+            for i in range(gpus_detected_num):
+                gpus_memory_usage_points[i].append(gpus_memory_usage[i])
+                gpus_util_points[i].append(gpus_util[i])
+
+    metrics: list[Metric] = [
+        Metric(
+            name="cpu_usage_percent",
+            timestamps=timestamps,
+            values=cpu_usage_points,
+        ),
+        Metric(
+            name="memory_usage_bytes",
+            timestamps=timestamps,
+            values=memory_usage_points,
+        ),
+        Metric(
+            name="memory_working_set_bytes",
+            timestamps=timestamps,
+            values=memory_working_set_points,
+        ),
+    ]
+    if cpus_detected_num is not None:
+        metrics.append(_make_constant_metric("cpus_detected_num", timestamps, cpus_detected_num))
+    if memory_total is not None:
+        metrics.append(_make_constant_metric("memory_total_bytes", timestamps, memory_total))
+    if gpus_detected_num_mismatch:
+        # If number of GPUs changed in the time window, skip GPU metrics altogether, otherwise
+        # results can be unpredictable (e.g, one GPU takes place of another, as they are
+        # identified by an array index only).
+        logger.warning("gpus_detected_num mismatch, skipping GPU metrics")
+    else:
+        metrics.append(
+            _make_constant_metric("gpus_detected_num", timestamps, get_or_error(gpus_detected_num))
+        )
+        if gpu_memory_total is not None:
+            metrics.append(
+                _make_constant_metric("gpu_memory_total_bytes", timestamps, gpu_memory_total)
+            )
+        for index, gpu_memory_usage_points in gpus_memory_usage_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_memory_usage_bytes_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_memory_usage_points,
+                )
+            )
+        for index, gpu_util_points in gpus_util_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_util_percent_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_util_points,
+                )
+            )
+    return JobMetrics(metrics=metrics)
+
+
+def _make_constant_metric(name: str, timestamps: list[datetime], value: float) -> Metric:
+    return Metric(
+        name=name,
+        timestamps=timestamps,
+        values=[value] * len(timestamps),
+    )
+
+
+def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int:
+    window = last_point.timestamp_micro - prev_point.timestamp_micro
+    if window == 0:
+        return 0
+    return round((last_point.cpu_usage_micro - prev_point.cpu_usage_micro) / window * 100)
+
+
+def _unix_time_micro_to_datetime(unix_time_ms: int) -> datetime:
+    return datetime.fromtimestamp(unix_time_ms / 1_000_000, tz=timezone.utc)
+
+
+def _datetime_to_unix_time_micro(dt: datetime) -> int:
+    return int(dt.timestamp() * 1_000_000)