matyan-client 0.1.0__tar.gz

matyan_client-0.1.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: matyan-client
+Version: 0.1.0
+Summary: Matyan SDK — Aim-compatible client for experiment tracking
+Author-email: Tigran Grigoryan <grigoryan.tigran119@gmail.com>
+Requires-Python: <4,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: httpx~=0.28.0
+Requires-Dist: loguru~=0.7.0
+Requires-Dist: matyan-api-models~=0.1.0
+Requires-Dist: psutil>=5.9
+Requires-Dist: pydantic~=2.0
+Requires-Dist: pydantic-settings~=2.0
+Requires-Dist: websockets~=15.0
+Provides-Extra: image
+Requires-Dist: Pillow>=9.0; extra == "image"
+Requires-Dist: numpy>=1.21; extra == "image"
+Provides-Extra: audio
+Requires-Dist: numpy>=1.21; extra == "audio"
+Provides-Extra: figure
+Requires-Dist: plotly>=5.0; extra == "figure"
+Provides-Extra: matplotlib
+Requires-Dist: matplotlib>=3.5; extra == "matplotlib"
+Requires-Dist: plotly>=5.0; extra == "matplotlib"
+Provides-Extra: numpy
+Requires-Dist: numpy>=1.21; extra == "numpy"
+Provides-Extra: msgpack
+Requires-Dist: msgpack>=1.0; extra == "msgpack"
+Provides-Extra: gpu
+Requires-Dist: nvidia-ml-py>=11.0; extra == "gpu"
+Provides-Extra: keras
+Requires-Dist: keras>=2.0; extra == "keras"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow>=2.0; extra == "tensorflow"
+Provides-Extra: pytorch
+Requires-Dist: torch>=1.9; extra == "pytorch"
+Provides-Extra: pytorch-lightning
+Requires-Dist: lightning>=2.0; python_version >= "3.10" and extra == "pytorch-lightning"
+Requires-Dist: omegaconf>=2.0; extra == "pytorch-lightning"
+Provides-Extra: pytorch-ignite
+Requires-Dist: pytorch-ignite>=0.4; extra == "pytorch-ignite"
+Requires-Dist: omegaconf>=2.0; extra == "pytorch-ignite"
+Provides-Extra: hugging-face
+Requires-Dist: transformers>=4.0; extra == "hugging-face"
+Provides-Extra: distributed-hugging-face
+Requires-Dist: transformers>=4.0; extra == "distributed-hugging-face"
+Requires-Dist: accelerate>=0.20; extra == "distributed-hugging-face"
+Provides-Extra: xgboost
+Requires-Dist: xgboost>=1.0; extra == "xgboost"
+Provides-Extra: lightgbm
+Requires-Dist: lightgbm>=3.0; extra == "lightgbm"
+Provides-Extra: catboost
+Requires-Dist: catboost>=1.0; extra == "catboost"
+Provides-Extra: optuna
+Requires-Dist: optuna>=3.0; extra == "optuna"
+Provides-Extra: keras-tuner
+Requires-Dist: keras-tuner>=1.0; extra == "keras-tuner"
+Provides-Extra: prophet
+Requires-Dist: prophet>=1.0; extra == "prophet"
+Provides-Extra: sb3
+Requires-Dist: stable-baselines3>=1.0; extra == "sb3"
+Provides-Extra: fastai
+Requires-Dist: fastai>=2.0; extra == "fastai"
+Requires-Dist: ipython; extra == "fastai"
+Provides-Extra: mxnet
+Requires-Dist: mxnet>=1.9; extra == "mxnet"
+Provides-Extra: adapters-all
+Requires-Dist: keras>=2.0; extra == "adapters-all"
+Requires-Dist: tensorflow>=2.0; extra == "adapters-all"
+Requires-Dist: torch>=1.9; extra == "adapters-all"
+Requires-Dist: lightning>=2.0; python_version >= "3.10" and extra == "adapters-all"
+Requires-Dist: pytorch-ignite>=0.4; extra == "adapters-all"
+Requires-Dist: omegaconf>=2.0; extra == "adapters-all"
+Requires-Dist: transformers>=4.0; extra == "adapters-all"
+Requires-Dist: accelerate>=0.20; extra == "adapters-all"
+Requires-Dist: xgboost>=1.0; extra == "adapters-all"
+Requires-Dist: lightgbm>=3.0; extra == "adapters-all"
+Requires-Dist: catboost>=1.0; extra == "adapters-all"
+Requires-Dist: optuna>=3.0; extra == "adapters-all"
+Requires-Dist: keras-tuner>=1.0; extra == "adapters-all"
+Requires-Dist: prophet>=1.0; extra == "adapters-all"
+Requires-Dist: stable-baselines3>=1.0; extra == "adapters-all"
+Requires-Dist: fastai>=2.0; extra == "adapters-all"
+Requires-Dist: ipython; extra == "adapters-all"
+Requires-Dist: mxnet>=1.9; extra == "adapters-all"
+Provides-Extra: extended
+Requires-Dist: Pillow>=9.0; extra == "extended"
+Requires-Dist: numpy>=1.21; extra == "extended"
+Requires-Dist: plotly>=5.0; extra == "extended"
+Requires-Dist: matplotlib>=3.5; extra == "extended"
+Requires-Dist: msgpack>=1.0; extra == "extended"
+Requires-Dist: nvidia-ml-py>=11.0; extra == "extended"

matyan_client-0.1.0/pyproject.toml

@@ -0,0 +1,129 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "matyan-client"
+version = "0.1.0"
+description = "Matyan SDK — Aim-compatible client for experiment tracking"
+readme = "README.md"
+authors = [
+    { name = "Tigran Grigoryan", email = "grigoryan.tigran119@gmail.com" }
+]
+requires-python = ">=3.10, <4"
+dependencies = [
+    "httpx~=0.28.0",
+    "loguru~=0.7.0",
+    "matyan-api-models~=0.1.0",
+    "psutil>=5.9",
+    "pydantic~=2.0",
+    "pydantic-settings~=2.0",
+    "websockets~=15.0",
+]
+
+[project.optional-dependencies]
+image = ["Pillow>=9.0", "numpy>=1.21"]
+audio = ["numpy>=1.21"]
+figure = ["plotly>=5.0"]
+matplotlib = ["matplotlib>=3.5", "plotly>=5.0"]
+numpy = ["numpy>=1.21"]
+msgpack = ["msgpack>=1.0"]
+gpu = ["nvidia-ml-py>=11.0"]
+# -- adapter framework extras --
+keras = ["keras>=2.0"]
+tensorflow = ["tensorflow>=2.0"]
+pytorch = ["torch>=1.9"]
+pytorch-lightning = ["lightning>=2.0; python_version>='3.10'", "omegaconf>=2.0"]
+pytorch-ignite = ["pytorch-ignite>=0.4", "omegaconf>=2.0"]
+hugging-face = ["transformers>=4.0"]
+distributed-hugging-face = ["transformers>=4.0", "accelerate>=0.20"]
+xgboost = ["xgboost>=1.0"]
+lightgbm = ["lightgbm>=3.0"]
+catboost = ["catboost>=1.0"]
+optuna = ["optuna>=3.0"]
+keras-tuner = ["keras-tuner>=1.0"]
+prophet = ["prophet>=1.0"]
+sb3 = ["stable-baselines3>=1.0"]
+# acme = ["dm-acme>=0.4"]
+fastai = ["fastai>=2.0", "ipython"]
+# paddle = ["paddlepaddle>=2.0"]
+mxnet = ["mxnet>=1.9"]
+adapters-all = [
+    "keras>=2.0",
+    "tensorflow>=2.0",
+    "torch>=1.9",
+    "lightning>=2.0; python_version>='3.10'",
+    "pytorch-ignite>=0.4",
+    "omegaconf>=2.0",
+    "transformers>=4.0",
+    "accelerate>=0.20",
+    "xgboost>=1.0",
+    "lightgbm>=3.0",
+    "catboost>=1.0",
+    "optuna>=3.0",
+    "keras-tuner>=1.0",
+    "prophet>=1.0",
+    "stable-baselines3>=1.0",
+    # "dm-acme>=0.4",
+    "fastai>=2.0",
+    "ipython",
+    # "paddlepaddle>=2.0",
+    "mxnet>=1.9",
+]
+extended = [
+    "Pillow>=9.0",
+    "numpy>=1.21",
+    "plotly>=5.0",
+    "matplotlib>=3.5",
+    "msgpack>=1.0",
+    "nvidia-ml-py>=11.0",
+]
+
+[dependency-groups]
+dev = [
+    "matyan-api-models",
+    "mypy~=1.15",
+    "pytest~=8.0",
+    "pytest-cov~=6.0",
+    "pytest-asyncio~=0.25",
+    "respx~=0.22",
+    "Pillow>=9.0",
+    "numpy>=1.21",
+    "plotly>=5.0",
+    "datasets~=4.6",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+exclude = ["*.ipynb"]
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "FBT001",
+    "FBT002",
+    "D100",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
+    "D105",
+    "D106",
+    "D107",
+    "D203",
+    "D213",
+    "D417",
+    "FBT003",
+    "EXE002",
+    "S311",
+    "TD003",
+    "D205",
+    "PLR2004",
+    "PLR0913",
+    "S101",
+    "S104"
+]
+
+[tool.uv.sources]
+matyan-api-models = { path = "../matyan-api-models", editable = true }

matyan_client-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

matyan_client-0.1.0/src/matyan_client/__init__.py

@@ -0,0 +1,15 @@
+"""Matyan Client SDK — Aim-compatible experiment tracking."""
+
+from .objects import Audio, Distribution, Figure, Image, Text
+from .repo import Repo
+from .run import Run
+
+__all__ = [
+    "Audio",
+    "Distribution",
+    "Figure",
+    "Image",
+    "Repo",
+    "Run",
+    "Text",
+]

matyan_client-0.1.0/src/matyan_client/_blob_uploader.py

@@ -0,0 +1,94 @@
+"""Background blob uploader using a thread pool.
+
+Presigned-URL acquisition and S3 PUT uploads run in worker threads so that
+``run.track(Image(...))`` returns immediately without blocking the training
+loop.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import TYPE_CHECKING
+
+import httpx
+from loguru import logger
+
+if TYPE_CHECKING:
+    from .transport.http import HttpTransport
+
+_MAX_WORKERS = 4
+
+
+class BlobUploader:
+    """Manages background S3 blob uploads via presigned URLs."""
+
+    def __init__(
+        self,
+        http: HttpTransport,
+        frontier_url: str,
+        run_id: str,
+    ) -> None:
+        self._http = http
+        self._frontier_url = frontier_url
+        self._run_id = run_id
+        self._pool = ThreadPoolExecutor(max_workers=_MAX_WORKERS, thread_name_prefix="blob-upload")
+        self._futures: list[Future[None]] = []
+
+    def submit(
+        self,
+        raw_bytes: bytes,
+        artifact_path: str,
+        content_type: str,
+    ) -> str:
+        """Submit a blob upload and return the deterministic ``s3_key`` immediately."""
+        s3_key = f"{self._run_id}/{artifact_path}"
+        fut = self._pool.submit(self._upload, raw_bytes, artifact_path, content_type, s3_key)
+        self._futures.append(fut)
+        return s3_key
+
+    def _upload(self, raw_bytes: bytes, artifact_path: str, content_type: str, s3_key: str) -> None:
+        try:
+            resp = self._http.presign_artifact(
+                self._frontier_url,
+                self._run_id,
+                artifact_path,
+                content_type=content_type,
+            )
+            upload_resp = httpx.put(
+                resp["upload_url"],
+                content=raw_bytes,
+                headers={"Content-Type": content_type},
+                timeout=120,
+            )
+            upload_resp.raise_for_status()
+        except httpx.HTTPError:
+            logger.exception("Background blob upload failed", s3_key=s3_key)
+
+    def drain(self, timeout: float = 60.0) -> int:
+        """Wait for all pending uploads to finish. Returns the number of failures."""
+        failures = 0
+        for fut in self._futures:
+            try:
+                fut.result(timeout=timeout)
+            except TimeoutError:  # noqa: PERF203
+                failures += 1
+                logger.warning("Blob upload timed out after {:.0f}s", timeout)
+            except httpx.HTTPError:
+                failures += 1
+                logger.exception("Blob upload failed with HTTP error")
+            except Exception:  # noqa: BLE001
+                failures += 1
+                logger.exception("Blob upload failed with unexpected error")
+        self._futures.clear()
+        return failures
+
+    @property
+    def pending(self) -> int:
+        return sum(1 for f in self._futures if not f.done())
+
+    def shutdown(self, timeout: float = 60.0) -> None:
+        """Drain pending uploads and shut down the thread pool."""
+        failures = self.drain(timeout=timeout)
+        if failures:
+            logger.warning("{} blob upload(s) failed during shutdown", failures)
+        self._pool.shutdown(wait=False)

matyan_client-0.1.0/src/matyan_client/_cache.py

@@ -0,0 +1,43 @@
+"""Client-side write cache for read-after-write consistency.
+
+All writes (track, __setitem__, property setters) are stored locally so
+that subsequent reads return the latest value even though ingestion to
+FoundationDB is asynchronous.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Any
+
+
+class WriteCache:
+    """Thread-safe in-memory cache keyed by ``(run_hash, dotted_path)``."""
+
+    def __init__(self) -> None:
+        self._data: dict[tuple[str, str], Any] = {}
+        self._lock = threading.Lock()
+
+    def set(self, run_hash: str, path: str, value: Any) -> None:  # noqa: ANN401
+        with self._lock:
+            self._data[(run_hash, path)] = value
+
+    def get(self, run_hash: str, path: str, default: Any = None) -> Any:  # noqa: ANN401
+        with self._lock:
+            return self._data.get((run_hash, path), default)
+
+    def has(self, run_hash: str, path: str) -> bool:
+        with self._lock:
+            return (run_hash, path) in self._data
+
+    def get_tree(self, run_hash: str) -> dict[str, Any]:
+        """Return all cached key-value pairs for *run_hash* as a flat dict."""
+        with self._lock:
+            return {path: val for (rh, path), val in self._data.items() if rh == run_hash}
+
+    def clear(self, run_hash: str | None = None) -> None:
+        with self._lock:
+            if run_hash is None:
+                self._data.clear()
+            else:
+                self._data = {k: v for k, v in self._data.items() if k[0] != run_hash}

matyan_client-0.1.0/src/matyan_client/_resource_tracker.py

@@ -0,0 +1,309 @@
+"""Background resource tracker for system metrics and terminal log capture.
+
+Adapted from ``aim/aim/ext/resource/tracker.py``.  Runs a single daemon
+thread that periodically collects CPU/memory/disk/GPU stats and (optionally)
+captures ``sys.stdout`` / ``sys.stderr`` output.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+import sys
+import time
+from threading import Event, Thread
+from typing import TYPE_CHECKING
+from weakref import WeakValueDictionary
+
+from loguru import logger
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+METRIC_PREFIX = "__system__"
+
+STAT_INTERVAL_MIN = 0.1
+STAT_INTERVAL_MAX = 86400.0  # 24 hours
+_LOG_CAPTURE_INTERVAL = 1.0  # seconds
+_TICK = 0.1  # main-loop sleep granularity
+
+_ANSI_CSI_RE = re.compile(rb"\001?\033\[((?:\d|;)*)([a-dA-D])\002?")
+
+
+class ResourceTracker:
+    """Collects system resource metrics and/or terminal logs in the background.
+
+    Parameters
+    ----------
+    track_fn:
+        ``(value, name, context) -> None`` — called for each metric sample.
+    interval:
+        Seconds between system-stat collections.  ``None`` disables stats.
+    capture_logs:
+        If ``True``, intercept ``sys.stdout`` / ``sys.stderr``.
+    send_terminal_line_fn:
+        ``(line, step) -> None`` — called for each captured terminal line.
+    log_offset:
+        Starting step number for terminal lines (for resume).
+
+    """
+
+    _buffer_registry: WeakValueDictionary[int, io.BytesIO] = WeakValueDictionary()
+    _old_out_write: Callable | None = None
+    _old_err_write: Callable | None = None
+    _patches_installed: bool = False
+
+    # ------------------------------------------------------------------
+    # Class-level stdout/stderr patching
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def _install_stream_patches(cls) -> None:
+        if cls._patches_installed:
+            return
+        cls._old_out_write = sys.stdout.write
+        cls._old_err_write = sys.stderr.write
+
+        def _new_out_write(data: str) -> int:
+            result = cls._old_out_write(data)  # type: ignore[misc]
+            raw = data.encode() if isinstance(data, str) else data
+            for buf in cls._buffer_registry.values():
+                buf.write(raw)
+            return result
+
+        def _new_err_write(data: str) -> int:
+            result = cls._old_err_write(data)  # type: ignore[misc]
+            raw = data.encode() if isinstance(data, str) else data
+            for buf in cls._buffer_registry.values():
+                buf.write(raw)
+            return result
+
+        sys.stdout.write = _new_out_write  # type: ignore[assignment]
+        sys.stderr.write = _new_err_write  # type: ignore[assignment]
+        cls._patches_installed = True
+
+    @classmethod
+    def _uninstall_stream_patches(cls) -> None:
+        if not cls._patches_installed:
+            return
+        if cls._old_out_write is not None:
+            sys.stdout.write = cls._old_out_write  # type: ignore[assignment]
+        if cls._old_err_write is not None:
+            sys.stderr.write = cls._old_err_write  # type: ignore[assignment]
+        cls._patches_installed = False
+
+    # ------------------------------------------------------------------
+    # Validation
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def check_interval(cls, interval: float | None, *, warn: bool = True) -> bool:
+        """Return ``True`` if *interval* is a valid stat-collection interval."""
+        if interval is None:
+            return False
+        if not isinstance(interval, (int, float)) or not (STAT_INTERVAL_MIN <= interval <= STAT_INTERVAL_MAX):
+            if warn:
+                logger.warning(
+                    "system_tracking_interval must be between {} and {} seconds",
+                    STAT_INTERVAL_MIN,
+                    STAT_INTERVAL_MAX,
+                )
+            return False
+        return True
+
+    # ------------------------------------------------------------------
+    # Init / lifecycle
+    # ------------------------------------------------------------------
+
+    def __init__(
+        self,
+        *,
+        track_fn: Callable[[float, str, dict | None], None],
+        interval: float | None = None,
+        capture_logs: bool = False,
+        send_terminal_line_fn: Callable[[str, int], None] | None = None,
+        log_offset: int = 0,
+    ) -> None:
+        self._track_fn = track_fn
+        self._stat_interval: float | None = interval if self.check_interval(interval, warn=False) else None
+        self._capture_logs = capture_logs
+        self._send_line_fn = send_terminal_line_fn
+        self._line_counter = log_offset
+
+        self._io_buffer = io.BytesIO()
+        self._shutdown = Event()
+        self._started = False
+        self._thread: Thread | None = None
+
+        self._process = None
+        if self._stat_interval is not None:
+            try:
+                import psutil  # noqa: PLC0415
+
+                self._process = psutil.Process()
+                psutil.cpu_percent(0.0)
+            except Exception:  # noqa: BLE001
+                logger.debug("psutil unavailable; system stats disabled")
+                self._stat_interval = None
+
+    def start(self) -> None:
+        if self._started:
+            return
+        if self._stat_interval is None and not self._capture_logs:
+            return
+        self._started = True
+        if self._capture_logs:
+            if not self._buffer_registry:
+                self._install_stream_patches()
+            self._buffer_registry[id(self)] = self._io_buffer
+        self._thread = Thread(target=self._loop, daemon=True, name="matyan-resource-tracker")
+        self._thread.start()
+
+    def stop(self) -> None:
+        if not self._started:
+            return
+        self._shutdown.set()
+        if self._thread is not None:
+            self._thread.join(timeout=5)
+        if self._capture_logs:
+            self._flush_logs()
+            self._buffer_registry.pop(id(self), None)
+            if not self._buffer_registry:
+                self._uninstall_stream_patches()
+        self._started = False
+
+    # ------------------------------------------------------------------
+    # Main collection loop
+    # ------------------------------------------------------------------
+
+    def _loop(self) -> None:
+        stat_elapsed = 0.0
+        log_elapsed = 0.0
+
+        if self._stat_interval is not None:
+            self._collect_stats()
+
+        while not self._shutdown.is_set():
+            time.sleep(_TICK)
+            stat_elapsed += _TICK
+            log_elapsed += _TICK
+
+            if self._stat_interval is not None and stat_elapsed >= self._stat_interval:
+                self._collect_stats()
+                stat_elapsed = 0.0
+
+            if self._capture_logs and log_elapsed >= _LOG_CAPTURE_INTERVAL:
+                self._flush_logs()
+                log_elapsed = 0.0
+
+    # ------------------------------------------------------------------
+    # System stats
+    # ------------------------------------------------------------------
+
+    def _collect_stats(self) -> None:
+        try:
+            self._collect_system_stats()
+        except Exception:  # noqa: BLE001
+            logger.debug("Error collecting system stats")
+        try:
+            self._collect_gpu_stats()
+        except Exception:  # noqa: BLE001
+            logger.debug("GPU stats collection failed", exc_info=True)
+
+    def _collect_system_stats(self) -> None:
+        import psutil  # noqa: PLC0415
+
+        proc = self._process
+        if proc is None:
+            return
+
+        metrics = {
+            "cpu": round(proc.cpu_percent(0.0), 5),
+            "p_memory_percent": round(proc.memory_percent(), 5),
+            "memory_percent": round(psutil.virtual_memory().percent, 5),
+            "disk_percent": round(psutil.disk_usage("/").percent, 5),
+        }
+        for name, value in metrics.items():
+            self._track_fn(value, f"{METRIC_PREFIX}{name}", None)
+
+    def _collect_gpu_stats(self) -> None:
+        try:
+            import pynvml  # noqa: PLC0415  # pyright: ignore[reportMissingImports]
+        except ImportError:
+            logger.debug("pynvml unavailable; GPU stats disabled")
+            return
+
+        try:
+            pynvml.nvmlInit()
+        except pynvml.NVMLError:
+            logger.debug("NVML initialization failed", exc_info=True)
+            return
+
+        try:
+            count = pynvml.nvmlDeviceGetCount()
+            for idx in range(count):
+                handle = pynvml.nvmlDeviceGetHandleByIndex(idx)
+                ctx = {"gpu": idx}
+                try:
+                    util = pynvml.nvmlDeviceGetUtilizationRates(handle)
+                    self._track_fn(round(util.gpu, 5), f"{METRIC_PREFIX}gpu", ctx)
+                except pynvml.NVMLError:
+                    logger.debug("GPU {} utilization query failed", idx, exc_info=True)
+                try:
+                    mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
+                    pct = round(mem.used * 100 / mem.total, 5) if mem.total else 0.0
+                    self._track_fn(pct, f"{METRIC_PREFIX}gpu_memory_percent", ctx)
+                except pynvml.NVMLError:
+                    logger.debug("GPU {} memory query failed", idx, exc_info=True)
+                try:
+                    temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
+                    self._track_fn(round(float(temp), 5), f"{METRIC_PREFIX}gpu_temp", ctx)
+                except pynvml.NVMLError:
+                    logger.debug("GPU {} temperature query failed", idx, exc_info=True)
+                try:
+                    power = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000
+                    self._track_fn(round(power, 5), f"{METRIC_PREFIX}gpu_power_watts", ctx)
+                except pynvml.NVMLError:
+                    logger.debug("GPU {} power query failed", idx, exc_info=True)
+        finally:
+            try:
+                pynvml.nvmlShutdown()
+            except pynvml.NVMLError:
+                logger.debug("NVML shutdown failed", exc_info=True)
+
+    # ------------------------------------------------------------------
+    # Terminal log capture
+    # ------------------------------------------------------------------
+
+    def _flush_logs(self) -> None:
+        if self._send_line_fn is None:
+            return
+        buf_size = self._io_buffer.tell()
+        if not buf_size:
+            return
+
+        self._io_buffer.seek(0)
+        data = self._io_buffer.read(buf_size)
+        self._io_buffer.seek(0)
+        self._io_buffer.truncate()
+
+        lines = data.split(b"\n")
+        for line in lines:
+            line = self._strip_ansi(line)  # noqa: PLW2901
+            line = line.rsplit(b"\r", maxsplit=1)[-1]  # noqa: PLW2901
+            try:
+                text = line.decode("utf-8", errors="replace")
+            except Exception:  # noqa: BLE001
+                logger.debug("Failed to decode terminal line", exc_info=True)
+                continue
+            self._send_line_fn(text, self._line_counter)
+            self._line_counter += 1
+
+        # last line without trailing newline stays in counter
+        self._line_counter -= 1
+        if lines[-1] != b"":
+            self._io_buffer.write(lines[-1])
+
+    @staticmethod
+    def _strip_ansi(line: bytes) -> bytes:
+        return re.sub(_ANSI_CSI_RE, b"", line)