euler-train 2.2.0__tar.gz → 2.4.0__tar.gz

{euler_train-2.2.0 → euler_train-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.2.0
+Version: 2.4.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy

{euler_train-2.2.0 → euler_train-2.4.0}/euler_train/__init__.py

@@ -6,8 +6,24 @@ from typing import Any
 from . import _dataset_contract  # noqa: F401
 from .run import Run
 from .architecture import export_architecture
+from .stream import (
+    EulerViewStreamConfig,
+    EulerViewStreamConsumer,
+    OutputStream,
+    OutputStreamConsumer,
+    StreamContext,
+)
 
-__all__ = ["init", "Run", "export_architecture"]
+__all__ = [
+    "EulerViewStreamConfig",
+    "EulerViewStreamConsumer",
+    "OutputStream",
+    "OutputStreamConsumer",
+    "StreamContext",
+    "init",
+    "Run",
+    "export_architecture",
+]
 __version__ = "0.1.0"
 
 
@@ -22,6 +38,7 @@ def init(
     run_name: str | None = None,
     evaluations: dict[str, dict] | None = None,
     mode: str | None = None,
+    stream: Any = None,
 ) -> Run:
     """Create a new run — or resume an existing one — and return the handle.
 
@@ -89,6 +106,12 @@ def init(
         ``"train"``, ``"val"``, or ``"eval"``).  When provided,
         lifecycle fields and crash details are mirrored into
         ``meta.json`` under ``modes[mode]``.
+    stream:
+        Optional output-stream consumer or config. Passing a mapping
+        with ``base_url`` plus either ``stream_token`` or the pair
+        ``model_id`` / ``api_token`` (or legacy ``access_token``)
+        enables best-effort dual-write streaming to the Euler View
+        model-run ingest API.
     """
     return Run(
         dir=dir, config=config, meta=meta,
@@ -97,4 +120,5 @@ def init(
         run_id=run_id,
         datasets=datasets, run_name=run_name,
         evaluations=evaluations, mode=mode,
+        stream=stream,
     )

{euler_train-2.2.0 → euler_train-2.4.0}/euler_train/run.py

@@ -2,15 +2,16 @@
 from __future__ import annotations
 
 import atexit
+import copy
 import logging
 import os
 import re
 import secrets
-import warnings
 import signal
 import sys
 import time
 import traceback
+import warnings
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
@@ -20,6 +21,7 @@ from .outputs import save_output_tree
 from .slurm import get_slurm_info
 from .git_info import get_code_ref
 from .environment import get_run_environment
+from .stream import OutputStream, StreamContext, coerce_output_stream
 
 log = logging.getLogger("euler_train")
 
@@ -45,6 +47,7 @@ class Run:
         run_name: str | None = None,
         evaluations: dict[str, dict[str, Any]] | None = None,
         mode: str | None = None,
+        stream: Any = None,
     ) -> None:
         # ── detect run-directory shorthand ────────────────────────
         # When `dir` points to an existing run directory (contains
@@ -83,6 +86,8 @@ class Run:
         self.run_name: str | None = run_name
         self.mode: str | None = _normalize_mode(mode)
         self._pending_updated_at: dict[str, dict[str, Any]] = {}
+        self._stream: OutputStream | None = coerce_output_stream(stream)
+        self._stream_event_counters: dict[str, int] = {}
 
         # ── config ────────────────────────────────────────────────
         if resuming:
@@ -97,14 +102,25 @@ class Run:
         write_json(config_path, self.config)
         self._record_artifact_update(config_path)
 
-        # ── code ref & run environment (fresh runs only) ─────────
-        if not resuming:
-            code_ref_path = self.dir / "code_ref.json"
-            write_json(code_ref_path, get_code_ref())
+        # ── code ref & run environment ───────────────────────────
+        code_ref_path = self.dir / "code_ref.json"
+        run_environment_path = self.dir / "run_environment.json"
+        if resuming:
+            self._code_ref_payload = (
+                read_json(code_ref_path) if code_ref_path.exists() else None
+            )
+            self._run_environment_payload = (
+                read_json(run_environment_path)
+                if run_environment_path.exists()
+                else None
+            )
+        else:
+            self._code_ref_payload = get_code_ref()
+            write_json(code_ref_path, self._code_ref_payload)
             self._record_artifact_update(code_ref_path)
 
-            run_environment_path = self.dir / "run_environment.json"
-            write_json(run_environment_path, get_run_environment())
+            self._run_environment_payload = get_run_environment()
+            write_json(run_environment_path, self._run_environment_payload)
             self._record_artifact_update(run_environment_path)
 
         # ── meta ──────────────────────────────────────────────────
@@ -161,6 +177,8 @@ class Run:
             )
         self._mark_mode_running()
         self._flush_meta()
+        self._bind_stream()
+        self._emit_stream_init()
         self._setup_hooks()
 
         verb = "Resumed" if resuming else "Started"
@@ -202,6 +220,19 @@ class Run:
             existing=self._meta.get("evaluations"),
         )
         self._flush_meta()
+        evaluations = self._meta.get("evaluations", {})
+        if isinstance(evaluations, dict) and key in evaluations:
+            self._emit_stream_event(
+                {
+                    "type": "meta",
+                    "eventId": self._make_stream_event_id("evaluation", key),
+                    "patch": {
+                        "evaluations": {
+                            key: copy.deepcopy(evaluations[key]),
+                        },
+                    },
+                },
+            )
 
     def finish_evaluation(self, key: str, status: str = "completed") -> None:
         """Mark an existing evaluation as finished.
@@ -216,6 +247,23 @@ class Run:
             )
         evals[key]["status"] = status
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "meta",
+                "eventId": self._make_stream_event_id(
+                    "evaluation_status",
+                    key,
+                    status,
+                ),
+                "patch": {
+                    "evaluations": {
+                        key: {
+                            "status": status,
+                        },
+                    },
+                },
+            },
+        )
 
     # ── logging ───────────────────────────────────────────────────
 
@@ -250,6 +298,15 @@ class Run:
         append_jsonl(log_path, record)
         self._record_artifact_update(log_path)
         self._flush_meta()
+        split = "train" if mode == "train" else "val"
+        self._emit_stream_event(
+            {
+                "type": "metric",
+                "eventId": self._make_stream_event_id(split, epoch, step),
+                "split": split,
+                "records": [copy.deepcopy(record)],
+            },
+        )
 
     # ── GPU stats ─────────────────────────────────────────────────
 
@@ -327,6 +384,19 @@ class Run:
         if base.exists():
             self._record_artifact_update(base)
             self._flush_meta()
+            snapshot = self._build_output_snapshot(epoch, step, output_types)
+            if snapshot is not None:
+                self._emit_stream_event(
+                    {
+                        "type": "output_snapshot",
+                        "eventId": self._make_stream_event_id(
+                            "outputs",
+                            epoch,
+                            step,
+                        ),
+                        "snapshot": snapshot,
+                    },
+                )
         return base
 
     # ── checkpoints ───────────────────────────────────────────────
@@ -450,6 +520,17 @@ class Run:
         self._meta["checkpoints"] = checkpoints
         self._record_artifact_update(checkpoint_path)
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "checkpoint",
+                "eventId": self._make_stream_event_id(
+                    "checkpoint",
+                    epoch,
+                    step,
+                ),
+                "checkpoint": copy.deepcopy(record),
+            },
+        )
 
     # ── architecture export ─────────────────────────────────────────
 
@@ -508,6 +589,16 @@ class Run:
         self._clear_terminal_fields(self._meta, status=status)
         self._finish_mode(status=status, end=end, duration=duration)
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "finish",
+                "eventId": self._make_stream_event_id("finish", status),
+                "status": status,
+                "patch": self._build_finish_patch(),
+            },
+            flush=True,
+        )
+        self._close_stream()
 
     def detach(self) -> None:
         """Disconnect from the run without changing its status.
@@ -527,6 +618,7 @@ class Run:
         self._finished = True
         self._teardown_hooks()
         self._flush_meta()
+        self._close_stream()
 
     # ── context manager ───────────────────────────────────────────
 
@@ -712,6 +804,136 @@ class Run:
         self._record_artifact_update(self.dir / "meta.json", when=time.time())
         write_json(self.dir / "meta.json", self._meta)
 
+    def _bind_stream(self) -> None:
+        if self._stream is None:
+            return
+        self._stream.bind(
+            StreamContext(
+                run_id=self.run_id,
+                project_dir=self.project_dir,
+                runs_dir=self.dir.parent,
+                run_dir=self.dir,
+                meta=copy.deepcopy(self._meta),
+                config=copy.deepcopy(self.config),
+                code_ref=copy.deepcopy(self._code_ref_payload),
+                run_environment=copy.deepcopy(self._run_environment_payload),
+            ),
+        )
+
+    def _emit_stream_init(self) -> None:
+        tags = self._extract_stream_tags()
+        self._emit_stream_event(
+            {
+                "type": "init",
+                "eventId": self._make_stream_event_id("init"),
+                "meta": copy.deepcopy(self._meta),
+                "config": copy.deepcopy(self.config),
+                "codeRef": copy.deepcopy(self._code_ref_payload),
+                "runEnvironment": copy.deepcopy(self._run_environment_payload),
+                "tags": tags,
+            },
+            flush=True,
+        )
+
+    def _emit_stream_event(
+        self,
+        event: dict[str, Any],
+        *,
+        flush: bool = False,
+    ) -> None:
+        if self._stream is None:
+            return
+        self._stream.emit(event)
+        if flush:
+            self._stream.flush()
+
+    def _close_stream(self) -> None:
+        if self._stream is None:
+            return
+        self._stream.close()
+
+    def _extract_stream_tags(self) -> list[str] | None:
+        raw_tags = self._meta.get("tags")
+        if not isinstance(raw_tags, list):
+            return None
+        tags = [
+            str(tag).strip()
+            for tag in raw_tags
+            if isinstance(tag, str) and str(tag).strip()
+        ]
+        return tags or None
+
+    def _build_output_snapshot(
+        self,
+        epoch: int | None,
+        step: int | None,
+        output_types: dict[str, Any],
+    ) -> dict[str, Any] | None:
+        if epoch is None or step is None:
+            return None
+
+        snapshot_types: dict[str, list[str]] = {}
+        for output_type, slots in output_types.items():
+            if slots is None or not isinstance(slots, Mapping):
+                continue
+
+            categories: list[str] = []
+            for slot_name, data in slots.items():
+                if data is None:
+                    continue
+                if slot_name == "aux" and isinstance(data, Mapping):
+                    for aux_name, aux_value in data.items():
+                        if aux_value is None:
+                            continue
+                        categories.append(f"aux/{aux_name}")
+                else:
+                    categories.append(str(slot_name))
+
+            if categories:
+                snapshot_types[str(output_type)] = sorted(set(categories))
+
+        if not snapshot_types:
+            return None
+
+        return {
+            "epoch": epoch,
+            "step": step,
+            "types": snapshot_types,
+        }
+
+    def _build_finish_patch(self) -> dict[str, Any]:
+        patch = {
+            "end_time": self._meta.get("end_time"),
+            "end_iso": self._meta.get("end_iso"),
+            "duration_sec": self._meta.get("duration_sec"),
+        }
+
+        if "error" in self._meta:
+            patch["error"] = self._meta["error"]
+        if "traceback" in self._meta:
+            patch["traceback"] = self._meta["traceback"]
+
+        if self.mode is not None:
+            modes = self._meta.get("modes")
+            if isinstance(modes, dict) and self.mode in modes:
+                patch["modes"] = {
+                    self.mode: copy.deepcopy(modes[self.mode]),
+                }
+
+        return patch
+
+    def _make_stream_event_id(self, prefix: str, *parts: Any) -> str:
+        counter = self._stream_event_counters.get(prefix, 0)
+        self._stream_event_counters[prefix] = counter + 1
+        tokens = [prefix]
+        tokens.extend(
+            str(part)
+            for part in parts
+            if part is not None and str(part).strip()
+        )
+        tokens.append(str(counter))
+        return ":".join(tokens)
+
     def __repr__(self) -> str:
         return f"Run(id={self.run_id!r}, dir={str(self.dir)!r}, status={self._meta['status']!r})"
 

euler_train-2.4.0/euler_train/stream.py

@@ -0,0 +1,572 @@
+"""Optional model-run stream consumers for Euler View."""
+from __future__ import annotations
+
+import copy
+import json
+import logging
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Protocol
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+from .serialization import _json_default
+
+log = logging.getLogger("euler_train.stream")
+
+
+class OutputStreamConsumer(Protocol):
+    """Minimal interface for best-effort event consumers."""
+
+    def bind(self, context: "StreamContext") -> None:
+        ...
+
+    def emit(self, event: dict[str, Any]) -> None:
+        ...
+
+    def flush(self) -> None:
+        ...
+
+    def close(self) -> None:
+        ...
+
+
+@dataclass
+class StreamContext:
+    run_id: str
+    project_dir: Path
+    runs_dir: Path
+    run_dir: Path
+    meta: dict[str, Any]
+    config: dict[str, Any]
+    code_ref: dict[str, Any] | None
+    run_environment: dict[str, Any] | None
+
+
+@dataclass
+class EulerViewStreamConfig:
+    """Connection settings for the built-in Euler View HTTP consumer."""
+
+    base_url: str
+    model_id: int | None = None
+    api_token: str | None = None
+    access_token: str | None = None
+    stream_token: str | None = None
+    datasource_id: int | None = None
+    euler_train_dir: str | None = None
+    run_dir: str | None = None
+    timeout_sec: float = 10.0
+    batch_size: int = 20
+    flush_interval_sec: float = 2.0
+    max_pending_events: int = 1000
+    session_expires_in_sec: int | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.base_url, str):
+            raise ValueError("stream.base_url must be a string")
+        self.base_url = self.base_url.strip().rstrip("/")
+        if not self.base_url:
+            raise ValueError("stream.base_url is required")
+
+        self.access_token = _normalize_optional_text(
+            self.access_token,
+            field_name="stream.access_token",
+        )
+        self.api_token = _normalize_optional_text(
+            self.api_token,
+            field_name="stream.api_token",
+        )
+        self.stream_token = _normalize_optional_text(
+            self.stream_token,
+            field_name="stream.stream_token",
+        )
+        self.euler_train_dir = _normalize_optional_text(
+            self.euler_train_dir,
+            field_name="stream.euler_train_dir",
+        )
+        self.run_dir = _normalize_optional_text(
+            self.run_dir,
+            field_name="stream.run_dir",
+        )
+
+        self.model_id = _require_positive_int(
+            self.model_id,
+            field_name="stream.model_id",
+            allow_none=True,
+        )
+        self.datasource_id = _require_positive_int(
+            self.datasource_id,
+            field_name="stream.datasource_id",
+            allow_none=True,
+        )
+        self.session_expires_in_sec = _require_positive_int(
+            self.session_expires_in_sec,
+            field_name="stream.session_expires_in_sec",
+            allow_none=True,
+        )
+        self.batch_size = _require_positive_int(
+            self.batch_size,
+            field_name="stream.batch_size",
+        )
+        self.max_pending_events = _require_positive_int(
+            self.max_pending_events,
+            field_name="stream.max_pending_events",
+        )
+        self.timeout_sec = _require_positive_number(
+            self.timeout_sec,
+            field_name="stream.timeout_sec",
+        )
+        self.flush_interval_sec = _require_non_negative_number(
+            self.flush_interval_sec,
+            field_name="stream.flush_interval_sec",
+        )
+
+        if self.stream_token is None:
+            if self.model_id is None:
+                raise ValueError(
+                    "stream.model_id is required when stream.stream_token is not provided",
+                )
+            if self.api_token is None and self.access_token is None:
+                raise ValueError(
+                    "stream.api_token or stream.access_token is required when stream.stream_token is not provided",
+                )
+
+
+class OutputStream:
+    """Dispatch events to one or more best-effort consumers."""
+
+    def __init__(self, consumers: Sequence[OutputStreamConsumer]) -> None:
+        self._consumers = list(consumers)
+        self._closed = False
+
+    @property
+    def consumers(self) -> tuple[OutputStreamConsumer, ...]:
+        return tuple(self._consumers)
+
+    def bind(self, context: StreamContext) -> None:
+        self._dispatch("bind", context)
+
+    def emit(self, event: dict[str, Any]) -> None:
+        self._dispatch("emit", event)
+
+    def flush(self) -> None:
+        self._dispatch("flush")
+
+    def close(self) -> None:
+        if self._closed:
+            return
+        self._dispatch("flush")
+        self._dispatch("close")
+        self._closed = True
+
+    def _dispatch(self, method_name: str, *args: Any) -> None:
+        if self._closed and method_name != "close":
+            return
+
+        active: list[OutputStreamConsumer] = []
+        for consumer in self._consumers:
+            method = getattr(consumer, method_name)
+            payload = copy.deepcopy(args)
+            try:
+                method(*payload)
+                active.append(consumer)
+            except Exception as exc:  # pragma: no cover - defensive safeguard
+                log.warning(
+                    "Disabling stream consumer after %s failure: %s",
+                    method_name,
+                    exc,
+                )
+        self._consumers = active
+
+
+class EulerViewStreamConsumer:
+    """Best-effort HTTP consumer for the model-run streaming API."""
+
+    def __init__(self, config: EulerViewStreamConfig) -> None:
+        self.config = config
+        self._context: StreamContext | None = None
+        self._pending: list[dict[str, Any]] = []
+        self._stream_token: str | None = config.stream_token
+        self._session_expires_at = float("inf") if config.stream_token else 0.0
+        self._ingest_url = f"{self.config.base_url}/api/model-run-stream/ingest"
+        self._last_flush_at = 0.0
+        self._next_retry_at = 0.0
+        self._last_warning: tuple[str, float] | None = None
+        self._closed = False
+
+    def bind(self, context: StreamContext) -> None:
+        self._context = context
+
+    def emit(self, event: dict[str, Any]) -> None:
+        if self._closed:
+            return
+
+        try:
+            payload = _to_jsonable(event)
+        except Exception as exc:
+            self._warn(
+                "Dropping stream event that could not be serialized",
+                exc,
+            )
+            return
+
+        self._pending.append(payload)
+        self._trim_pending()
+
+        force = payload.get("type") in {
+            "init",
+            "checkpoint",
+            "output_snapshot",
+            "finish",
+        }
+        self._maybe_flush(force=force)
+
+    def flush(self) -> None:
+        self._flush_pending(force=True)
+
+    def close(self) -> None:
+        if self._closed:
+            return
+        self._flush_pending(force=True)
+        self._closed = True
+
+    def _maybe_flush(self, *, force: bool = False) -> None:
+        if not self._pending:
+            return
+
+        now = time.time()
+        should_flush = force
+        should_flush = should_flush or len(self._pending) >= self.config.batch_size
+        should_flush = should_flush or (
+            now - self._last_flush_at >= self.config.flush_interval_sec
+        )
+        if should_flush:
+            self._flush_pending(force=force)
+
+    def _flush_pending(self, *, force: bool = False) -> None:
+        if not self._pending:
+            return
+
+        now = time.time()
+        if not force and now < self._next_retry_at:
+            return
+
+        if not self._ensure_session(force=force):
+            return
+
+        while self._pending:
+            batch = self._pending[: self.config.batch_size]
+            try:
+                self._post_json(
+                    self._ingest_url,
+                    {"events": batch},
+                    token=self._stream_token,
+                )
+            except Exception as exc:
+                self._schedule_retry()
+                self._warn("Model-run stream ingest failed", exc)
+                return
+
+            del self._pending[: len(batch)]
+            self._last_flush_at = time.time()
+            self._next_retry_at = 0.0
+
+    def _ensure_session(self, *, force: bool = False) -> bool:
+        if self._stream_token and time.time() + 5 < self._session_expires_at:
+            return True
+
+        if self.config.stream_token:
+            self._stream_token = self.config.stream_token
+            self._session_expires_at = float("inf")
+            return True
+
+        if self._context is None:
+            return False
+
+        now = time.time()
+        if not force and now < self._next_retry_at:
+            return False
+
+        session_url = (
+            f"{self.config.base_url}/api/models/{self.config.model_id}"
+            f"/runs/{quote(self._context.run_id, safe='')}/stream/session"
+        )
+        session_body: dict[str, Any] = {
+            "runDir": self.config.run_dir or str(self._context.run_dir),
+            "eulerTrainDir": self.config.euler_train_dir or str(self._context.runs_dir),
+        }
+        if self.config.datasource_id is not None:
+            session_body["datasourceId"] = self.config.datasource_id
+        if self.config.session_expires_in_sec is not None:
+            session_body["expiresInSec"] = self.config.session_expires_in_sec
+
+        try:
+            payload = self._post_json(
+                session_url,
+                session_body,
+                token=self.config.api_token or self.config.access_token,
+            )
+        except Exception as exc:
+            self._schedule_retry()
+            self._warn("Model-run stream session setup failed", exc)
+            return False
+
+        token = payload.get("token")
+        if not isinstance(token, str) or not token.strip():
+            self._schedule_retry()
+            self._warn("Model-run stream session response did not contain a token")
+            return False
+
+        self._stream_token = token.strip()
+        ingest_url = payload.get("ingestUrl")
+        if isinstance(ingest_url, str) and ingest_url.strip():
+            self._ingest_url = ingest_url.strip()
+        else:
+            self._ingest_url = f"{self.config.base_url}/api/model-run-stream/ingest"
+
+        self._session_expires_at = _parse_timestamp(payload.get("expiresAt"))
+        self._next_retry_at = 0.0
+        return True
+
+    def _post_json(
+        self,
+        url: str,
+        payload: dict[str, Any],
+        *,
+        token: str | None,
+    ) -> dict[str, Any]:
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+
+        request = Request(
+            url,
+            data=json.dumps(payload, default=_json_default).encode("utf-8"),
+            headers=headers,
+            method="POST",
+        )
+
+        try:
+            with urlopen(request, timeout=self.config.timeout_sec) as response:
+                raw = response.read()
+        except HTTPError as exc:
+            message = _extract_http_error_message(exc)
+            raise RuntimeError(message) from exc
+        except URLError as exc:
+            reason = exc.reason if exc.reason else str(exc)
+            raise RuntimeError(str(reason)) from exc
+        except OSError as exc:
+            raise RuntimeError(str(exc)) from exc
+
+        if not raw:
+            return {}
+
+        try:
+            parsed = json.loads(raw.decode("utf-8"))
+        except json.JSONDecodeError as exc:
+            raise RuntimeError("Invalid JSON response from stream API") from exc
+
+        if isinstance(parsed, dict):
+            return parsed
+        raise RuntimeError("Unexpected stream API response payload")
+
+    def _trim_pending(self) -> None:
+        overflow = len(self._pending) - self.config.max_pending_events
+        if overflow <= 0:
+            return
+        del self._pending[:overflow]
+        self._warn(
+            f"Dropped {overflow} pending stream event(s) after queue overflow",
+        )
+
+    def _schedule_retry(self) -> None:
+        backoff = max(self.config.flush_interval_sec, 2.0)
+        self._next_retry_at = time.time() + backoff
+
+    def _warn(self, message: str, exc: Exception | None = None) -> None:
+        detail = f"{message}: {exc}" if exc is not None else message
+        now = time.time()
+        if self._last_warning is not None:
+            previous_detail, previous_at = self._last_warning
+            if previous_detail == detail and now - previous_at < 30:
+                return
+
+        log.warning(detail)
+        self._last_warning = (detail, now)
+
+
+def coerce_output_stream(spec: Any) -> OutputStream | None:
+    if spec is None:
+        return None
+    if isinstance(spec, OutputStream):
+        return spec
+    if isinstance(spec, EulerViewStreamConfig):
+        return OutputStream([EulerViewStreamConsumer(spec)])
+    if isinstance(spec, Mapping):
+        return OutputStream([EulerViewStreamConsumer(_config_from_mapping(spec))])
+    if _looks_like_consumer(spec):
+        return OutputStream([spec])
+    if isinstance(spec, Sequence) and not isinstance(spec, (str, bytes, bytearray)):
+        consumers: list[OutputStreamConsumer] = []
+        for item in spec:
+            stream = coerce_output_stream(item)
+            if stream is None:
+                continue
+            consumers.extend(stream.consumers)
+        return OutputStream(consumers)
+    raise TypeError(
+        "stream must be a mapping, EulerViewStreamConfig, consumer, or sequence of consumers",
+    )
+
+
+def _looks_like_consumer(value: Any) -> bool:
+    return all(
+        callable(getattr(value, name, None))
+        for name in ("bind", "emit", "flush", "close")
+    )
+
+
+def _config_from_mapping(raw: Mapping[str, Any]) -> EulerViewStreamConfig:
+    provider = _mapping_get(raw, "provider", "kind", "type")
+    if provider is not None:
+        normalized = str(provider).strip().lower().replace("-", "_")
+        if normalized not in {"euler_view", "eulerview"}:
+            raise ValueError(
+                f"Unsupported stream provider {provider!r}; expected 'euler_view'",
+            )
+
+    return EulerViewStreamConfig(
+        base_url=_mapping_get(raw, "base_url", "baseUrl"),
+        model_id=_mapping_get(raw, "model_id", "modelId"),
+        access_token=_mapping_get(raw, "access_token", "accessToken"),
+        api_token=_mapping_get(raw, "api_token", "apiToken"),
+        stream_token=_mapping_get(raw, "stream_token", "streamToken"),
+        datasource_id=_mapping_get(raw, "datasource_id", "datasourceId"),
+        euler_train_dir=_mapping_get(raw, "euler_train_dir", "eulerTrainDir"),
+        run_dir=_mapping_get(raw, "run_dir", "runDir"),
+        timeout_sec=_mapping_get(raw, "timeout_sec", "timeoutSec", default=10.0),
+        batch_size=_mapping_get(raw, "batch_size", "batchSize", default=20),
+        flush_interval_sec=_mapping_get(
+            raw,
+            "flush_interval_sec",
+            "flushIntervalSec",
+            default=2.0,
+        ),
+        max_pending_events=_mapping_get(
+            raw,
+            "max_pending_events",
+            "maxPendingEvents",
+            default=1000,
+        ),
+        session_expires_in_sec=_mapping_get(
+            raw,
+            "session_expires_in_sec",
+            "sessionExpiresInSec",
+        ),
+    )
+
+
+def _mapping_get(
+    mapping: Mapping[str, Any],
+    *names: str,
+    default: Any = None,
+) -> Any:
+    for name in names:
+        if name in mapping:
+            return mapping[name]
+    return default
+
+
+def _normalize_optional_text(
+    value: Any,
+    *,
+    field_name: str,
+) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ValueError(f"{field_name} must be a string when provided")
+    normalized = value.strip()
+    return normalized or None
+
+
+def _require_positive_int(
+    value: Any,
+    *,
+    field_name: str,
+    allow_none: bool = False,
+) -> int | None:
+    if value is None:
+        if allow_none:
+            return None
+        raise ValueError(f"{field_name} is required")
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        raise ValueError(f"{field_name} must be a positive integer")
+    return value
+
+
+def _require_positive_number(value: Any, *, field_name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+        raise ValueError(f"{field_name} must be a positive number")
+    return float(value)
+
+
+def _require_non_negative_number(value: Any, *, field_name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
+        raise ValueError(f"{field_name} must be a non-negative number")
+    return float(value)
+
+
+def _parse_timestamp(value: Any) -> float:
+    if not isinstance(value, str) or not value.strip():
+        return time.time() + 300
+
+    try:
+        return datetime.fromisoformat(
+            value.replace("Z", "+00:00"),
+        ).timestamp()
+    except ValueError:
+        return time.time() + 300
+
+
+def _extract_http_error_message(error: HTTPError) -> str:
+    try:
+        raw = error.read()
+    except OSError:
+        raw = b""
+
+    if raw:
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            text = raw.decode("utf-8", errors="replace").strip()
+            if text:
+                return f"HTTP {error.code}: {text}"
+        else:
+            if isinstance(payload, dict):
+                body_error = payload.get("error")
+                if isinstance(body_error, str) and body_error.strip():
+                    return f"HTTP {error.code}: {body_error.strip()}"
+
+    return f"HTTP {error.code}: {error.reason}"
+
+
+def _to_jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value, default=_json_default))
+
+
+__all__ = [
+    "EulerViewStreamConfig",
+    "EulerViewStreamConsumer",
+    "OutputStream",
+    "OutputStreamConsumer",
+    "StreamContext",
+    "coerce_output_stream",
+]

{euler_train-2.2.0 → euler_train-2.4.0}/euler_train.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.2.0
+Version: 2.4.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy

{euler_train-2.2.0 → euler_train-2.4.0}/euler_train.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ euler_train/outputs.py
 euler_train/run.py
 euler_train/serialization.py
 euler_train/slurm.py
+euler_train/stream.py
 euler_train.egg-info/PKG-INFO
 euler_train.egg-info/SOURCES.txt
 euler_train.egg-info/dependency_links.txt

{euler_train-2.2.0 → euler_train-2.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "euler-train"
-version = "2.2.0"
+version = "2.4.0"
 description = "Lightweight file-based experiment logger for PyTorch"
 requires-python = ">=3.9"
 dependencies = ["numpy", "euler-dataset-contract"]

{euler_train-2.2.0 → euler_train-2.4.0}/tests/test_runlog.py

@@ -7,6 +7,7 @@ import signal
 import sys
 from argparse import Namespace
 from pathlib import Path
+from urllib.error import URLError
 
 import numpy as np
 import pytest
@@ -14,6 +15,7 @@ import torch
 from PIL import Image
 
 import euler_train
+import euler_train.stream as stream_mod
 
 # ═══════════════════════════════════════════════════════════════════════════
 #  Helpers
@@ -72,6 +74,40 @@ class _DatasetWithRunlogDescription:
         return self._description
 
 
+class _RecordingStreamConsumer:
+    def __init__(self) -> None:
+        self.context = None
+        self.events: list[dict] = []
+        self.flush_count = 0
+        self.close_count = 0
+
+    def bind(self, context) -> None:
+        self.context = context
+
+    def emit(self, event: dict) -> None:
+        self.events.append(json.loads(json.dumps(event)))
+
+    def flush(self) -> None:
+        self.flush_count += 1
+
+    def close(self) -> None:
+        self.close_count += 1
+
+
+class _FakeHttpResponse:
+    def __init__(self, payload: dict) -> None:
+        self._payload = payload
+
+    def read(self) -> bytes:
+        return json.dumps(self._payload).encode("utf-8")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        return False
+
+
 # ═══════════════════════════════════════════════════════════════════════════
 #  init  /  config normalisation
 # ═══════════════════════════════════════════════════════════════════════════
@@ -1605,6 +1641,228 @@ class TestLogSavedCheckpoint:
         run.finish()
 
 
+# ═══════════════════════════════════════════════════════════════════════════
+#  streaming
+# ═══════════════════════════════════════════════════════════════════════════
+
+class TestStreaming:
+    def test_run_emits_expected_stream_events(self, tmp_path):
+        consumer = _RecordingStreamConsumer()
+
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={"lr": 1e-3},
+            meta={"tags": ["baseline", "fast"]},
+            stream=consumer,
+        )
+        run.add_evaluation(
+            "eval_rgb",
+            name="RGB Eval",
+            status="running",
+            metadata={"seed": 7},
+        )
+        run.finish_evaluation("eval_rgb")
+        run.log({"loss": 0.5}, step=0, epoch=0, mode="train")
+        run.log({"psnr": 28.1}, step=1, epoch=0, mode="val")
+        arr = np.zeros((4, 4, 3), dtype=np.uint8)
+        aux = np.ones((4, 4), dtype=np.float32)
+        run.save_outputs(
+            epoch=0,
+            step=1,
+            rgb=dict(pred=arr, gt=arr),
+            depth=dict(aux=dict(transmission=aux)),
+        )
+        run.log_saved_checkpoint("/tmp/model.pt", epoch=0, step=1, is_best=True)
+        run.finish()
+
+        assert consumer.context.run_id == run.run_id
+        assert consumer.context.run_dir == run.dir
+        assert consumer.close_count == 1
+        assert consumer.flush_count >= 2
+
+        event_types = [event["type"] for event in consumer.events]
+        assert event_types == [
+            "init",
+            "meta",
+            "meta",
+            "metric",
+            "metric",
+            "output_snapshot",
+            "checkpoint",
+            "finish",
+        ]
+
+        init_event = consumer.events[0]
+        assert init_event["meta"]["run_id"] == run.run_id
+        assert init_event["config"] == {"lr": 1e-3}
+        assert init_event["tags"] == ["baseline", "fast"]
+        assert isinstance(init_event["codeRef"], dict)
+        assert isinstance(init_event["runEnvironment"], dict)
+
+        add_eval_event = consumer.events[1]
+        assert add_eval_event["patch"] == {
+            "evaluations": {
+                "eval_rgb": {
+                    "name": "RGB Eval",
+                    "status": "running",
+                    "metadata": {"seed": 7},
+                },
+            },
+        }
+
+        finish_eval_event = consumer.events[2]
+        assert finish_eval_event["patch"] == {
+            "evaluations": {
+                "eval_rgb": {
+                    "status": "completed",
+                },
+            },
+        }
+
+        train_event = consumer.events[3]
+        assert train_event["split"] == "train"
+        assert train_event["records"][0]["loss"] == 0.5
+        assert "elapsed_sec" in train_event["records"][0]
+
+        val_event = consumer.events[4]
+        assert val_event["split"] == "val"
+        assert val_event["records"][0]["psnr"] == 28.1
+        assert "elapsed_sec" not in val_event["records"][0]
+
+        snapshot_event = consumer.events[5]
+        assert snapshot_event["snapshot"] == {
+            "epoch": 0,
+            "step": 1,
+            "types": {
+                "depth": ["aux/transmission"],
+                "rgb": ["gt", "pred"],
+            },
+        }
+
+        checkpoint_event = consumer.events[6]
+        assert checkpoint_event["checkpoint"] == {
+            "path": "/tmp/model.pt",
+            "epoch": 0,
+            "step": 1,
+            "is_best": True,
+        }
+
+        finish_event = consumer.events[7]
+        assert finish_event["status"] == "completed"
+        assert finish_event["patch"]["end_time"] is not None
+        assert finish_event["patch"]["duration_sec"] >= 0
+
+    def test_euler_view_http_consumer_uses_session_and_ingest_routes(
+        self,
+        tmp_path,
+        monkeypatch,
+    ):
+        calls: list[dict] = []
+
+        def fake_urlopen(request, timeout=0):
+            headers = {k.lower(): v for k, v in request.header_items()}
+            body = json.loads(request.data.decode("utf-8"))
+            calls.append(
+                {
+                    "url": request.full_url,
+                    "method": request.get_method(),
+                    "headers": headers,
+                    "body": body,
+                    "timeout": timeout,
+                },
+            )
+            if request.full_url.endswith("/stream/session"):
+                return _FakeHttpResponse(
+                    {
+                        "token": "stream-token",
+                        "expiresAt": "2099-01-01T00:00:00+00:00",
+                        "ingestUrl": "https://sync.example/api/model-run-stream/ingest",
+                    },
+                )
+            if request.full_url.endswith("/api/model-run-stream/ingest"):
+                return _FakeHttpResponse(
+                    {
+                        "success": True,
+                        "events": [],
+                        "latestCursor": len(calls),
+                    },
+                )
+            raise AssertionError(f"Unexpected URL {request.full_url}")
+
+        monkeypatch.setattr(stream_mod, "urlopen", fake_urlopen)
+
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={"lr": 1e-3},
+            stream={
+                "base_url": "https://sync.example",
+                "model_id": 42,
+                "api_token": "user-token",
+                "datasource_id": 7,
+                "batch_size": 1,
+            },
+        )
+        run.log({"loss": 0.25}, step=3, epoch=1)
+        run.finish()
+
+        assert calls[0]["url"] == (
+            f"https://sync.example/api/models/42/runs/{run.run_id}/stream/session"
+        )
+        assert calls[0]["method"] == "POST"
+        assert calls[0]["headers"]["authorization"] == "Bearer user-token"
+        assert calls[0]["body"] == {
+            "runDir": str(run.dir),
+            "eulerTrainDir": str(run.dir.parent),
+            "datasourceId": 7,
+        }
+
+        ingest_calls = calls[1:]
+        assert [call["body"]["events"][0]["type"] for call in ingest_calls] == [
+            "init",
+            "metric",
+            "finish",
+        ]
+        for ingest_call in ingest_calls:
+            assert ingest_call["url"] == "https://sync.example/api/model-run-stream/ingest"
+            assert ingest_call["headers"]["authorization"] == "Bearer stream-token"
+
+        metric_call = ingest_calls[1]
+        metric_event = metric_call["body"]["events"][0]
+        assert metric_event["split"] == "train"
+        assert metric_event["records"][0]["step"] == 3
+        assert metric_event["records"][0]["epoch"] == 1
+        assert metric_event["records"][0]["loss"] == 0.25
+
+        finish_event = ingest_calls[2]["body"]["events"][0]
+        assert finish_event["status"] == "completed"
+
+    def test_stream_network_failures_do_not_break_training(
+        self,
+        tmp_path,
+        monkeypatch,
+    ):
+        def fake_urlopen(request, timeout=0):
+            raise URLError("offline")
+
+        monkeypatch.setattr(stream_mod, "urlopen", fake_urlopen)
+
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+            stream={
+                "base_url": "https://sync.example",
+                "model_id": 42,
+                "access_token": "user-token",
+                "batch_size": 1,
+            },
+        )
+        run.log({"loss": 1.0}, step=0, epoch=0)
+        run.finish()
+
+        meta = _read_json(run.dir / "meta.json")
+        assert meta["status"] == "completed"
+
+
 # ═══════════════════════════════════════════════════════════════════════════
 #  lifecycle  (finish, context manager, crash)
 # ═══════════════════════════════════════════════════════════════════════════