euler-train 2.1.0__tar.gz → 2.3.0__tar.gz

{euler_train-2.1.0 → euler_train-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.1.0
+Version: 2.3.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy

{euler_train-2.1.0 → euler_train-2.3.0}/README.md

@@ -82,7 +82,7 @@ The run ID and directory are available as `run.run_id` and `run.dir`.
 
 ### `euler_train.init(dir, config=None, meta=None, output_formats=None, output_visualization=None, run_id=None, datasets=None, run_name=None, evaluations=None, mode=None) → Run`
 
-Creates the run directory and writes `meta.json`, `config.json`, `code_ref.json`, and `run_environment.json`. On resume (`run_id` provided), only `meta.json` and `config.json` are updated.
+Creates the run directory and writes `meta.json`, `config.json`, `code_ref.json`, and `run_environment.json`. On resume (`run_id` provided), only `meta.json` and `config.json` are updated. `meta.json` also maintains an `updated_at` map with last-write timestamps for tracked artifacts.
 
 | Parameter | Type | Description |
 |---|---|---|
@@ -365,6 +365,24 @@ Auto-managed, not written to directly.
       "traceback": "Traceback (most recent call last):\n  ..."
     }
   },
+  "updated_at": {
+    "meta.json": {
+      "time": 1706403300.0,
+      "iso": "2025-01-28T16:25:00"
+    },
+    "config.json": {
+      "time": 1706400000.0,
+      "iso": "2025-01-28T15:30:42"
+    },
+    "train.jsonl": {
+      "time": 1706402999.0,
+      "iso": "2025-01-28T16:19:59"
+    },
+    "outputs/epoch_12_step_4800": {
+      "time": 1706403010.0,
+      "iso": "2025-01-28T16:20:10"
+    }
+  },
   "error": "RuntimeError: CUDA OOM",
   "traceback": "Traceback (most recent call last):\n  ..."
 }
@@ -375,6 +393,7 @@ Auto-managed, not written to directly.
 - `datasets` is only present when `datasets=...` is passed to `euler_train.init`.
 - `evaluations` is only present when evaluations are provided via `evaluations=...` on `init()` or added via `run.add_evaluation()`.
 - `modes` is only present when `mode=...` is passed to `euler_train.init`; each key stores the latest lifecycle snapshot for that mode.
+- `updated_at` stores last-write timestamps for tracked artifacts. Keys are run-relative paths when the artifact lives inside the run directory (for example `train.jsonl` or `outputs/epoch_2_step_500`) and absolute paths for external artifacts such as custom checkpoint locations.
 - `error` is only present when `status` is `"crashed"` (context manager / excepthook) or `"interrupted"` (SIGTERM/SIGINT). `traceback` is only present when `status` is `"crashed"`. When `mode=...` is set, the same fields are mirrored under `modes[mode]`.
 
 A formal JSON Schema for `meta.json` is available at [`meta-schema.json`](meta-schema.json).

{euler_train-2.1.0 → euler_train-2.3.0}/euler_train/__init__.py

@@ -6,8 +6,24 @@ from typing import Any
 from . import _dataset_contract  # noqa: F401
 from .run import Run
 from .architecture import export_architecture
+from .stream import (
+    EulerViewStreamConfig,
+    EulerViewStreamConsumer,
+    OutputStream,
+    OutputStreamConsumer,
+    StreamContext,
+)
 
-__all__ = ["init", "Run", "export_architecture"]
+__all__ = [
+    "EulerViewStreamConfig",
+    "EulerViewStreamConsumer",
+    "OutputStream",
+    "OutputStreamConsumer",
+    "StreamContext",
+    "init",
+    "Run",
+    "export_architecture",
+]
 __version__ = "0.1.0"
 
 
@@ -22,6 +38,7 @@ def init(
     run_name: str | None = None,
     evaluations: dict[str, dict] | None = None,
     mode: str | None = None,
+    stream: Any = None,
 ) -> Run:
     """Create a new run — or resume an existing one — and return the handle.
 
@@ -89,6 +106,11 @@ def init(
         ``"train"``, ``"val"``, or ``"eval"``).  When provided,
         lifecycle fields and crash details are mirrored into
         ``meta.json`` under ``modes[mode]``.
+    stream:
+        Optional output-stream consumer or config. Passing a mapping
+        with ``base_url`` plus either ``stream_token`` or the pair
+        ``model_id`` / ``access_token`` enables best-effort dual-write
+        streaming to the Euler View model-run ingest API.
     """
     return Run(
         dir=dir, config=config, meta=meta,
@@ -97,4 +119,5 @@ def init(
         run_id=run_id,
         datasets=datasets, run_name=run_name,
         evaluations=evaluations, mode=mode,
+        stream=stream,
     )

{euler_train-2.1.0 → euler_train-2.3.0}/euler_train/run.py

@@ -2,15 +2,16 @@
 from __future__ import annotations
 
 import atexit
+import copy
 import logging
 import os
 import re
 import secrets
-import warnings
 import signal
 import sys
 import time
 import traceback
+import warnings
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
@@ -20,6 +21,7 @@ from .outputs import save_output_tree
 from .slurm import get_slurm_info
 from .git_info import get_code_ref
 from .environment import get_run_environment
+from .stream import OutputStream, StreamContext, coerce_output_stream
 
 log = logging.getLogger("euler_train")
 
@@ -45,6 +47,7 @@ class Run:
         run_name: str | None = None,
         evaluations: dict[str, dict[str, Any]] | None = None,
         mode: str | None = None,
+        stream: Any = None,
     ) -> None:
         # ── detect run-directory shorthand ────────────────────────
         # When `dir` points to an existing run directory (contains
@@ -82,6 +85,9 @@ class Run:
         self._gpu_stats_every: int = gpu_stats_every
         self.run_name: str | None = run_name
         self.mode: str | None = _normalize_mode(mode)
+        self._pending_updated_at: dict[str, dict[str, Any]] = {}
+        self._stream: OutputStream | None = coerce_output_stream(stream)
+        self._stream_event_counters: dict[str, int] = {}
 
         # ── config ────────────────────────────────────────────────
         if resuming:
@@ -92,12 +98,30 @@ class Run:
         else:
             self.config: dict = normalize_config(config)
 
-        write_json(self.dir / "config.json", self.config)
+        config_path = self.dir / "config.json"
+        write_json(config_path, self.config)
+        self._record_artifact_update(config_path)
+
+        # ── code ref & run environment ───────────────────────────
+        code_ref_path = self.dir / "code_ref.json"
+        run_environment_path = self.dir / "run_environment.json"
+        if resuming:
+            self._code_ref_payload = (
+                read_json(code_ref_path) if code_ref_path.exists() else None
+            )
+            self._run_environment_payload = (
+                read_json(run_environment_path)
+                if run_environment_path.exists()
+                else None
+            )
+        else:
+            self._code_ref_payload = get_code_ref()
+            write_json(code_ref_path, self._code_ref_payload)
+            self._record_artifact_update(code_ref_path)
 
-        # ── code ref & run environment (fresh runs only) ─────────
-        if not resuming:
-            write_json(self.dir / "code_ref.json", get_code_ref())
-            write_json(self.dir / "run_environment.json", get_run_environment())
+            self._run_environment_payload = get_run_environment()
+            write_json(run_environment_path, self._run_environment_payload)
+            self._record_artifact_update(run_environment_path)
 
         # ── meta ──────────────────────────────────────────────────
         if resuming:
@@ -153,6 +177,8 @@ class Run:
             )
         self._mark_mode_running()
         self._flush_meta()
+        self._bind_stream()
+        self._emit_stream_init()
         self._setup_hooks()
 
         verb = "Resumed" if resuming else "Started"
@@ -194,6 +220,19 @@ class Run:
             existing=self._meta.get("evaluations"),
         )
         self._flush_meta()
+        evaluations = self._meta.get("evaluations", {})
+        if isinstance(evaluations, dict) and key in evaluations:
+            self._emit_stream_event(
+                {
+                    "type": "meta",
+                    "eventId": self._make_stream_event_id("evaluation", key),
+                    "patch": {
+                        "evaluations": {
+                            key: copy.deepcopy(evaluations[key]),
+                        },
+                    },
+                },
+            )
 
     def finish_evaluation(self, key: str, status: str = "completed") -> None:
         """Mark an existing evaluation as finished.
@@ -208,6 +247,23 @@ class Run:
             )
         evals[key]["status"] = status
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "meta",
+                "eventId": self._make_stream_event_id(
+                    "evaluation_status",
+                    key,
+                    status,
+                ),
+                "patch": {
+                    "evaluations": {
+                        key: {
+                            "status": status,
+                        },
+                    },
+                },
+            },
+        )
 
     # ── logging ───────────────────────────────────────────────────
 
@@ -238,7 +294,19 @@ class Run:
             record.update(self._get_gpu_stats())
 
         filename = "train.jsonl" if mode == "train" else "val.jsonl"
-        append_jsonl(self.dir / filename, record)
+        log_path = self.dir / filename
+        append_jsonl(log_path, record)
+        self._record_artifact_update(log_path)
+        self._flush_meta()
+        split = "train" if mode == "train" else "val"
+        self._emit_stream_event(
+            {
+                "type": "metric",
+                "eventId": self._make_stream_event_id(split, epoch, step),
+                "split": split,
+                "records": [copy.deepcopy(record)],
+            },
+        )
 
     # ── GPU stats ─────────────────────────────────────────────────
 
@@ -313,6 +381,22 @@ class Run:
                 self._output_visualization,
                 output_type,
             )
+        if base.exists():
+            self._record_artifact_update(base)
+            self._flush_meta()
+            snapshot = self._build_output_snapshot(epoch, step, output_types)
+            if snapshot is not None:
+                self._emit_stream_event(
+                    {
+                        "type": "output_snapshot",
+                        "eventId": self._make_stream_event_id(
+                            "outputs",
+                            epoch,
+                            step,
+                        ),
+                        "snapshot": snapshot,
+                    },
+                )
         return base
 
     # ── checkpoints ───────────────────────────────────────────────
@@ -339,6 +423,7 @@ class Run:
         if self.checkpoint_dir is not None:
             self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
             self._meta["checkpoint_dir"] = str(self.checkpoint_dir)
+            self._record_artifact_update(self.checkpoint_dir)
             self._flush_meta()
             log.info("Checkpoint dir: %s", self.checkpoint_dir)
             return self.checkpoint_dir
@@ -357,6 +442,7 @@ class Run:
             )
         self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
         self._meta["checkpoint_dir"] = str(self.checkpoint_dir)
+        self._record_artifact_update(self.checkpoint_dir)
         self._flush_meta()
         log.info("Checkpoint dir: %s", self.checkpoint_dir)
         return self.checkpoint_dir
@@ -415,6 +501,7 @@ class Run:
         :meth:`save_checkpoint` call it automatically.  When *is_best*
         is ``True``, any previous checkpoint marked as best is cleared.
         """
+        checkpoint_path = Path(path)
         checkpoints: list[dict[str, Any]] = self._meta.get("checkpoints", [])
 
         if is_best:
@@ -422,7 +509,7 @@ class Run:
                 entry.pop("is_best", None)
 
         record: dict[str, Any] = {
-            "path": str(path),
+            "path": str(checkpoint_path),
             "epoch": epoch,
             "step": step,
         }
@@ -431,7 +518,19 @@ class Run:
 
         checkpoints.append(record)
         self._meta["checkpoints"] = checkpoints
+        self._record_artifact_update(checkpoint_path)
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "checkpoint",
+                "eventId": self._make_stream_event_id(
+                    "checkpoint",
+                    epoch,
+                    step,
+                ),
+                "checkpoint": copy.deepcopy(record),
+            },
+        )
 
     # ── architecture export ─────────────────────────────────────────
 
@@ -467,6 +566,7 @@ class Run:
             model, dummy_input, self.dir / "architecture.onnx"
         )
         self._meta["architecture"] = "architecture.onnx"
+        self._record_artifact_update(output_path)
         self._flush_meta()
         return output_path
 
@@ -489,6 +589,16 @@ class Run:
         self._clear_terminal_fields(self._meta, status=status)
         self._finish_mode(status=status, end=end, duration=duration)
         self._flush_meta()
+        self._emit_stream_event(
+            {
+                "type": "finish",
+                "eventId": self._make_stream_event_id("finish", status),
+                "status": status,
+                "patch": self._build_finish_patch(),
+            },
+            flush=True,
+        )
+        self._close_stream()
 
     def detach(self) -> None:
         """Disconnect from the run without changing its status.
@@ -508,6 +618,7 @@ class Run:
         self._finished = True
         self._teardown_hooks()
         self._flush_meta()
+        self._close_stream()
 
     # ── context manager ───────────────────────────────────────────
 
@@ -652,9 +763,177 @@ class Run:
         self._meta["modes"] = modes
         return modes
 
+    def _updated_at_store(self) -> dict[str, Any]:
+        if not hasattr(self, "_meta"):
+            return self._pending_updated_at
+
+        updated_at = self._meta.get("updated_at")
+        if not isinstance(updated_at, dict):
+            updated_at = {}
+            self._meta["updated_at"] = updated_at
+
+        if self._pending_updated_at:
+            updated_at.update(self._pending_updated_at)
+            self._pending_updated_at.clear()
+
+        return updated_at
+
+    def _record_artifact_update(
+        self,
+        path: str | Path,
+        *,
+        when: float | None = None,
+    ) -> None:
+        artifact_path = Path(path)
+        ts = when if when is not None else _artifact_mtime(artifact_path)
+        if ts is None:
+            ts = time.time()
+        self._updated_at_store()[self._artifact_key(artifact_path)] = (
+            _updated_at_entry(ts)
+        )
+
+    def _artifact_key(self, path: Path) -> str:
+        if path.is_absolute():
+            try:
+                return path.relative_to(self.dir).as_posix()
+            except ValueError:
+                return str(path)
+        return path.as_posix()
+
     def _flush_meta(self) -> None:
+        self._record_artifact_update(self.dir / "meta.json", when=time.time())
         write_json(self.dir / "meta.json", self._meta)
 
+    def _bind_stream(self) -> None:
+        if self._stream is None:
+            return
+        self._stream.bind(
+            StreamContext(
+                run_id=self.run_id,
+                project_dir=self.project_dir,
+                runs_dir=self.dir.parent,
+                run_dir=self.dir,
+                meta=copy.deepcopy(self._meta),
+                config=copy.deepcopy(self.config),
+                code_ref=copy.deepcopy(self._code_ref_payload),
+                run_environment=copy.deepcopy(self._run_environment_payload),
+            ),
+        )
+
+    def _emit_stream_init(self) -> None:
+        tags = self._extract_stream_tags()
+        self._emit_stream_event(
+            {
+                "type": "init",
+                "eventId": self._make_stream_event_id("init"),
+                "meta": copy.deepcopy(self._meta),
+                "config": copy.deepcopy(self.config),
+                "codeRef": copy.deepcopy(self._code_ref_payload),
+                "runEnvironment": copy.deepcopy(self._run_environment_payload),
+                "tags": tags,
+            },
+            flush=True,
+        )
+
+    def _emit_stream_event(
+        self,
+        event: dict[str, Any],
+        *,
+        flush: bool = False,
+    ) -> None:
+        if self._stream is None:
+            return
+        self._stream.emit(event)
+        if flush:
+            self._stream.flush()
+
+    def _close_stream(self) -> None:
+        if self._stream is None:
+            return
+        self._stream.close()
+
+    def _extract_stream_tags(self) -> list[str] | None:
+        raw_tags = self._meta.get("tags")
+        if not isinstance(raw_tags, list):
+            return None
+        tags = [
+            str(tag).strip()
+            for tag in raw_tags
+            if isinstance(tag, str) and str(tag).strip()
+        ]
+        return tags or None
+
+    def _build_output_snapshot(
+        self,
+        epoch: int | None,
+        step: int | None,
+        output_types: dict[str, Any],
+    ) -> dict[str, Any] | None:
+        if epoch is None or step is None:
+            return None
+
+        snapshot_types: dict[str, list[str]] = {}
+        for output_type, slots in output_types.items():
+            if slots is None or not isinstance(slots, Mapping):
+                continue
+
+            categories: list[str] = []
+            for slot_name, data in slots.items():
+                if data is None:
+                    continue
+                if slot_name == "aux" and isinstance(data, Mapping):
+                    for aux_name, aux_value in data.items():
+                        if aux_value is None:
+                            continue
+                        categories.append(f"aux/{aux_name}")
+                else:
+                    categories.append(str(slot_name))
+
+            if categories:
+                snapshot_types[str(output_type)] = sorted(set(categories))
+
+        if not snapshot_types:
+            return None
+
+        return {
+            "epoch": epoch,
+            "step": step,
+            "types": snapshot_types,
+        }
+
+    def _build_finish_patch(self) -> dict[str, Any]:
+        patch = {
+            "end_time": self._meta.get("end_time"),
+            "end_iso": self._meta.get("end_iso"),
+            "duration_sec": self._meta.get("duration_sec"),
+        }
+
+        if "error" in self._meta:
+            patch["error"] = self._meta["error"]
+        if "traceback" in self._meta:
+            patch["traceback"] = self._meta["traceback"]
+
+        if self.mode is not None:
+            modes = self._meta.get("modes")
+            if isinstance(modes, dict) and self.mode in modes:
+                patch["modes"] = {
+                    self.mode: copy.deepcopy(modes[self.mode]),
+                }
+
+        return patch
+
+    def _make_stream_event_id(self, prefix: str, *parts: Any) -> str:
+        counter = self._stream_event_counters.get(prefix, 0)
+        self._stream_event_counters[prefix] = counter + 1
+        tokens = [prefix]
+        tokens.extend(
+            str(part)
+            for part in parts
+            if part is not None and str(part).strip()
+        )
+        tokens.append(str(counter))
+        return ":".join(tokens)
+
     def __repr__(self) -> str:
         return f"Run(id={self.run_id!r}, dir={str(self.dir)!r}, status={self._meta['status']!r})"
 
@@ -732,6 +1011,20 @@ def _isotime(ts: float) -> str:
     return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(ts))
 
 
+def _updated_at_entry(ts: float) -> dict[str, Any]:
+    return {
+        "time": ts,
+        "iso": _isotime(ts),
+    }
+
+
+def _artifact_mtime(path: Path) -> float | None:
+    try:
+        return path.stat().st_mtime
+    except OSError:
+        return None
+
+
 def _normalize_mode(mode: str | None) -> str | None:
     if mode is None:
         return None
@@ -1050,6 +1343,9 @@ def _read_ds_crawler_descriptor(path: str) -> dict[str, Any]:
     try:
         cfg = load_dataset_config({"path": path})
     except Exception:
+        descriptor = _read_output_json_descriptor(path)
+        if descriptor:
+            return descriptor
         try:
             contract = get_dataset_contract(path)
         except Exception:
@@ -1082,6 +1378,69 @@ def _read_ds_crawler_descriptor(path: str) -> dict[str, Any]:
     return descriptor
 
 
+def _read_output_json_descriptor(path: str) -> dict[str, Any]:
+    try:
+        from ds_crawler import zip_utils
+    except Exception:
+        return {}
+
+    try:
+        raw = zip_utils.read_metadata_json(Path(path), "output.json")
+    except Exception:
+        return {}
+
+    return _descriptor_from_output_json(raw, path)
+
+
+def _descriptor_from_output_json(raw: Any, path: str) -> dict[str, Any]:
+    if isinstance(raw, Mapping):
+        candidates: list[Any]
+        modalities = raw.get("modalities")
+        if isinstance(modalities, list):
+            candidates = modalities
+        else:
+            candidates = [raw]
+    elif isinstance(raw, list):
+        candidates = raw
+    else:
+        return {}
+
+    selected: Mapping[str, Any] | None = None
+    path_name = Path(path).name
+    for candidate in candidates:
+        if not isinstance(candidate, Mapping):
+            continue
+        if _as_non_empty_str(candidate.get("name")) == path_name:
+            selected = candidate
+            break
+        if selected is None:
+            selected = candidate
+
+    if selected is None:
+        return {}
+
+    descriptor: dict[str, Any] = {}
+    properties = selected.get("properties")
+    merged_properties = dict(properties) if isinstance(properties, Mapping) else {}
+
+    euler_train_props = selected.get("euler_train")
+    if isinstance(euler_train_props, Mapping):
+        merged_properties["euler_train"] = dict(euler_train_props)
+
+    if merged_properties:
+        descriptor["properties"] = merged_properties
+
+    modality_key = _as_non_empty_str(selected.get("type"))
+    if modality_key is not None:
+        descriptor["modality_key"] = modality_key
+
+    hierarchy_regex = _as_non_empty_str(selected.get("hierarchy_regex"))
+    if hierarchy_regex is not None:
+        descriptor["hierarchy_regex"] = hierarchy_regex
+
+    return descriptor
+
+
 def _build_modality_entry(
     name: str,
     path: str,