euler-train 2.4.0__tar.gz → 2.5.0__tar.gz

{euler_train-2.4.0 → euler_train-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.4.0
+Version: 2.5.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy
@@ -13,6 +13,8 @@ Provides-Extra: architecture
 Requires-Dist: onnx; extra == "architecture"
 Requires-Dist: onnxruntime; extra == "architecture"
 Requires-Dist: onnxsim; extra == "architecture"
+Provides-Extra: naming
+Requires-Dist: euler-metric-naming; extra == "naming"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: Pillow; extra == "dev"

{euler_train-2.4.0 → euler_train-2.5.0}/euler_train/__init__.py

@@ -39,6 +39,7 @@ def init(
     evaluations: dict[str, dict] | None = None,
     mode: str | None = None,
     stream: Any = None,
+    metric_naming: dict[str, Any] | None = None,
 ) -> Run:
     """Create a new run — or resume an existing one — and return the handle.
 
@@ -112,6 +113,12 @@ def init(
         ``model_id`` / ``api_token`` (or legacy ``access_token``)
         enables best-effort dual-write streaming to the Euler View
         model-run ingest API.
+    metric_naming:
+        Optional structured metric naming declaration.  When provided,
+        stored in ``meta.json`` under ``metric_naming`` and included
+        in the stream init event so euler-view can resolve per-metric
+        namespaces.  Typically generated by
+        ``euler_metric_naming.MetricNamespace.training_naming_config()``.
     """
     return Run(
         dir=dir, config=config, meta=meta,
@@ -121,4 +128,5 @@ def init(
         datasets=datasets, run_name=run_name,
         evaluations=evaluations, mode=mode,
         stream=stream,
+        metric_naming=metric_naming,
     )

{euler_train-2.4.0 → euler_train-2.5.0}/euler_train/outputs.py

@@ -68,10 +68,19 @@ def _save_slot(
     visualization_overrides: dict[str, Any],
 ) -> None:
     slot_dir.mkdir(parents=True, exist_ok=True)
-    items = _unpack(data)
     visualization = _resolve_visualization(
         output_type, leaf_name, visualization_overrides,
     )
+
+    # Dict with string keys → named outputs (e.g. {"scene_042": img}).
+    if isinstance(data, dict):
+        for name, raw in data.items():
+            item = _prepare(raw)
+            fmt = _resolve_format(item, output_type, leaf_name, format_overrides)
+            _save_item(slot_dir / f"{name}.{fmt}", item, fmt, visualization)
+        return
+
+    items = _unpack(data)
     for idx, item in enumerate(items):
         fmt = _resolve_format(item, output_type, leaf_name, format_overrides)
         _save_item(slot_dir / f"{idx:04d}.{fmt}", item, fmt, visualization)

{euler_train-2.4.0 → euler_train-2.5.0}/euler_train/run.py

@@ -48,6 +48,7 @@ class Run:
         evaluations: dict[str, dict[str, Any]] | None = None,
         mode: str | None = None,
         stream: Any = None,
+        metric_naming: dict[str, Any] | None = None,
     ) -> None:
         # ── detect run-directory shorthand ────────────────────────
         # When `dir` points to an existing run directory (contains
@@ -175,6 +176,12 @@ class Run:
                 evaluations=evaluations,
                 existing=self._meta.get("evaluations"),
             )
+
+        # ── metric naming (optional structured naming declaration) ──
+        if metric_naming is not None:
+            _validate_metric_naming(metric_naming)
+            self._meta["metric_naming"] = metric_naming
+
         self._mark_mode_running()
         self._flush_meta()
         self._bind_stream()
@@ -311,7 +318,14 @@ class Run:
     # ── GPU stats ─────────────────────────────────────────────────
 
     def _get_gpu_stats(self) -> dict[str, Any]:
-        """Return GPU utilization & memory stats, or {} if unavailable."""
+        """Return GPU utilization & memory stats, or {} if unavailable.
+
+        When the run has a ``metric_naming`` declaration, keys are
+        namespaced under ``sys.train.*`` so euler-view can resolve them
+        against the declared ``sys.train`` namespace.  Legacy runs
+        (without ``metric_naming``) retain the flat key names for
+        backward compatibility.
+        """
         if self._gpu_available is None:
             try:
                 import pynvml
@@ -328,7 +342,7 @@ class Run:
         try:
             util = pynvml.nvmlDeviceGetUtilizationRates(self._gpu_handle)
             mem = pynvml.nvmlDeviceGetMemoryInfo(self._gpu_handle)
-            return {
+            raw = {
                 "gpu_util_pct": util.gpu,
                 "gpu_mem_util_pct": util.memory,
                 "gpu_mem_used_gb": round(mem.used / 1e9, 3),
@@ -336,6 +350,9 @@ class Run:
             }
         except Exception:
             return {}
+        if self._meta.get("metric_naming") is not None:
+            return {f"sys.train.{k}": v for k, v in raw.items()}
+        return raw
 
     # ── visual / heavy outputs ────────────────────────────────────
 
@@ -358,6 +375,14 @@ class Run:
                             aux=dict(transmission=t_map)),
             )
 
+        Slot values can also be dicts mapping custom string IDs to
+        items, producing named files instead of sequential indices::
+
+            run.save_outputs(
+                epoch=1, step=500,
+                rgb=dict(pred={"scene_042": img_a, "scene_117": img_b}),
+            )
+
         Accepted slot keys: ``pred``, ``gt``, ``input``, ``aux``.
         ``aux`` expects a sub-dict of named arrays.
 
@@ -1035,6 +1060,26 @@ def _normalize_mode(mode: str | None) -> str | None:
     return normalized
 
 
+def _validate_metric_naming(value: Any) -> None:
+    """Lightweight structural validation of a metric_naming payload."""
+    if not isinstance(value, dict):
+        raise TypeError("metric_naming must be a dict")
+    namespaces = value.get("namespaces")
+    if not isinstance(namespaces, dict):
+        raise ValueError(
+            "metric_naming must contain a 'namespaces' dict"
+        )
+    for ns_key, ns_decl in namespaces.items():
+        if not isinstance(ns_key, str):
+            raise ValueError(
+                f"metric_naming namespace key must be a string, got {type(ns_key).__name__}"
+            )
+        if not isinstance(ns_decl, dict):
+            raise ValueError(
+                f"metric_naming namespace {ns_key!r} must be a dict"
+            )
+
+
 def _build_datasets_meta(
     datasets: Mapping[str, Any],
     existing: Any,

{euler_train-2.4.0 → euler_train-2.5.0}/euler_train.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.4.0
+Version: 2.5.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy
@@ -13,6 +13,8 @@ Provides-Extra: architecture
 Requires-Dist: onnx; extra == "architecture"
 Requires-Dist: onnxruntime; extra == "architecture"
 Requires-Dist: onnxsim; extra == "architecture"
+Provides-Extra: naming
+Requires-Dist: euler-metric-naming; extra == "naming"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: Pillow; extra == "dev"

{euler_train-2.4.0 → euler_train-2.5.0}/euler_train.egg-info/requires.txt

@@ -20,3 +20,6 @@ nvidia-ml-py
 
 [images]
 Pillow
+
+[naming]
+euler-metric-naming

{euler_train-2.4.0 → euler_train-2.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "euler-train"
-version = "2.4.0"
+version = "2.5.0"
 description = "Lightweight file-based experiment logger for PyTorch"
 requires-python = ">=3.9"
 dependencies = ["numpy", "euler-dataset-contract"]
@@ -13,6 +13,7 @@ dependencies = ["numpy", "euler-dataset-contract"]
 images = ["Pillow"]
 gpu = ["nvidia-ml-py"]
 architecture = ["onnx", "onnxruntime", "onnxsim"]
+naming = ["euler-metric-naming"]
 dev = ["pytest", "Pillow", "torch", "nvidia-ml-py", "onnx", "onnxruntime", "onnxsim"]
 
 [tool.pytest.ini_options]

{euler_train-2.4.0 → euler_train-2.5.0}/tests/test_runlog.py

@@ -1444,6 +1444,42 @@ class TestSaveOutputsVariants:
         assert path.name == "epoch_2_step_500"
         run.finish()
 
+    def test_dict_slot_produces_named_files(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+        a = np.zeros((4, 4, 3), dtype=np.uint8)
+        b = np.ones((4, 4, 3), dtype=np.uint8) * 128
+        run.save_outputs(epoch=0, step=0, rgb=dict(pred={"scene_a": a, "scene_b": b}))
+
+        pred_dir = run.dir / "outputs" / "epoch_0_step_0" / "rgb" / "pred"
+        names = sorted(f.name for f in pred_dir.iterdir())
+        assert names == ["scene_a.png", "scene_b.png"]
+        run.finish()
+
+    def test_dict_slot_in_aux(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+        m1 = np.ones((4, 4), dtype=np.float32)
+        m2 = np.zeros((4, 4), dtype=np.float32)
+        run.save_outputs(
+            epoch=0, step=0,
+            depth=dict(aux=dict(attn={"head_0": m1, "head_1": m2})),
+        )
+
+        attn_dir = run.dir / "outputs" / "epoch_0_step_0" / "depth" / "aux" / "attn"
+        names = sorted(f.name for f in attn_dir.iterdir())
+        assert names == ["head_0.npy", "head_1.npy"]
+        run.finish()
+
+    def test_mixed_dict_and_array_slots(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+        a = np.zeros((4, 4, 3), dtype=np.uint8)
+        run.save_outputs(epoch=0, step=0, rgb=dict(pred={"frame_x": a}, gt=a))
+
+        pred_dir = run.dir / "outputs" / "epoch_0_step_0" / "rgb" / "pred"
+        gt_dir = run.dir / "outputs" / "epoch_0_step_0" / "rgb" / "gt"
+        assert sorted(f.name for f in pred_dir.iterdir()) == ["frame_x.png"]
+        assert sorted(f.name for f in gt_dir.iterdir()) == ["0000.png"]
+        run.finish()
+
     def test_returns_output_base_path(self, tmp_path):
         run = euler_train.init(dir=str(tmp_path / "r"), config={})
         arr = np.zeros((4, 4, 3), dtype=np.uint8)
@@ -2042,3 +2078,179 @@ class TestInterruptHandling:
         meta = _read_json(run.dir / "meta.json")
         assert meta["status"] == "interrupted"
         assert "SIGINT" in meta["error"]
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+#  Metric naming
+# ═══════════════════════════════════════════════════════════════════════════
+
+_SAMPLE_METRIC_NAMING = {
+    "producer_key": "euler_train.weather_metric",
+    "producer_version": "0.1.0",
+    "namespaces": {
+        "depth.train": {
+            "axes": {
+                "kind": {"position": 0, "optional": False, "values": ["loss", "diag", "stat"]},
+                "stage": {"position": 1, "optional": True, "values": ["prior", "final"]},
+            }
+        },
+        "sys.train": {
+            "axes": {}
+        },
+    },
+}
+
+
+class TestMetricNaming:
+    def test_metric_naming_stored_in_meta_json(self, tmp_path):
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={"lr": 1e-3},
+            metric_naming=_SAMPLE_METRIC_NAMING,
+        )
+        meta = _read_json(run.dir / "meta.json")
+        assert meta["metric_naming"] == _SAMPLE_METRIC_NAMING
+        run.finish()
+
+    def test_metric_naming_none_by_default(self, tmp_path):
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+        )
+        meta = _read_json(run.dir / "meta.json")
+        assert "metric_naming" not in meta
+        run.finish()
+
+    def test_metric_naming_in_stream_init_event(self, tmp_path):
+        consumer = _RecordingStreamConsumer()
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+            stream=consumer,
+            metric_naming=_SAMPLE_METRIC_NAMING,
+        )
+        run.finish()
+
+        init_event = consumer.events[0]
+        assert init_event["type"] == "init"
+        assert init_event["meta"]["metric_naming"] == _SAMPLE_METRIC_NAMING
+
+    def test_metric_naming_not_in_stream_when_absent(self, tmp_path):
+        consumer = _RecordingStreamConsumer()
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+            stream=consumer,
+        )
+        run.finish()
+
+        init_event = consumer.events[0]
+        assert "metric_naming" not in init_event["meta"]
+
+    def test_metric_naming_preserved_on_resume(self, tmp_path):
+        run1 = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+            metric_naming=_SAMPLE_METRIC_NAMING,
+        )
+        run1.finish()
+
+        run2 = euler_train.init(
+            dir=str(run1.dir),
+        )
+        meta = _read_json(run2.dir / "meta.json")
+        assert meta["metric_naming"] == _SAMPLE_METRIC_NAMING
+        run2.finish()
+
+    def test_metric_naming_validation_not_dict(self, tmp_path):
+        with pytest.raises(TypeError, match="metric_naming must be a dict"):
+            euler_train.init(
+                dir=str(tmp_path / "r"),
+                config={},
+                metric_naming="invalid",
+            )
+
+    def test_metric_naming_validation_missing_namespaces(self, tmp_path):
+        with pytest.raises(ValueError, match="namespaces"):
+            euler_train.init(
+                dir=str(tmp_path / "r"),
+                config={},
+                metric_naming={"producer_key": "test"},
+            )
+
+    def test_metric_naming_validation_namespaces_not_dict(self, tmp_path):
+        with pytest.raises(ValueError, match="namespaces"):
+            euler_train.init(
+                dir=str(tmp_path / "r"),
+                config={},
+                metric_naming={"namespaces": "invalid"},
+            )
+
+    def test_metric_naming_validation_namespace_entry_not_dict(self, tmp_path):
+        with pytest.raises(ValueError, match="must be a dict"):
+            euler_train.init(
+                dir=str(tmp_path / "r"),
+                config={},
+                metric_naming={"namespaces": {"depth.train": "invalid"}},
+            )
+
+    def test_gpu_stats_namespaced_when_metric_naming_present(self, tmp_path):
+        """GPU stats should use sys.train.* keys when metric_naming is set."""
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+            metric_naming=_SAMPLE_METRIC_NAMING,
+        )
+        # Force GPU stats to be "available" with a mock
+        run._gpu_available = True
+
+        class _FakeUtil:
+            gpu = 75
+            memory = 50
+
+        class _FakeMem:
+            used = 4_000_000_000
+            total = 8_000_000_000
+
+        import unittest.mock as mock
+        with mock.patch.dict("sys.modules", {"pynvml": mock.MagicMock()}) as _:
+            import sys as _sys
+            pynvml_mock = _sys.modules["pynvml"]
+            pynvml_mock.nvmlDeviceGetUtilizationRates.return_value = _FakeUtil()
+            pynvml_mock.nvmlDeviceGetMemoryInfo.return_value = _FakeMem()
+
+            stats = run._get_gpu_stats()
+
+        assert "sys.train.gpu_util_pct" in stats
+        assert "sys.train.gpu_mem_used_gb" in stats
+        assert "gpu_util_pct" not in stats
+        run.finish()
+
+    def test_gpu_stats_flat_when_no_metric_naming(self, tmp_path):
+        """GPU stats should use flat keys when metric_naming is absent."""
+        run = euler_train.init(
+            dir=str(tmp_path / "r"),
+            config={},
+        )
+        run._gpu_available = True
+
+        class _FakeUtil:
+            gpu = 75
+            memory = 50
+
+        class _FakeMem:
+            used = 4_000_000_000
+            total = 8_000_000_000
+
+        import unittest.mock as mock
+        with mock.patch.dict("sys.modules", {"pynvml": mock.MagicMock()}) as _:
+            import sys as _sys
+            pynvml_mock = _sys.modules["pynvml"]
+            pynvml_mock.nvmlDeviceGetUtilizationRates.return_value = _FakeUtil()
+            pynvml_mock.nvmlDeviceGetMemoryInfo.return_value = _FakeMem()
+
+            stats = run._get_gpu_stats()
+
+        assert "gpu_util_pct" in stats
+        assert "sys.train.gpu_util_pct" not in stats
+        run.finish()