euler-train 2.9.0__tar.gz → 2.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {euler_train-2.9.0 → euler_train-2.10.0}/PKG-INFO +1 -1
  2. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/__init__.py +9 -0
  3. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/run.py +27 -0
  4. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/PKG-INFO +1 -1
  5. {euler_train-2.9.0 → euler_train-2.10.0}/pyproject.toml +1 -1
  6. {euler_train-2.9.0 → euler_train-2.10.0}/tests/test_runlog.py +156 -0
  7. {euler_train-2.9.0 → euler_train-2.10.0}/README.md +0 -0
  8. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/_dataset_contract.py +0 -0
  9. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/architecture.py +0 -0
  10. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/environment.py +0 -0
  11. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/git_info.py +0 -0
  12. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/outputs.py +0 -0
  13. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/serialization.py +0 -0
  14. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/slurm.py +0 -0
  15. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/stream.py +0 -0
  16. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/stream_check_cli.py +0 -0
  17. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/SOURCES.txt +0 -0
  18. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/dependency_links.txt +0 -0
  19. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/entry_points.txt +0 -0
  20. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/requires.txt +0 -0
  21. {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/top_level.txt +0 -0
  22. {euler_train-2.9.0 → euler_train-2.10.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-train
3
- Version: 2.9.0
3
+ Version: 2.10.0
4
4
  Summary: Lightweight file-based experiment logger for PyTorch
5
5
  Requires-Python: >=3.9
6
6
  Requires-Dist: numpy
@@ -42,6 +42,7 @@ def init(
42
42
  mode: str | None = None,
43
43
  stream: Any = None,
44
44
  metric_naming: dict[str, Any] | None = None,
45
+ pipeline: dict[str, Any] | None = None,
45
46
  ) -> Run:
46
47
  """Create a new run — or resume an existing one — and return the handle.
47
48
 
@@ -121,6 +122,13 @@ def init(
121
122
  in the stream init event so euler-view can resolve per-metric
122
123
  namespaces. Typically generated by
123
124
  ``euler_metric_naming.MetricNamespace.training_naming_config()``.
125
+ pipeline:
126
+ Optional pipeline identity mapping. When the run is launched by
127
+ an automated pipeline, pass a dict with at least ``attach_id``
128
+ (a non-empty string) so downstream processes can correlate
129
+ file-based artifacts back to the originating pipeline stage.
130
+ Stored in ``meta.json`` under ``pipeline``. Additional keys
131
+ (e.g. ``stage``, ``pipeline_id``) are preserved as-is.
124
132
  """
125
133
  return Run(
126
134
  dir=dir, config=config, meta=meta,
@@ -131,4 +139,5 @@ def init(
131
139
  evaluations=evaluations, mode=mode,
132
140
  stream=stream,
133
141
  metric_naming=metric_naming,
142
+ pipeline=pipeline,
134
143
  )
@@ -49,6 +49,7 @@ class Run:
49
49
  mode: str | None = None,
50
50
  stream: Any = None,
51
51
  metric_naming: dict[str, Any] | None = None,
52
+ pipeline: dict[str, Any] | None = None,
52
53
  ) -> None:
53
54
  # ── detect run-directory shorthand ────────────────────────
54
55
  # When `dir` points to an existing run directory (contains
@@ -182,6 +183,13 @@ class Run:
182
183
  _validate_metric_naming(metric_naming)
183
184
  self._meta["metric_naming"] = metric_naming
184
185
 
186
+ # ── pipeline identity (optional, for automated pipelines) ──
187
+ if pipeline is None:
188
+ pipeline = _pipeline_from_env()
189
+ if pipeline is not None:
190
+ _validate_pipeline(pipeline)
191
+ self._meta["pipeline"] = pipeline
192
+
185
193
  self._mark_mode_running()
186
194
  self._flush_meta()
187
195
  self._bind_stream()
@@ -1081,6 +1089,25 @@ def _validate_metric_naming(value: Any) -> None:
1081
1089
  )
1082
1090
 
1083
1091
 
1092
+ def _pipeline_from_env() -> dict[str, Any] | None:
1093
+ """Build a pipeline dict from ``$EULER_SESSION_ID`` when set."""
1094
+ session_id = os.environ.get("EULER_SESSION_ID", "").strip()
1095
+ if not session_id:
1096
+ return None
1097
+ return {"attach_id": session_id}
1098
+
1099
+
1100
+ def _validate_pipeline(value: Any) -> None:
1101
+ """Validate the pipeline identity payload."""
1102
+ if not isinstance(value, dict):
1103
+ raise TypeError("pipeline must be a dict")
1104
+ attach_id = value.get("attach_id")
1105
+ if not isinstance(attach_id, str) or not attach_id.strip():
1106
+ raise ValueError(
1107
+ "pipeline must contain a non-empty 'attach_id' string"
1108
+ )
1109
+
1110
+
1084
1111
  def _build_datasets_meta(
1085
1112
  datasets: Mapping[str, Any],
1086
1113
  existing: Any,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-train
3
- Version: 2.9.0
3
+ Version: 2.10.0
4
4
  Summary: Lightweight file-based experiment logger for PyTorch
5
5
  Requires-Python: >=3.9
6
6
  Requires-Dist: numpy
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "euler-train"
7
- version = "2.9.0"
7
+ version = "2.10.0"
8
8
  description = "Lightweight file-based experiment logger for PyTorch"
9
9
  requires-python = ">=3.9"
10
10
  dependencies = ["numpy", "euler-dataset-contract"]
@@ -2592,3 +2592,159 @@ class TestMetricNaming:
2592
2592
  assert "gpu_util_pct" in stats
2593
2593
  assert "sys.train.gpu_util_pct" not in stats
2594
2594
  run.finish()
2595
+
2596
+
2597
+ class TestPipeline:
2598
+ def test_pipeline_stored_in_meta_json(self, tmp_path):
2599
+ pipeline = {"attach_id": "pipe-abc-123"}
2600
+ run = euler_train.init(
2601
+ dir=str(tmp_path / "r"),
2602
+ config={"lr": 1e-3},
2603
+ pipeline=pipeline,
2604
+ )
2605
+ meta = _read_json(run.dir / "meta.json")
2606
+ assert meta["pipeline"] == pipeline
2607
+ run.finish()
2608
+
2609
+ def test_pipeline_none_by_default(self, tmp_path):
2610
+ run = euler_train.init(
2611
+ dir=str(tmp_path / "r"),
2612
+ config={},
2613
+ )
2614
+ meta = _read_json(run.dir / "meta.json")
2615
+ assert "pipeline" not in meta
2616
+ run.finish()
2617
+
2618
+ def test_pipeline_with_extra_fields(self, tmp_path):
2619
+ pipeline = {
2620
+ "attach_id": "pipe-xyz",
2621
+ "stage": "training",
2622
+ "pipeline_id": "nightly-run-42",
2623
+ }
2624
+ run = euler_train.init(
2625
+ dir=str(tmp_path / "r"),
2626
+ config={},
2627
+ pipeline=pipeline,
2628
+ )
2629
+ meta = _read_json(run.dir / "meta.json")
2630
+ assert meta["pipeline"]["attach_id"] == "pipe-xyz"
2631
+ assert meta["pipeline"]["stage"] == "training"
2632
+ assert meta["pipeline"]["pipeline_id"] == "nightly-run-42"
2633
+ run.finish()
2634
+
2635
+ def test_pipeline_preserved_on_resume(self, tmp_path):
2636
+ pipeline = {"attach_id": "pipe-resume-test"}
2637
+ run1 = euler_train.init(
2638
+ dir=str(tmp_path / "r"),
2639
+ config={},
2640
+ pipeline=pipeline,
2641
+ )
2642
+ run1.finish()
2643
+
2644
+ run2 = euler_train.init(
2645
+ dir=str(run1.dir),
2646
+ )
2647
+ meta = _read_json(run2.dir / "meta.json")
2648
+ assert meta["pipeline"] == pipeline
2649
+ run2.finish()
2650
+
2651
+ def test_pipeline_in_stream_init_event(self, tmp_path):
2652
+ consumer = _RecordingStreamConsumer()
2653
+ pipeline = {"attach_id": "stream-pipe-id"}
2654
+ run = euler_train.init(
2655
+ dir=str(tmp_path / "r"),
2656
+ config={},
2657
+ stream=consumer,
2658
+ pipeline=pipeline,
2659
+ )
2660
+ run.finish()
2661
+
2662
+ init_event = consumer.events[0]
2663
+ assert init_event["type"] == "init"
2664
+ assert init_event["meta"]["pipeline"] == pipeline
2665
+
2666
+ def test_pipeline_not_in_stream_when_absent(self, tmp_path):
2667
+ consumer = _RecordingStreamConsumer()
2668
+ run = euler_train.init(
2669
+ dir=str(tmp_path / "r"),
2670
+ config={},
2671
+ stream=consumer,
2672
+ )
2673
+ run.finish()
2674
+
2675
+ init_event = consumer.events[0]
2676
+ assert "pipeline" not in init_event["meta"]
2677
+
2678
+ def test_pipeline_validation_not_dict(self, tmp_path):
2679
+ with pytest.raises(TypeError, match="pipeline must be a dict"):
2680
+ euler_train.init(
2681
+ dir=str(tmp_path / "r"),
2682
+ config={},
2683
+ pipeline="invalid",
2684
+ )
2685
+
2686
+ def test_pipeline_validation_missing_attach_id(self, tmp_path):
2687
+ with pytest.raises(ValueError, match="attach_id"):
2688
+ euler_train.init(
2689
+ dir=str(tmp_path / "r"),
2690
+ config={},
2691
+ pipeline={},
2692
+ )
2693
+
2694
+ def test_pipeline_validation_empty_attach_id(self, tmp_path):
2695
+ with pytest.raises(ValueError, match="attach_id"):
2696
+ euler_train.init(
2697
+ dir=str(tmp_path / "r"),
2698
+ config={},
2699
+ pipeline={"attach_id": " "},
2700
+ )
2701
+
2702
+ def test_pipeline_validation_non_string_attach_id(self, tmp_path):
2703
+ with pytest.raises(ValueError, match="attach_id"):
2704
+ euler_train.init(
2705
+ dir=str(tmp_path / "r"),
2706
+ config={},
2707
+ pipeline={"attach_id": 42},
2708
+ )
2709
+
2710
+ def test_pipeline_from_env_var(self, tmp_path, monkeypatch):
2711
+ monkeypatch.setenv("EULER_SESSION_ID", "env-session-99")
2712
+ run = euler_train.init(
2713
+ dir=str(tmp_path / "r"),
2714
+ config={},
2715
+ )
2716
+ meta = _read_json(run.dir / "meta.json")
2717
+ assert meta["pipeline"] == {"attach_id": "env-session-99"}
2718
+ run.finish()
2719
+
2720
+ def test_pipeline_env_var_ignored_when_explicit(self, tmp_path, monkeypatch):
2721
+ monkeypatch.setenv("EULER_SESSION_ID", "env-session-99")
2722
+ explicit = {"attach_id": "explicit-id", "stage": "eval"}
2723
+ run = euler_train.init(
2724
+ dir=str(tmp_path / "r"),
2725
+ config={},
2726
+ pipeline=explicit,
2727
+ )
2728
+ meta = _read_json(run.dir / "meta.json")
2729
+ assert meta["pipeline"] == explicit
2730
+ run.finish()
2731
+
2732
+ def test_pipeline_env_var_empty_ignored(self, tmp_path, monkeypatch):
2733
+ monkeypatch.setenv("EULER_SESSION_ID", " ")
2734
+ run = euler_train.init(
2735
+ dir=str(tmp_path / "r"),
2736
+ config={},
2737
+ )
2738
+ meta = _read_json(run.dir / "meta.json")
2739
+ assert "pipeline" not in meta
2740
+ run.finish()
2741
+
2742
+ def test_pipeline_env_var_unset_ignored(self, tmp_path, monkeypatch):
2743
+ monkeypatch.delenv("EULER_SESSION_ID", raising=False)
2744
+ run = euler_train.init(
2745
+ dir=str(tmp_path / "r"),
2746
+ config={},
2747
+ )
2748
+ meta = _read_json(run.dir / "meta.json")
2749
+ assert "pipeline" not in meta
2750
+ run.finish()
File without changes
File without changes