euler-train 2.9.0__tar.gz → 2.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_train-2.9.0 → euler_train-2.10.0}/PKG-INFO +1 -1
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/__init__.py +9 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/run.py +27 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/PKG-INFO +1 -1
- {euler_train-2.9.0 → euler_train-2.10.0}/pyproject.toml +1 -1
- {euler_train-2.9.0 → euler_train-2.10.0}/tests/test_runlog.py +156 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/README.md +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/_dataset_contract.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/architecture.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/environment.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/git_info.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/outputs.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/serialization.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/slurm.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/stream.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train/stream_check_cli.py +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/SOURCES.txt +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/dependency_links.txt +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/entry_points.txt +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/requires.txt +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/euler_train.egg-info/top_level.txt +0 -0
- {euler_train-2.9.0 → euler_train-2.10.0}/setup.cfg +0 -0
|
@@ -42,6 +42,7 @@ def init(
|
|
|
42
42
|
mode: str | None = None,
|
|
43
43
|
stream: Any = None,
|
|
44
44
|
metric_naming: dict[str, Any] | None = None,
|
|
45
|
+
pipeline: dict[str, Any] | None = None,
|
|
45
46
|
) -> Run:
|
|
46
47
|
"""Create a new run — or resume an existing one — and return the handle.
|
|
47
48
|
|
|
@@ -121,6 +122,13 @@ def init(
|
|
|
121
122
|
in the stream init event so euler-view can resolve per-metric
|
|
122
123
|
namespaces. Typically generated by
|
|
123
124
|
``euler_metric_naming.MetricNamespace.training_naming_config()``.
|
|
125
|
+
pipeline:
|
|
126
|
+
Optional pipeline identity mapping. When the run is launched by
|
|
127
|
+
an automated pipeline, pass a dict with at least ``attach_id``
|
|
128
|
+
(a non-empty string) so downstream processes can correlate
|
|
129
|
+
file-based artifacts back to the originating pipeline stage.
|
|
130
|
+
Stored in ``meta.json`` under ``pipeline``. Additional keys
|
|
131
|
+
(e.g. ``stage``, ``pipeline_id``) are preserved as-is.
|
|
124
132
|
"""
|
|
125
133
|
return Run(
|
|
126
134
|
dir=dir, config=config, meta=meta,
|
|
@@ -131,4 +139,5 @@ def init(
|
|
|
131
139
|
evaluations=evaluations, mode=mode,
|
|
132
140
|
stream=stream,
|
|
133
141
|
metric_naming=metric_naming,
|
|
142
|
+
pipeline=pipeline,
|
|
134
143
|
)
|
|
@@ -49,6 +49,7 @@ class Run:
|
|
|
49
49
|
mode: str | None = None,
|
|
50
50
|
stream: Any = None,
|
|
51
51
|
metric_naming: dict[str, Any] | None = None,
|
|
52
|
+
pipeline: dict[str, Any] | None = None,
|
|
52
53
|
) -> None:
|
|
53
54
|
# ── detect run-directory shorthand ────────────────────────
|
|
54
55
|
# When `dir` points to an existing run directory (contains
|
|
@@ -182,6 +183,13 @@ class Run:
|
|
|
182
183
|
_validate_metric_naming(metric_naming)
|
|
183
184
|
self._meta["metric_naming"] = metric_naming
|
|
184
185
|
|
|
186
|
+
# ── pipeline identity (optional, for automated pipelines) ──
|
|
187
|
+
if pipeline is None:
|
|
188
|
+
pipeline = _pipeline_from_env()
|
|
189
|
+
if pipeline is not None:
|
|
190
|
+
_validate_pipeline(pipeline)
|
|
191
|
+
self._meta["pipeline"] = pipeline
|
|
192
|
+
|
|
185
193
|
self._mark_mode_running()
|
|
186
194
|
self._flush_meta()
|
|
187
195
|
self._bind_stream()
|
|
@@ -1081,6 +1089,25 @@ def _validate_metric_naming(value: Any) -> None:
|
|
|
1081
1089
|
)
|
|
1082
1090
|
|
|
1083
1091
|
|
|
1092
|
+
def _pipeline_from_env() -> dict[str, Any] | None:
|
|
1093
|
+
"""Build a pipeline dict from ``$EULER_SESSION_ID`` when set."""
|
|
1094
|
+
session_id = os.environ.get("EULER_SESSION_ID", "").strip()
|
|
1095
|
+
if not session_id:
|
|
1096
|
+
return None
|
|
1097
|
+
return {"attach_id": session_id}
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def _validate_pipeline(value: Any) -> None:
|
|
1101
|
+
"""Validate the pipeline identity payload."""
|
|
1102
|
+
if not isinstance(value, dict):
|
|
1103
|
+
raise TypeError("pipeline must be a dict")
|
|
1104
|
+
attach_id = value.get("attach_id")
|
|
1105
|
+
if not isinstance(attach_id, str) or not attach_id.strip():
|
|
1106
|
+
raise ValueError(
|
|
1107
|
+
"pipeline must contain a non-empty 'attach_id' string"
|
|
1108
|
+
)
|
|
1109
|
+
|
|
1110
|
+
|
|
1084
1111
|
def _build_datasets_meta(
|
|
1085
1112
|
datasets: Mapping[str, Any],
|
|
1086
1113
|
existing: Any,
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "euler-train"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.10.0"
|
|
8
8
|
description = "Lightweight file-based experiment logger for PyTorch"
|
|
9
9
|
requires-python = ">=3.9"
|
|
10
10
|
dependencies = ["numpy", "euler-dataset-contract"]
|
|
@@ -2592,3 +2592,159 @@ class TestMetricNaming:
|
|
|
2592
2592
|
assert "gpu_util_pct" in stats
|
|
2593
2593
|
assert "sys.train.gpu_util_pct" not in stats
|
|
2594
2594
|
run.finish()
|
|
2595
|
+
|
|
2596
|
+
|
|
2597
|
+
class TestPipeline:
|
|
2598
|
+
def test_pipeline_stored_in_meta_json(self, tmp_path):
|
|
2599
|
+
pipeline = {"attach_id": "pipe-abc-123"}
|
|
2600
|
+
run = euler_train.init(
|
|
2601
|
+
dir=str(tmp_path / "r"),
|
|
2602
|
+
config={"lr": 1e-3},
|
|
2603
|
+
pipeline=pipeline,
|
|
2604
|
+
)
|
|
2605
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2606
|
+
assert meta["pipeline"] == pipeline
|
|
2607
|
+
run.finish()
|
|
2608
|
+
|
|
2609
|
+
def test_pipeline_none_by_default(self, tmp_path):
|
|
2610
|
+
run = euler_train.init(
|
|
2611
|
+
dir=str(tmp_path / "r"),
|
|
2612
|
+
config={},
|
|
2613
|
+
)
|
|
2614
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2615
|
+
assert "pipeline" not in meta
|
|
2616
|
+
run.finish()
|
|
2617
|
+
|
|
2618
|
+
def test_pipeline_with_extra_fields(self, tmp_path):
|
|
2619
|
+
pipeline = {
|
|
2620
|
+
"attach_id": "pipe-xyz",
|
|
2621
|
+
"stage": "training",
|
|
2622
|
+
"pipeline_id": "nightly-run-42",
|
|
2623
|
+
}
|
|
2624
|
+
run = euler_train.init(
|
|
2625
|
+
dir=str(tmp_path / "r"),
|
|
2626
|
+
config={},
|
|
2627
|
+
pipeline=pipeline,
|
|
2628
|
+
)
|
|
2629
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2630
|
+
assert meta["pipeline"]["attach_id"] == "pipe-xyz"
|
|
2631
|
+
assert meta["pipeline"]["stage"] == "training"
|
|
2632
|
+
assert meta["pipeline"]["pipeline_id"] == "nightly-run-42"
|
|
2633
|
+
run.finish()
|
|
2634
|
+
|
|
2635
|
+
def test_pipeline_preserved_on_resume(self, tmp_path):
|
|
2636
|
+
pipeline = {"attach_id": "pipe-resume-test"}
|
|
2637
|
+
run1 = euler_train.init(
|
|
2638
|
+
dir=str(tmp_path / "r"),
|
|
2639
|
+
config={},
|
|
2640
|
+
pipeline=pipeline,
|
|
2641
|
+
)
|
|
2642
|
+
run1.finish()
|
|
2643
|
+
|
|
2644
|
+
run2 = euler_train.init(
|
|
2645
|
+
dir=str(run1.dir),
|
|
2646
|
+
)
|
|
2647
|
+
meta = _read_json(run2.dir / "meta.json")
|
|
2648
|
+
assert meta["pipeline"] == pipeline
|
|
2649
|
+
run2.finish()
|
|
2650
|
+
|
|
2651
|
+
def test_pipeline_in_stream_init_event(self, tmp_path):
|
|
2652
|
+
consumer = _RecordingStreamConsumer()
|
|
2653
|
+
pipeline = {"attach_id": "stream-pipe-id"}
|
|
2654
|
+
run = euler_train.init(
|
|
2655
|
+
dir=str(tmp_path / "r"),
|
|
2656
|
+
config={},
|
|
2657
|
+
stream=consumer,
|
|
2658
|
+
pipeline=pipeline,
|
|
2659
|
+
)
|
|
2660
|
+
run.finish()
|
|
2661
|
+
|
|
2662
|
+
init_event = consumer.events[0]
|
|
2663
|
+
assert init_event["type"] == "init"
|
|
2664
|
+
assert init_event["meta"]["pipeline"] == pipeline
|
|
2665
|
+
|
|
2666
|
+
def test_pipeline_not_in_stream_when_absent(self, tmp_path):
|
|
2667
|
+
consumer = _RecordingStreamConsumer()
|
|
2668
|
+
run = euler_train.init(
|
|
2669
|
+
dir=str(tmp_path / "r"),
|
|
2670
|
+
config={},
|
|
2671
|
+
stream=consumer,
|
|
2672
|
+
)
|
|
2673
|
+
run.finish()
|
|
2674
|
+
|
|
2675
|
+
init_event = consumer.events[0]
|
|
2676
|
+
assert "pipeline" not in init_event["meta"]
|
|
2677
|
+
|
|
2678
|
+
def test_pipeline_validation_not_dict(self, tmp_path):
|
|
2679
|
+
with pytest.raises(TypeError, match="pipeline must be a dict"):
|
|
2680
|
+
euler_train.init(
|
|
2681
|
+
dir=str(tmp_path / "r"),
|
|
2682
|
+
config={},
|
|
2683
|
+
pipeline="invalid",
|
|
2684
|
+
)
|
|
2685
|
+
|
|
2686
|
+
def test_pipeline_validation_missing_attach_id(self, tmp_path):
|
|
2687
|
+
with pytest.raises(ValueError, match="attach_id"):
|
|
2688
|
+
euler_train.init(
|
|
2689
|
+
dir=str(tmp_path / "r"),
|
|
2690
|
+
config={},
|
|
2691
|
+
pipeline={},
|
|
2692
|
+
)
|
|
2693
|
+
|
|
2694
|
+
def test_pipeline_validation_empty_attach_id(self, tmp_path):
|
|
2695
|
+
with pytest.raises(ValueError, match="attach_id"):
|
|
2696
|
+
euler_train.init(
|
|
2697
|
+
dir=str(tmp_path / "r"),
|
|
2698
|
+
config={},
|
|
2699
|
+
pipeline={"attach_id": " "},
|
|
2700
|
+
)
|
|
2701
|
+
|
|
2702
|
+
def test_pipeline_validation_non_string_attach_id(self, tmp_path):
|
|
2703
|
+
with pytest.raises(ValueError, match="attach_id"):
|
|
2704
|
+
euler_train.init(
|
|
2705
|
+
dir=str(tmp_path / "r"),
|
|
2706
|
+
config={},
|
|
2707
|
+
pipeline={"attach_id": 42},
|
|
2708
|
+
)
|
|
2709
|
+
|
|
2710
|
+
def test_pipeline_from_env_var(self, tmp_path, monkeypatch):
|
|
2711
|
+
monkeypatch.setenv("EULER_SESSION_ID", "env-session-99")
|
|
2712
|
+
run = euler_train.init(
|
|
2713
|
+
dir=str(tmp_path / "r"),
|
|
2714
|
+
config={},
|
|
2715
|
+
)
|
|
2716
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2717
|
+
assert meta["pipeline"] == {"attach_id": "env-session-99"}
|
|
2718
|
+
run.finish()
|
|
2719
|
+
|
|
2720
|
+
def test_pipeline_env_var_ignored_when_explicit(self, tmp_path, monkeypatch):
|
|
2721
|
+
monkeypatch.setenv("EULER_SESSION_ID", "env-session-99")
|
|
2722
|
+
explicit = {"attach_id": "explicit-id", "stage": "eval"}
|
|
2723
|
+
run = euler_train.init(
|
|
2724
|
+
dir=str(tmp_path / "r"),
|
|
2725
|
+
config={},
|
|
2726
|
+
pipeline=explicit,
|
|
2727
|
+
)
|
|
2728
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2729
|
+
assert meta["pipeline"] == explicit
|
|
2730
|
+
run.finish()
|
|
2731
|
+
|
|
2732
|
+
def test_pipeline_env_var_empty_ignored(self, tmp_path, monkeypatch):
|
|
2733
|
+
monkeypatch.setenv("EULER_SESSION_ID", " ")
|
|
2734
|
+
run = euler_train.init(
|
|
2735
|
+
dir=str(tmp_path / "r"),
|
|
2736
|
+
config={},
|
|
2737
|
+
)
|
|
2738
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2739
|
+
assert "pipeline" not in meta
|
|
2740
|
+
run.finish()
|
|
2741
|
+
|
|
2742
|
+
def test_pipeline_env_var_unset_ignored(self, tmp_path, monkeypatch):
|
|
2743
|
+
monkeypatch.delenv("EULER_SESSION_ID", raising=False)
|
|
2744
|
+
run = euler_train.init(
|
|
2745
|
+
dir=str(tmp_path / "r"),
|
|
2746
|
+
config={},
|
|
2747
|
+
)
|
|
2748
|
+
meta = _read_json(run.dir / "meta.json")
|
|
2749
|
+
assert "pipeline" not in meta
|
|
2750
|
+
run.finish()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|