euler-train 2.1.0__tar.gz → 2.2.0__tar.gz

{euler_train-2.1.0 → euler_train-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.1.0
+Version: 2.2.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy

{euler_train-2.1.0 → euler_train-2.2.0}/README.md

@@ -82,7 +82,7 @@ The run ID and directory are available as `run.run_id` and `run.dir`.
 
 ### `euler_train.init(dir, config=None, meta=None, output_formats=None, output_visualization=None, run_id=None, datasets=None, run_name=None, evaluations=None, mode=None) → Run`
 
-Creates the run directory and writes `meta.json`, `config.json`, `code_ref.json`, and `run_environment.json`. On resume (`run_id` provided), only `meta.json` and `config.json` are updated.
+Creates the run directory and writes `meta.json`, `config.json`, `code_ref.json`, and `run_environment.json`. On resume (`run_id` provided), only `meta.json` and `config.json` are updated. `meta.json` also maintains an `updated_at` map with last-write timestamps for tracked artifacts.
 
 | Parameter | Type | Description |
 |---|---|---|
@@ -365,6 +365,24 @@ Auto-managed, not written to directly.
       "traceback": "Traceback (most recent call last):\n  ..."
     }
   },
+  "updated_at": {
+    "meta.json": {
+      "time": 1706403300.0,
+      "iso": "2025-01-28T16:25:00"
+    },
+    "config.json": {
+      "time": 1706400000.0,
+      "iso": "2025-01-28T15:30:42"
+    },
+    "train.jsonl": {
+      "time": 1706402999.0,
+      "iso": "2025-01-28T16:19:59"
+    },
+    "outputs/epoch_12_step_4800": {
+      "time": 1706403010.0,
+      "iso": "2025-01-28T16:20:10"
+    }
+  },
   "error": "RuntimeError: CUDA OOM",
   "traceback": "Traceback (most recent call last):\n  ..."
 }
@@ -375,6 +393,7 @@ Auto-managed, not written to directly.
 - `datasets` is only present when `datasets=...` is passed to `euler_train.init`.
 - `evaluations` is only present when evaluations are provided via `evaluations=...` on `init()` or added via `run.add_evaluation()`.
 - `modes` is only present when `mode=...` is passed to `euler_train.init`; each key stores the latest lifecycle snapshot for that mode.
+- `updated_at` stores last-write timestamps for tracked artifacts. Keys are run-relative paths when the artifact lives inside the run directory (for example `train.jsonl` or `outputs/epoch_2_step_500`) and absolute paths for external artifacts such as custom checkpoint locations.
 - `error` is only present when `status` is `"crashed"` (context manager / excepthook) or `"interrupted"` (SIGTERM/SIGINT). `traceback` is only present when `status` is `"crashed"`. When `mode=...` is set, the same fields are mirrored under `modes[mode]`.
 
 A formal JSON Schema for `meta.json` is available at [`meta-schema.json`](meta-schema.json).

{euler_train-2.1.0 → euler_train-2.2.0}/euler_train/run.py

@@ -82,6 +82,7 @@ class Run:
         self._gpu_stats_every: int = gpu_stats_every
         self.run_name: str | None = run_name
         self.mode: str | None = _normalize_mode(mode)
+        self._pending_updated_at: dict[str, dict[str, Any]] = {}
 
         # ── config ────────────────────────────────────────────────
         if resuming:
@@ -92,12 +93,19 @@ class Run:
         else:
             self.config: dict = normalize_config(config)
 
-        write_json(self.dir / "config.json", self.config)
+        config_path = self.dir / "config.json"
+        write_json(config_path, self.config)
+        self._record_artifact_update(config_path)
 
         # ── code ref & run environment (fresh runs only) ─────────
         if not resuming:
-            write_json(self.dir / "code_ref.json", get_code_ref())
-            write_json(self.dir / "run_environment.json", get_run_environment())
+            code_ref_path = self.dir / "code_ref.json"
+            write_json(code_ref_path, get_code_ref())
+            self._record_artifact_update(code_ref_path)
+
+            run_environment_path = self.dir / "run_environment.json"
+            write_json(run_environment_path, get_run_environment())
+            self._record_artifact_update(run_environment_path)
 
         # ── meta ──────────────────────────────────────────────────
         if resuming:
@@ -238,7 +246,10 @@ class Run:
             record.update(self._get_gpu_stats())
 
         filename = "train.jsonl" if mode == "train" else "val.jsonl"
-        append_jsonl(self.dir / filename, record)
+        log_path = self.dir / filename
+        append_jsonl(log_path, record)
+        self._record_artifact_update(log_path)
+        self._flush_meta()
 
     # ── GPU stats ─────────────────────────────────────────────────
 
@@ -313,6 +324,9 @@ class Run:
                 self._output_visualization,
                 output_type,
             )
+        if base.exists():
+            self._record_artifact_update(base)
+            self._flush_meta()
         return base
 
     # ── checkpoints ───────────────────────────────────────────────
@@ -339,6 +353,7 @@ class Run:
         if self.checkpoint_dir is not None:
             self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
             self._meta["checkpoint_dir"] = str(self.checkpoint_dir)
+            self._record_artifact_update(self.checkpoint_dir)
             self._flush_meta()
             log.info("Checkpoint dir: %s", self.checkpoint_dir)
             return self.checkpoint_dir
@@ -357,6 +372,7 @@ class Run:
             )
         self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
         self._meta["checkpoint_dir"] = str(self.checkpoint_dir)
+        self._record_artifact_update(self.checkpoint_dir)
         self._flush_meta()
         log.info("Checkpoint dir: %s", self.checkpoint_dir)
         return self.checkpoint_dir
@@ -415,6 +431,7 @@ class Run:
         :meth:`save_checkpoint` call it automatically.  When *is_best*
         is ``True``, any previous checkpoint marked as best is cleared.
         """
+        checkpoint_path = Path(path)
         checkpoints: list[dict[str, Any]] = self._meta.get("checkpoints", [])
 
         if is_best:
@@ -422,7 +439,7 @@ class Run:
                 entry.pop("is_best", None)
 
         record: dict[str, Any] = {
-            "path": str(path),
+            "path": str(checkpoint_path),
             "epoch": epoch,
             "step": step,
         }
@@ -431,6 +448,7 @@ class Run:
 
         checkpoints.append(record)
         self._meta["checkpoints"] = checkpoints
+        self._record_artifact_update(checkpoint_path)
         self._flush_meta()
 
     # ── architecture export ─────────────────────────────────────────
@@ -467,6 +485,7 @@ class Run:
             model, dummy_input, self.dir / "architecture.onnx"
         )
         self._meta["architecture"] = "architecture.onnx"
+        self._record_artifact_update(output_path)
         self._flush_meta()
         return output_path
 
@@ -652,7 +671,45 @@ class Run:
         self._meta["modes"] = modes
         return modes
 
+    def _updated_at_store(self) -> dict[str, Any]:
+        if not hasattr(self, "_meta"):
+            return self._pending_updated_at
+
+        updated_at = self._meta.get("updated_at")
+        if not isinstance(updated_at, dict):
+            updated_at = {}
+            self._meta["updated_at"] = updated_at
+
+        if self._pending_updated_at:
+            updated_at.update(self._pending_updated_at)
+            self._pending_updated_at.clear()
+
+        return updated_at
+
+    def _record_artifact_update(
+        self,
+        path: str | Path,
+        *,
+        when: float | None = None,
+    ) -> None:
+        artifact_path = Path(path)
+        ts = when if when is not None else _artifact_mtime(artifact_path)
+        if ts is None:
+            ts = time.time()
+        self._updated_at_store()[self._artifact_key(artifact_path)] = (
+            _updated_at_entry(ts)
+        )
+
+    def _artifact_key(self, path: Path) -> str:
+        if path.is_absolute():
+            try:
+                return path.relative_to(self.dir).as_posix()
+            except ValueError:
+                return str(path)
+        return path.as_posix()
+
     def _flush_meta(self) -> None:
+        self._record_artifact_update(self.dir / "meta.json", when=time.time())
         write_json(self.dir / "meta.json", self._meta)
 
     def __repr__(self) -> str:
@@ -732,6 +789,20 @@ def _isotime(ts: float) -> str:
     return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(ts))
 
 
+def _updated_at_entry(ts: float) -> dict[str, Any]:
+    return {
+        "time": ts,
+        "iso": _isotime(ts),
+    }
+
+
+def _artifact_mtime(path: Path) -> float | None:
+    try:
+        return path.stat().st_mtime
+    except OSError:
+        return None
+
+
 def _normalize_mode(mode: str | None) -> str | None:
     if mode is None:
         return None
@@ -1050,6 +1121,9 @@ def _read_ds_crawler_descriptor(path: str) -> dict[str, Any]:
     try:
         cfg = load_dataset_config({"path": path})
     except Exception:
+        descriptor = _read_output_json_descriptor(path)
+        if descriptor:
+            return descriptor
         try:
             contract = get_dataset_contract(path)
         except Exception:
@@ -1082,6 +1156,69 @@ def _read_ds_crawler_descriptor(path: str) -> dict[str, Any]:
     return descriptor
 
 
+def _read_output_json_descriptor(path: str) -> dict[str, Any]:
+    try:
+        from ds_crawler import zip_utils
+    except Exception:
+        return {}
+
+    try:
+        raw = zip_utils.read_metadata_json(Path(path), "output.json")
+    except Exception:
+        return {}
+
+    return _descriptor_from_output_json(raw, path)
+
+
+def _descriptor_from_output_json(raw: Any, path: str) -> dict[str, Any]:
+    if isinstance(raw, Mapping):
+        candidates: list[Any]
+        modalities = raw.get("modalities")
+        if isinstance(modalities, list):
+            candidates = modalities
+        else:
+            candidates = [raw]
+    elif isinstance(raw, list):
+        candidates = raw
+    else:
+        return {}
+
+    selected: Mapping[str, Any] | None = None
+    path_name = Path(path).name
+    for candidate in candidates:
+        if not isinstance(candidate, Mapping):
+            continue
+        if _as_non_empty_str(candidate.get("name")) == path_name:
+            selected = candidate
+            break
+        if selected is None:
+            selected = candidate
+
+    if selected is None:
+        return {}
+
+    descriptor: dict[str, Any] = {}
+    properties = selected.get("properties")
+    merged_properties = dict(properties) if isinstance(properties, Mapping) else {}
+
+    euler_train_props = selected.get("euler_train")
+    if isinstance(euler_train_props, Mapping):
+        merged_properties["euler_train"] = dict(euler_train_props)
+
+    if merged_properties:
+        descriptor["properties"] = merged_properties
+
+    modality_key = _as_non_empty_str(selected.get("type"))
+    if modality_key is not None:
+        descriptor["modality_key"] = modality_key
+
+    hierarchy_regex = _as_non_empty_str(selected.get("hierarchy_regex"))
+    if hierarchy_regex is not None:
+        descriptor["hierarchy_regex"] = hierarchy_regex
+
+    return descriptor
+
+
 def _build_modality_entry(
     name: str,
     path: str,

{euler_train-2.1.0 → euler_train-2.2.0}/euler_train.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-train
-Version: 2.1.0
+Version: 2.2.0
 Summary: Lightweight file-based experiment logger for PyTorch
 Requires-Python: >=3.9
 Requires-Dist: numpy

{euler_train-2.1.0 → euler_train-2.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "euler-train"
-version = "2.1.0"
+version = "2.2.0"
 description = "Lightweight file-based experiment logger for PyTorch"
 requires-python = ">=3.9"
 dependencies = ["numpy", "euler-dataset-contract"]

{euler_train-2.1.0 → euler_train-2.2.0}/tests/test_runlog.py

@@ -27,6 +27,12 @@ def _read_jsonl(path: Path) -> list[dict]:
     return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
 
 
+def _assert_updated_at_entry(entry: dict) -> None:
+    assert isinstance(entry["time"], (int, float))
+    assert isinstance(entry["iso"], str)
+    assert entry["iso"]
+
+
 class _DummyModality:
     def __init__(self, path: str) -> None:
         self.path = path
@@ -228,6 +234,57 @@ class TestInit:
         run.finish()
 
 
+class TestUpdatedAt:
+    def test_init_tracks_bootstrap_artifacts(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+
+        meta = _read_json(run.dir / "meta.json")
+        updated_at = meta["updated_at"]
+
+        for key in (
+            "meta.json",
+            "config.json",
+            "code_ref.json",
+            "run_environment.json",
+        ):
+            assert key in updated_at
+            _assert_updated_at_entry(updated_at[key])
+        run.finish()
+
+    def test_log_tracks_train_and_val_jsonl(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+
+        run.log({"loss": 1.0}, step=0, epoch=0)
+        run.log({"metric": 0.9}, step=1, epoch=0, mode="val")
+
+        meta = _read_json(run.dir / "meta.json")
+        updated_at = meta["updated_at"]
+        _assert_updated_at_entry(updated_at["train.jsonl"])
+        _assert_updated_at_entry(updated_at["val.jsonl"])
+        _assert_updated_at_entry(updated_at["meta.json"])
+        run.finish()
+
+    def test_save_outputs_tracks_output_directory(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+        arr = np.zeros((4, 4, 3), dtype=np.uint8)
+
+        run.save_outputs(epoch=0, step=0, rgb=dict(pred=arr))
+
+        meta = _read_json(run.dir / "meta.json")
+        _assert_updated_at_entry(meta["updated_at"]["outputs/epoch_0_step_0"])
+        run.finish()
+
+    def test_log_saved_checkpoint_tracks_checkpoint_path(self, tmp_path):
+        run = euler_train.init(dir=str(tmp_path / "r"), config={})
+        checkpoint_path = tmp_path / "external" / "epoch_1.pt"
+
+        run.log_saved_checkpoint(checkpoint_path, epoch=1, step=10)
+
+        meta = _read_json(run.dir / "meta.json")
+        _assert_updated_at_entry(meta["updated_at"][str(checkpoint_path)])
+        run.finish()
+
+
 class TestDatasetMetadata:
     def test_prefers_dataset_describe_for_runlog_contract(
         self, tmp_path, monkeypatch