legend-dataflow-scripts 0.4.0a1__tar.gz → 0.4.0a2__tar.gz

{legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.4.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.4.0a1
+Version: 0.4.0a2
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a2/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.4.0a1
+Version: 0.4.0a2
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a2}/src/legend_dataflow_scripts.egg-info/scm_file_list.json

@@ -60,6 +60,7 @@
     "tests/test_execenv.py",
     "tests/test_lgdo_utils.py",
     "tests/test_peak_selection.py",
+    "tests/test_pre_compile_catalog.py",
     "tests/test_util.py",
     "tests/test_utils_helpers.py"
   ]

legend_dataflow_scripts-0.4.0a2/src/legend_dataflow_scripts.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.4.0a2",
+  "distance": 0,
+  "node": "g4e182698a9613efc0e62adc5eff80c46ec6449c7",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-06"
+}

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a2}/src/legenddataflowscripts/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.4.0a1'
-__version_tuple__ = version_tuple = (0, 4, 0, 'a1')
+__version__ = version = '0.4.0a2'
+__version_tuple__ = version_tuple = (0, 4, 0, 'a2')
 
-__commit_id__ = commit_id = 'g30429c604'
+__commit_id__ = commit_id = 'g4e182698a'

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a2}/src/legenddataflowscripts/utils/alias_table.py

@@ -7,7 +7,7 @@ import h5py
 
 
 def convert_parents_to_structs(h5group):
-    """Recursively annotate HDF5 parent groups with LGDO ``struct`` datatype attributes.
+    """Annotate HDF5 parent groups with LGDO ``struct`` datatype attributes.
 
     When a new alias (hard link) is created inside an HDF5 file the parent
     groups may not carry the ``datatype`` attribute expected by the LGDO reader.
@@ -15,38 +15,48 @@ def convert_parents_to_structs(h5group):
     every ancestor group carries a ``datatype`` attribute of the form
     ``struct{child1,child2,…}``.
 
+    The root group is only updated if it already carries a ``datatype``; the
+    walk always stops there.
+
     Parameters
     ----------
     h5group : h5py.Group
         Leaf group whose parent hierarchy should be annotated.
     """
-    child = h5group.name.split("/")[-1]
-    if h5group.parent.name != "/" and len(h5group.parent.attrs) == 0:
-        h5group.parent.attrs.update({"datatype": "struct{" + child + "}"})
-    elif len(h5group.parent.attrs) > 0:
-        datatype = h5group.parent.attrs["datatype"]
-        if not (datatype.startswith("struct{") and datatype.endswith("}")):
-            msg = (
-                f"cannot register alias {child!r}: parent group "
-                f"{h5group.parent.name!r} has non-struct datatype {datatype!r}"
+    # the walk must be iterative: in h5py the root group is its own parent, so
+    # recursing on ``h5group.parent`` never terminates once ``/`` is reached
+    while h5group.name != "/":
+        parent = h5group.parent
+        child = h5group.name.rsplit("/", 1)[-1]
+
+        if "datatype" not in parent.attrs:
+            # never invent a datatype on the file root
+            if parent.name == "/":
+                return
+            parent.attrs.update({"datatype": "struct{" + child + "}"})
+        else:
+            datatype = parent.attrs["datatype"]
+            if not (datatype.startswith("struct{") and datatype.endswith("}")):
+                msg = (
+                    f"cannot register alias {child!r}: parent group "
+                    f"{parent.name!r} has non-struct datatype {datatype!r}"
+                )
+                raise ValueError(msg)
+            # membership must be tested against the parsed field list, not by
+            # substring (e.g. 'raw' is a substring of 'raw_blind')
+            fields = [
+                field.strip()
+                for field in datatype[len("struct{") : -1].split(",")
+                if field.strip()
+            ]
+            if child in fields:
+                # ancestors were annotated when this member was first added
+                return
+            parent.attrs.update(
+                {"datatype": "struct{" + ",".join([*fields, child]) + "}"}
             )
-            raise ValueError(msg)
-        # membership must be tested against the parsed field list, not by
-        # substring (e.g. 'raw' is a substring of 'raw_blind')
-        fields = [
-            field.strip()
-            for field in datatype[len("struct{") : -1].split(",")
-            if field.strip()
-        ]
-        if child in fields:
-            return
-        h5group.parent.attrs.update(
-            {"datatype": "struct{" + ",".join([*fields, child]) + "}"}
-        )
-    else:
-        return
-    convert_parents_to_structs(h5group.parent)
-    return
+
+        h5group = parent
 
 
 def alias_table(file: str | Path, mapping: str):

legend_dataflow_scripts-0.4.0a2/src/legenddataflowscripts/workflow/pre_compile_catalog.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import pickle
+import tempfile
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path
+
+from dbetto import TextDB
+from dbetto.catalog import Catalog
+
+
+def _content_digest(validity_path: Path) -> str:
+    """Digest of the directory contents (relative path, mtime and size of
+    every file) plus the dbetto version, identifying one compiled state of
+    the catalog."""
+    state = [version("dbetto")]
+    for f in sorted(validity_path.rglob("*")):
+        if f.is_file():
+            st = f.stat()
+            state.append(
+                f"{f.relative_to(validity_path)}:{st.st_mtime_ns}:{st.st_size}"
+            )
+    return hashlib.sha256("\n".join(state).encode()).hexdigest()[:16]
+
+
+def _cache_dir(validity_path: Path) -> Path:
+    """Per-directory cache namespace, keyed on the resolved directory path, so
+    same-named directories in different locations cannot evict each other's
+    pickles."""
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    dir_key = hashlib.sha256(str(validity_path.resolve()).encode()).hexdigest()[:16]
+    return (
+        Path(base)
+        / "legend-dataflow"
+        / "precompiled-catalogs"
+        / f"{validity_path.name}-{dir_key}"
+    )
+
+
+def pre_compile_catalog(validity_path: str | Path):
+    """Pre-compile a dbetto validity catalog for fast repeated access.
+
+    Reads the ``validity.yaml`` catalog from *validity_path* and, for each
+    system and each entry in the catalog, eagerly loads the corresponding
+    :class:`dbetto.TextDB` state (instead of loading it lazily on first
+    access).  The resulting :class:`dbetto.catalog.Catalog` can be serialised
+    and reused across many Snakemake jobs without re-parsing YAML on every
+    invocation.
+
+    The compiled catalog is cached on disk under
+    ``$XDG_CACHE_HOME/legend-dataflow/precompiled-catalogs``, in a
+    subdirectory per (resolved) input directory, keyed on the directory
+    contents (file mtimes/sizes) and the dbetto version, so repeated
+    workflow parses skip the eager compilation entirely.
+
+    Parameters
+    ----------
+    validity_path : str or pathlib.Path
+        Directory containing the ``validity.yaml`` file and all referenced
+        database files.
+
+    Returns
+    -------
+    dbetto.catalog.Catalog
+        Pre-compiled catalog with all entries eagerly resolved.
+    """
+    if isinstance(validity_path, str):
+        validity_path = Path(validity_path)
+
+    cache_dir = _cache_dir(validity_path)
+    cache_file = cache_dir / f"{_content_digest(validity_path)}.pkl"
+    if cache_file.is_file() and not cache_file.is_symlink():
+        try:
+            with cache_file.open("rb") as f:
+                cached = pickle.load(f)
+            # tampered/corrupt caches must never be handed back as a catalog
+            if isinstance(cached, Catalog):
+                return cached
+        except Exception:  # unreadable pickle: rebuild below
+            pass
+
+    catalog = Catalog.read_from(validity_path / "validity.yaml")
+    entries = {}
+    textdb = TextDB(validity_path, lazy=False)
+    for system in catalog.entries:
+        entries[system] = []
+        for entry in catalog.entries[system]:
+            db = textdb.on(
+                datetime.fromtimestamp(entry.valid_from, tz=UTC), category=system
+            )
+            new_entry = Catalog.Entry(entry.valid_from, db)
+            entries[system].append(new_entry)
+    compiled = Catalog(entries)
+
+    try:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        for stale in cache_dir.glob("*.pkl"):
+            stale.unlink(missing_ok=True)
+        fd, tmp_name = tempfile.mkstemp(dir=cache_dir, suffix=".tmp")
+        try:
+            with os.fdopen(fd, "wb") as f:
+                pickle.dump(compiled, f, protocol=pickle.HIGHEST_PROTOCOL)
+            Path(tmp_name).replace(cache_file)
+        except BaseException:
+            Path(tmp_name).unlink(missing_ok=True)
+            raise
+    except Exception:
+        pass  # caching is best-effort; a failed write must not break parsing
+
+    return compiled

legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.4.0a1",
-  "distance": 0,
-  "node": "g30429c60446a5d3e19023e462981054d0cda7d7d",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-16"
-}

legend_dataflow_scripts-0.4.0a1/src/legenddataflowscripts/workflow/pre_compile_catalog.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from pathlib import Path
-
-from dbetto import TextDB
-from dbetto.catalog import Catalog
-
-
-def pre_compile_catalog(validity_path: str | Path):
-    """Pre-compile a dbetto validity catalog for fast repeated access.
-
-    Reads the ``validity.yaml`` catalog from *validity_path* and, for each
-    system and each entry in the catalog, eagerly loads the corresponding
-    :class:`dbetto.TextDB` state (instead of loading it lazily on first
-    access).  The resulting :class:`dbetto.catalog.Catalog` can be serialised
-    and reused across many Snakemake jobs without re-parsing YAML on every
-    invocation.
-
-    Parameters
-    ----------
-    validity_path : str or pathlib.Path
-        Directory containing the ``validity.yaml`` file and all referenced
-        database files.
-
-    Returns
-    -------
-    dbetto.catalog.Catalog
-        Pre-compiled catalog with all entries eagerly resolved.
-    """
-    if isinstance(validity_path, str):
-        validity_path = Path(validity_path)
-    catalog = Catalog.read_from(validity_path / "validity.yaml")
-    entries = {}
-    textdb = TextDB(validity_path, lazy=False)
-    for system in catalog.entries:
-        entries[system] = []
-        for entry in catalog.entries[system]:
-            db = textdb.on(
-                datetime.fromtimestamp(entry.valid_from, tz=UTC), system=system
-            )
-            new_entry = Catalog.Entry(entry.valid_from, db)
-            entries[system].append(new_entry)
-    return Catalog(entries)