legend-dataflow-scripts 0.4.0a1__tar.gz → 0.4.0a3__tar.gz

{legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.4.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.4.0a1
+Version: 0.4.0a3
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.4.0a1
+Version: 0.4.0a3
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legend_dataflow_scripts.egg-info/scm_file_list.json

@@ -60,6 +60,7 @@
     "tests/test_execenv.py",
     "tests/test_lgdo_utils.py",
     "tests/test_peak_selection.py",
+    "tests/test_pre_compile_catalog.py",
     "tests/test_util.py",
     "tests/test_utils_helpers.py"
   ]

legend_dataflow_scripts-0.4.0a3/src/legend_dataflow_scripts.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.4.0a3",
+  "distance": 0,
+  "node": "gefba089d925c92cfa07af5e93737d253ec830fa5",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-10-16"
+}

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.4.0a1'
-__version_tuple__ = version_tuple = (0, 4, 0, 'a1')
+__version__ = version = '0.4.0a3'
+__version_tuple__ = version_tuple = (0, 4, 0, 'a3')
 
-__commit_id__ = commit_id = 'g30429c604'
+__commit_id__ = commit_id = 'gefba089d9'

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/dsp/dplms.py

@@ -6,6 +6,7 @@ import time
 from pathlib import Path
 
 import lh5
+import matplotlib as mpl
 import numpy as np
 import pygama.math.distributions as pmd  # noqa: F401
 from dbetto.catalog import Props
@@ -25,6 +26,8 @@ from ....utils import (
     take_table_rows,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 
 def par_geds_dsp_dplms() -> None:
     """Compute DPLMS optimal filter coefficients for HPGe detectors.

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/dsp/eopt.py

@@ -7,6 +7,7 @@ import warnings
 from pathlib import Path
 
 import lh5
+import matplotlib as mpl
 import numpy as np
 import pygama.pargen.energy_optimisation as om  # noqa: F401
 import sklearn.gaussian_process.kernels as ker
@@ -27,6 +28,8 @@ from ....utils import (
     require_peaks_present,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 try:
     warnings.filterwarnings(action="ignore", category=np.exceptions.RankWarning)

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/dsp/evtsel.py

@@ -133,6 +133,34 @@ def get_out_data(
     return out_tbl, int(np.count_nonzero(final_mask))
 
 
+def initial_bin_width(energy):
+    """
+    Freedman-Diaconis starting bin width for the peak-finding histogram.
+
+    Parameters
+    ----------
+    energy : numpy.ndarray
+        uncalibrated energies of the candidate events.
+
+    Returns
+    -------
+    float or None
+        bin width capped at 2 ADC, or ``None`` when the sample is degenerate
+        (fewer than two finite values, or no spread) and no histogram can be built.
+    """
+    energy = energy[np.isfinite(energy)]
+    if len(energy) < 2:
+        return None
+    width = (
+        2
+        * (np.percentile(energy, 75) - np.percentile(energy, 25))
+        * len(energy) ** (-1 / 3)
+    )
+    if not width > 0:
+        return None
+    return min(width, 2)
+
+
 def build_peak_dicts(peaks_kev, kev_widths, masks, log=None):
     """Build the per-peak accumulator state for the file x peak read loop.
 
@@ -455,16 +483,17 @@ def par_geds_dsp_evtsel() -> None:
                             )
                             energy = tb_out[energy_parameter].nda
 
-                            init_bin_width = (
-                                2
-                                * (
-                                    np.nanpercentile(energy, 75)
-                                    - np.nanpercentile(energy, 25)
+                            init_bin_width = initial_bin_width(energy)
+                            if init_bin_width is None:  # too few events to histogram
+                                msg = (
+                                    f"{peak}: only {np.isfinite(energy).sum()} "
+                                    "usable events, skipping peak"
                                 )
-                                * len(energy) ** (-1 / 3)
-                            )
-
-                            init_bin_width = min(init_bin_width, 2)
+                                log.warning(msg)
+                                peak_dict["obj_buf"] = None
+                                peak_dict["obj_buf_start"] = 0
+                                peak_dict["idxs"] = None
+                                continue
 
                             hist, bins, var = pgh.get_hist(
                                 energy,
@@ -487,7 +516,7 @@ def par_geds_dsp_evtsel() -> None:
                                 mu = peak_top_pars[0]
                                 if mu > np.nanmax(bins) or mu < np.nanmin(bins):
                                     raise ValueError
-                            except Exception:
+                            except (ValueError, TypeError, IndexError):
                                 mu = np.nan
                             if mu is None or np.isnan(mu):
                                 log.debug("Fit failed, using max guess")

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/dsp/nopt.py

@@ -6,6 +6,7 @@ import time
 from pathlib import Path
 
 import lh5
+import matplotlib as mpl
 import numpy as np
 import pygama.pargen.noise_optimization as pno
 from dbetto.catalog import Props
@@ -20,6 +21,8 @@ from ....utils import (
     require_config_keys,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 
 def par_geds_dsp_nopt() -> None:
     """Optimise DSP filter parameters for noise performance.

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/dsp/pz.py

@@ -6,6 +6,7 @@ import pickle as pkl
 from pathlib import Path
 
 import lh5
+import matplotlib as mpl
 import numpy as np
 from dbetto.catalog import Props
 from dspeed import build_dsp
@@ -24,6 +25,8 @@ from ....utils import (
     take_table_rows,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 
 def par_geds_dsp_pz() -> None:
     """Determine the pole-zero (PZ) decay constant(s) for HPGe waveforms.

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/hit/aoe.py

@@ -10,9 +10,10 @@ import time
 import warnings
 from pathlib import Path
 
+import matplotlib as mpl
 import numpy as np
 from dbetto.catalog import Props
-from pygama.pargen.AoE_cal import *  # noqa: F403
+from pygama.pargen.AoE_cal import *
 from pygama.pargen.AoE_cal import CalAoE
 from pygama.pargen.utils import load_data
 
@@ -28,6 +29,8 @@ from ....utils import (
     require_unique_suffixes,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 log = logging.getLogger(__name__)
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/hit/ecal.py

@@ -5,7 +5,7 @@ import copy
 import inspect
 import pickle as pkl
 import warnings
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 
 import lh5
@@ -104,14 +104,14 @@ def plot_2614_timemap(
 
     ticks, _ = plt.xticks()
     plt.xlabel(
-        f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
+        f"Time starting : {datetime.fromtimestamp(ticks[0], tz=UTC).strftime('%d/%m/%y %H:%M')}"
     )
     plt.ylabel("Energy(keV)")
     plt.ylim([erange[0], erange[1]])
 
     plt.xticks(
         ticks,
-        [datetime.utcfromtimestamp(tick).strftime("%H:%M") for tick in ticks],
+        [datetime.fromtimestamp(tick, tz=UTC).strftime("%H:%M") for tick in ticks],
     )
     plt.close()
     return fig
@@ -188,13 +188,13 @@ def plot_pulser_timemap(
         plt.ylim([mean - n_spread * spread, mean + n_spread * spread])
     ticks, _ = plt.xticks()
     plt.xlabel(
-        f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
+        f"Time starting : {datetime.fromtimestamp(ticks[0], tz=UTC).strftime('%d/%m/%y %H:%M')}"
     )
     plt.ylabel("Energy(keV)")
 
     plt.xticks(
         ticks,
-        [datetime.utcfromtimestamp(tick).strftime("%H:%M") for tick in ticks],
+        [datetime.fromtimestamp(tick, tz=UTC).strftime("%H:%M") for tick in ticks],
     )
     plt.close()
     return fig
@@ -488,14 +488,14 @@ def plot_baseline_timemap(
 
     ticks, _ = plt.xticks()
     plt.xlabel(
-        f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
+        f"Time starting : {datetime.fromtimestamp(ticks[0], tz=UTC).strftime('%d/%m/%y %H:%M')}"
     )
     plt.ylabel("Baseline Value")
     plt.ylim([mean - n_spread * spread, mean + n_spread * spread])
 
     plt.xticks(
         ticks,
-        [datetime.utcfromtimestamp(tick).strftime("%H:%M") for tick in ticks],
+        [datetime.fromtimestamp(tick, tz=UTC).strftime("%H:%M") for tick in ticks],
     )
     plt.close()
     return fig
@@ -683,7 +683,7 @@ def get_results_dict(ecal_class, data, cal_energy_param, selection_string):
 
     pk_dict = results_dict["peak_parameters"]
 
-    for _, dic in pk_dict.items():
+    for dic in pk_dict.values():
         dic["function"] = dic["function"].name
         dic["parameters"] = dic["parameters"].to_dict()
         dic["uncertainties"] = dic["uncertainties"].to_dict()

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/hit/lq.py

@@ -9,12 +9,13 @@ import time
 import warnings
 from pathlib import Path
 
+import matplotlib as mpl
 import numpy as np
 import pandas as pd
 from dbetto.catalog import Props
 from pygama.math.distributions import gaussian
-from pygama.pargen.AoE_cal import *  # noqa: F403
-from pygama.pargen.lq_cal import *  # noqa: F403
+from pygama.pargen.AoE_cal import *
+from pygama.pargen.lq_cal import *
 from pygama.pargen.lq_cal import LQCal
 from pygama.pargen.utils import load_data
 
@@ -30,6 +31,8 @@ from ....utils import (
     require_unique_suffixes,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 log = logging.getLogger(__name__)
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/par/geds/hit/qc.py

@@ -9,6 +9,7 @@ import time
 import warnings
 from pathlib import Path
 
+import matplotlib as mpl
 import numpy as np
 from dbetto.catalog import Props
 from lh5 import ls
@@ -28,6 +29,8 @@ from ....utils import (
     prepare_output_paths,
 )
 
+mpl.use("agg")  # no display needed; avoids X11 when DISPLAY is set
+
 log = logging.getLogger(__name__)
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/tier/dsp.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 import argparse
 import time
 import warnings
-from multiprocessing import Pool
+from concurrent.futures import ProcessPoolExecutor
+from multiprocessing import get_context
 from pathlib import Path
 
 import lh5
@@ -51,7 +52,7 @@ def build_dsp_wrapper(kwargs):
     """Unpack *kwargs* and delegate to :func:`dspeed.build_dsp`.
 
     This thin wrapper exists so that a single-argument callable can be passed
-    to :meth:`multiprocessing.Pool.map`.
+    to :meth:`concurrent.futures.Executor.map`.
 
     Parameters
     ----------
@@ -239,10 +240,12 @@ def build_tier_dsp() -> None:
             }
             process_kwargs_list.append(kwargs)
 
-        # Create a multiprocessing pool
-        with Pool(processes=args.n_processes) as pool:
-            # Use starmap to pass multiple arguments to the process function
-            pool.map(build_dsp_wrapper, process_kwargs_list)
+        # fork pinned: default start method changes in py3.14
+        # a killed worker raises BrokenProcessPool here; Pool.map would hang forever
+        with ProcessPoolExecutor(
+            max_workers=args.n_processes, mp_context=get_context("fork")
+        ) as pool:
+            list(pool.map(build_dsp_wrapper, process_kwargs_list))
 
         # merge the DSPs
         log.info("Merging DSPs")

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/utils/alias_table.py

@@ -7,7 +7,7 @@ import h5py
 
 
 def convert_parents_to_structs(h5group):
-    """Recursively annotate HDF5 parent groups with LGDO ``struct`` datatype attributes.
+    """Annotate HDF5 parent groups with LGDO ``struct`` datatype attributes.
 
     When a new alias (hard link) is created inside an HDF5 file the parent
     groups may not carry the ``datatype`` attribute expected by the LGDO reader.
@@ -15,38 +15,48 @@ def convert_parents_to_structs(h5group):
     every ancestor group carries a ``datatype`` attribute of the form
     ``struct{child1,child2,…}``.
 
+    The root group is only updated if it already carries a ``datatype``; the
+    walk always stops there.
+
     Parameters
     ----------
     h5group : h5py.Group
         Leaf group whose parent hierarchy should be annotated.
     """
-    child = h5group.name.split("/")[-1]
-    if h5group.parent.name != "/" and len(h5group.parent.attrs) == 0:
-        h5group.parent.attrs.update({"datatype": "struct{" + child + "}"})
-    elif len(h5group.parent.attrs) > 0:
-        datatype = h5group.parent.attrs["datatype"]
-        if not (datatype.startswith("struct{") and datatype.endswith("}")):
-            msg = (
-                f"cannot register alias {child!r}: parent group "
-                f"{h5group.parent.name!r} has non-struct datatype {datatype!r}"
+    # the walk must be iterative: in h5py the root group is its own parent, so
+    # recursing on ``h5group.parent`` never terminates once ``/`` is reached
+    while h5group.name != "/":
+        parent = h5group.parent
+        child = h5group.name.rsplit("/", 1)[-1]
+
+        if "datatype" not in parent.attrs:
+            # never invent a datatype on the file root
+            if parent.name == "/":
+                return
+            parent.attrs.update({"datatype": "struct{" + child + "}"})
+        else:
+            datatype = parent.attrs["datatype"]
+            if not (datatype.startswith("struct{") and datatype.endswith("}")):
+                msg = (
+                    f"cannot register alias {child!r}: parent group "
+                    f"{parent.name!r} has non-struct datatype {datatype!r}"
+                )
+                raise ValueError(msg)
+            # membership must be tested against the parsed field list, not by
+            # substring (e.g. 'raw' is a substring of 'raw_blind')
+            fields = [
+                field.strip()
+                for field in datatype[len("struct{") : -1].split(",")
+                if field.strip()
+            ]
+            if child in fields:
+                # ancestors were annotated when this member was first added
+                return
+            parent.attrs.update(
+                {"datatype": "struct{" + ",".join([*fields, child]) + "}"}
             )
-            raise ValueError(msg)
-        # membership must be tested against the parsed field list, not by
-        # substring (e.g. 'raw' is a substring of 'raw_blind')
-        fields = [
-            field.strip()
-            for field in datatype[len("struct{") : -1].split(",")
-            if field.strip()
-        ]
-        if child in fields:
-            return
-        h5group.parent.attrs.update(
-            {"datatype": "struct{" + ",".join([*fields, child]) + "}"}
-        )
-    else:
-        return
-    convert_parents_to_structs(h5group.parent)
-    return
+
+        h5group = parent
 
 
 def alias_table(file: str | Path, mapping: str):

{legend_dataflow_scripts-0.4.0a1 → legend_dataflow_scripts-0.4.0a3}/src/legenddataflowscripts/utils/cfgtools.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 from collections.abc import Iterable, Mapping
 from pathlib import Path
 
@@ -133,14 +134,15 @@ def get_rule_config(configs_path, rule_name, timestamp, datatype):
 
     Wraps ``TextDB(...).on(...)["snakemake_rules"][rule_name]`` so that a
     missing key names the rule, timestamp, datatype and config path instead
-    of raising a bare :class:`KeyError`.
+    of raising a bare :class:`KeyError`. Returns a writable deep copy, since
+    ``TextDB.on()`` results are shared and read-only.
     """
     if not Path(configs_path).is_dir():
         msg = f"config directory {configs_path} does not exist"
         raise FileNotFoundError(msg)
     configs = TextDB(configs_path, lazy=True).on(timestamp, category=datatype)
     try:
-        return configs["snakemake_rules"][rule_name]
+        return copy.deepcopy(configs["snakemake_rules"][rule_name])
     except KeyError as err:
         msg = (
             f"config resolved from {configs_path} for timestamp {timestamp} "

legend_dataflow_scripts-0.4.0a3/src/legenddataflowscripts/workflow/pre_compile_catalog.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import pickle
+import tempfile
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path
+
+from dbetto import TextDB
+from dbetto.catalog import Catalog
+
+log = logging.getLogger(__name__)
+
+
+def _content_digest(validity_path: Path) -> str:
+    """Digest of the directory contents (relative path, mtime and size of
+    every file) plus the dbetto version, identifying one compiled state of
+    the catalog."""
+    state = [version("dbetto")]
+    for f in sorted(validity_path.rglob("*")):
+        if f.is_file():
+            st = f.stat()
+            state.append(
+                f"{f.relative_to(validity_path)}:{st.st_mtime_ns}:{st.st_size}"
+            )
+    return hashlib.sha256("\n".join(state).encode()).hexdigest()[:16]
+
+
+def _cache_dir(validity_path: Path) -> Path:
+    """Per-directory cache namespace, keyed on the resolved directory path, so
+    same-named directories in different locations cannot evict each other's
+    pickles."""
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    dir_key = hashlib.sha256(str(validity_path.resolve()).encode()).hexdigest()[:16]
+    return (
+        Path(base)
+        / "legend-dataflow"
+        / "precompiled-catalogs"
+        / f"{validity_path.name}-{dir_key}"
+    )
+
+
+def pre_compile_catalog(validity_path: str | Path):
+    """Pre-compile a dbetto validity catalog for fast repeated access.
+
+    Reads the ``validity.yaml`` catalog from *validity_path* and, for each
+    system and each entry in the catalog, eagerly loads the corresponding
+    :class:`dbetto.TextDB` state (instead of loading it lazily on first
+    access).  The resulting :class:`dbetto.catalog.Catalog` can be serialised
+    and reused across many Snakemake jobs without re-parsing YAML on every
+    invocation.
+
+    The compiled catalog is cached on disk under
+    ``$XDG_CACHE_HOME/legend-dataflow/precompiled-catalogs``, in a
+    subdirectory per (resolved) input directory, keyed on the directory
+    contents (file mtimes/sizes) and the dbetto version, so repeated
+    workflow parses skip the eager compilation entirely.
+
+    Parameters
+    ----------
+    validity_path : str or pathlib.Path
+        Directory containing the ``validity.yaml`` file and all referenced
+        database files.
+
+    Returns
+    -------
+    dbetto.catalog.Catalog
+        Pre-compiled catalog with all entries eagerly resolved.
+    """
+    if isinstance(validity_path, str):
+        validity_path = Path(validity_path)
+
+    cache_dir = _cache_dir(validity_path)
+    cache_file = cache_dir / f"{_content_digest(validity_path)}.pkl"
+    if cache_file.is_file() and not cache_file.is_symlink():
+        try:
+            with cache_file.open("rb") as f:
+                cached = pickle.load(f)
+            # tampered/corrupt caches must never be handed back as a catalog
+            if isinstance(cached, Catalog):
+                return cached
+        except Exception:  # unreadable pickle: rebuild below
+            log.debug("could not read catalog cache %s", cache_file, exc_info=True)
+
+    catalog = Catalog.read_from(validity_path / "validity.yaml")
+    entries = {}
+    textdb = TextDB(validity_path, lazy=False)
+    for system in catalog.entries:
+        entries[system] = []
+        for entry in catalog.entries[system]:
+            db = textdb.on(
+                datetime.fromtimestamp(entry.valid_from, tz=UTC), category=system
+            )
+            new_entry = Catalog.Entry(entry.valid_from, db)
+            entries[system].append(new_entry)
+    compiled = Catalog(entries)
+
+    try:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        for stale in cache_dir.glob("*.pkl"):
+            stale.unlink(missing_ok=True)
+        fd, tmp_name = tempfile.mkstemp(dir=cache_dir, suffix=".tmp")
+        try:
+            with os.fdopen(fd, "wb") as f:
+                pickle.dump(compiled, f, protocol=pickle.HIGHEST_PROTOCOL)
+            Path(tmp_name).replace(cache_file)
+        except BaseException:
+            Path(tmp_name).unlink(missing_ok=True)
+            raise
+    except Exception:
+        # caching is best-effort; a failed write must not break parsing
+        log.debug("could not write catalog cache %s", cache_file, exc_info=True)
+
+    return compiled

legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.4.0a1",
-  "distance": 0,
-  "node": "g30429c60446a5d3e19023e462981054d0cda7d7d",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-16"
-}

legend_dataflow_scripts-0.4.0a1/src/legenddataflowscripts/workflow/pre_compile_catalog.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from pathlib import Path
-
-from dbetto import TextDB
-from dbetto.catalog import Catalog
-
-
-def pre_compile_catalog(validity_path: str | Path):
-    """Pre-compile a dbetto validity catalog for fast repeated access.
-
-    Reads the ``validity.yaml`` catalog from *validity_path* and, for each
-    system and each entry in the catalog, eagerly loads the corresponding
-    :class:`dbetto.TextDB` state (instead of loading it lazily on first
-    access).  The resulting :class:`dbetto.catalog.Catalog` can be serialised
-    and reused across many Snakemake jobs without re-parsing YAML on every
-    invocation.
-
-    Parameters
-    ----------
-    validity_path : str or pathlib.Path
-        Directory containing the ``validity.yaml`` file and all referenced
-        database files.
-
-    Returns
-    -------
-    dbetto.catalog.Catalog
-        Pre-compiled catalog with all entries eagerly resolved.
-    """
-    if isinstance(validity_path, str):
-        validity_path = Path(validity_path)
-    catalog = Catalog.read_from(validity_path / "validity.yaml")
-    entries = {}
-    textdb = TextDB(validity_path, lazy=False)
-    for system in catalog.entries:
-        entries[system] = []
-        for entry in catalog.entries[system]:
-            db = textdb.on(
-                datetime.fromtimestamp(entry.valid_from, tz=UTC), system=system
-            )
-            new_entry = Catalog.Entry(entry.valid_from, db)
-            entries[system].append(new_entry)
-    return Catalog(entries)