legend-dataflow-scripts 0.3.2__tar.gz → 0.4.0a1__tar.gz

{legend_dataflow_scripts-0.3.2/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.4.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.3.2
+Version: 0.4.0a1
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -18,8 +18,8 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: colorlog
-Requires-Dist: dbetto>=1.3.6
-Requires-Dist: pygama>=2.3.10
+Requires-Dist: dbetto>=1.4
+Requires-Dist: pygama>=2.7.0a1
 Requires-Dist: dspeed>=2.1.4
 Requires-Dist: pylegendmeta>=1.4.4
 Requires-Dist: legend-pydataobj>=2.0.2

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/pyproject.toml

@@ -51,8 +51,8 @@ dynamic = ["version"]
 
 dependencies = [
     "colorlog",
-    "dbetto>=1.3.6",
-    "pygama>=2.3.10",
+    "dbetto>=1.4",
+    "pygama>=2.7.0a1",
     "dspeed>=2.1.4",
     "pylegendmeta>=1.4.4",
     "legend-pydataobj>=2.0.2",
@@ -162,7 +162,7 @@ isort.required-imports = ["from __future__ import annotations"]
 
 
 [tool.pylint]
-py-version = "3.9"
+py-version = "3.11"
 ignore-paths = [".*/_version.py"]
 reports.output-format = "colorized"
 similarities.ignore-imports = "yes"

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.3.2
+Version: 0.4.0a1
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -18,8 +18,8 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: colorlog
-Requires-Dist: dbetto>=1.3.6
-Requires-Dist: pygama>=2.3.10
+Requires-Dist: dbetto>=1.4
+Requires-Dist: pygama>=2.7.0a1
 Requires-Dist: dspeed>=2.1.4
 Requires-Dist: pylegendmeta>=1.4.4
 Requires-Dist: legend-pydataobj>=2.0.2

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/src/legend_dataflow_scripts.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ src/legend_dataflow_scripts.egg-info/dependency_links.txt
 src/legend_dataflow_scripts.egg-info/entry_points.txt
 src/legend_dataflow_scripts.egg-info/not-zip-safe
 src/legend_dataflow_scripts.egg-info/requires.txt
+src/legend_dataflow_scripts.egg-info/scm_file_list.json
+src/legend_dataflow_scripts.egg-info/scm_version.json
 src/legend_dataflow_scripts.egg-info/top_level.txt
 src/legenddataflowscripts/__init__.py
 src/legenddataflowscripts/_version.py
@@ -32,6 +34,9 @@ src/legenddataflowscripts/utils/__init__.py
 src/legenddataflowscripts/utils/alias_table.py
 src/legenddataflowscripts/utils/cfgtools.py
 src/legenddataflowscripts/utils/convert_np.py
+src/legenddataflowscripts/utils/discharge.py
+src/legenddataflowscripts/utils/files.py
+src/legenddataflowscripts/utils/lgdo_utils.py
 src/legenddataflowscripts/utils/log.py
 src/legenddataflowscripts/utils/plot_dict.py
 src/legenddataflowscripts/utils/pulser_removal.py

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/src/legend_dataflow_scripts.egg-info/requires.txt

@@ -1,6 +1,6 @@
 colorlog
-dbetto>=1.3.6
-pygama>=2.3.10
+dbetto>=1.4
+pygama>=2.7.0a1
 dspeed>=2.1.4
 pylegendmeta>=1.4.4
 legend-pydataobj>=2.0.2

legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info/scm_file_list.json

@@ -0,0 +1,66 @@
+{
+  "files": [
+    ".github/dependabot.yml",
+    ".github/release.yml",
+    ".github/workflows/cd.yml",
+    ".github/workflows/main.yml",
+    ".gitignore",
+    ".pre-commit-config.yaml",
+    ".readthedocs.yaml",
+    "AI_POLICY.md",
+    "LICENSE",
+    "MANIFEST.in",
+    "README.md",
+    "docs/Makefile",
+    "docs/source/callables.rst",
+    "docs/source/conf.py",
+    "docs/source/developer.rst",
+    "docs/source/index.rst",
+    "docs/source/overview.rst",
+    "pyproject.toml",
+    "src/legenddataflowscripts/__init__.py",
+    "src/legenddataflowscripts/par/__init__.py",
+    "src/legenddataflowscripts/par/geds/__init__.py",
+    "src/legenddataflowscripts/par/geds/dsp/__init__.py",
+    "src/legenddataflowscripts/par/geds/dsp/dplms.py",
+    "src/legenddataflowscripts/par/geds/dsp/eopt.py",
+    "src/legenddataflowscripts/par/geds/dsp/evtsel.py",
+    "src/legenddataflowscripts/par/geds/dsp/nopt.py",
+    "src/legenddataflowscripts/par/geds/dsp/pz.py",
+    "src/legenddataflowscripts/par/geds/dsp/svm.py",
+    "src/legenddataflowscripts/par/geds/dsp/svm_build.py",
+    "src/legenddataflowscripts/par/geds/hit/__init__.py",
+    "src/legenddataflowscripts/par/geds/hit/aoe.py",
+    "src/legenddataflowscripts/par/geds/hit/ecal.py",
+    "src/legenddataflowscripts/par/geds/hit/lq.py",
+    "src/legenddataflowscripts/par/geds/hit/qc.py",
+    "src/legenddataflowscripts/tier/__init__.py",
+    "src/legenddataflowscripts/tier/dsp.py",
+    "src/legenddataflowscripts/tier/hit.py",
+    "src/legenddataflowscripts/utils/__init__.py",
+    "src/legenddataflowscripts/utils/alias_table.py",
+    "src/legenddataflowscripts/utils/cfgtools.py",
+    "src/legenddataflowscripts/utils/convert_np.py",
+    "src/legenddataflowscripts/utils/discharge.py",
+    "src/legenddataflowscripts/utils/files.py",
+    "src/legenddataflowscripts/utils/lgdo_utils.py",
+    "src/legenddataflowscripts/utils/log.py",
+    "src/legenddataflowscripts/utils/plot_dict.py",
+    "src/legenddataflowscripts/utils/pulser_removal.py",
+    "src/legenddataflowscripts/workflow/__init__.py",
+    "src/legenddataflowscripts/workflow/execenv.py",
+    "src/legenddataflowscripts/workflow/filedb.py",
+    "src/legenddataflowscripts/workflow/pre_compile_catalog.py",
+    "src/legenddataflowscripts/workflow/utils.py",
+    "tests/dataflow-config.yaml",
+    "tests/test_build_tier_hit.py",
+    "tests/test_console_scripts.py",
+    "tests/test_convert_py.py",
+    "tests/test_discharge.py",
+    "tests/test_execenv.py",
+    "tests/test_lgdo_utils.py",
+    "tests/test_peak_selection.py",
+    "tests/test_util.py",
+    "tests/test_utils_helpers.py"
+  ]
+}

legend_dataflow_scripts-0.4.0a1/src/legend_dataflow_scripts.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.4.0a1",
+  "distance": 0,
+  "node": "g30429c60446a5d3e19023e462981054d0cda7d7d",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-08-16"
+}

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/src/legenddataflowscripts/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.3.2'
-__version_tuple__ = version_tuple = (0, 3, 2)
+__version__ = version = '0.4.0a1'
+__version_tuple__ = version_tuple = (0, 4, 0, 'a1')
 
-__commit_id__ = commit_id = 'g30a72af68'
+__commit_id__ = commit_id = 'g30429c604'

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/src/legenddataflowscripts/par/geds/dsp/dplms.py

@@ -14,7 +14,16 @@ from pygama.pargen.data_cleaning import generate_cuts
 from pygama.pargen.dplms_ge_dict import dplms_ge_dict
 from pygama.pargen.dsp_optimize import run_one_dsp
 
-from ....utils import build_log, convert_dict_np_to_float
+from ....utils import (
+    build_log,
+    check_input_files,
+    convert_dict_np_to_float,
+    expand_filelist,
+    prepare_output_paths,
+    require_config_keys,
+    require_peaks_present,
+    take_table_rows,
+)
 
 
 def par_geds_dsp_dplms() -> None:
@@ -55,7 +64,13 @@ def par_geds_dsp_dplms() -> None:
         Processing chain configuration file(s).
     ``--config-file`` : list of str
         DPLMS configuration file(s).  Must contain ``run_dplms`` (bool),
-        ``n_baselines`` (int), and ``peaks_kev`` (list).
+        ``n_baselines`` (int), and ``peaks_kev`` (list).  The optional key
+        ``use_log_pdf`` (bool, default false) is forwarded to
+        :func:`pygama.pargen.dplms_ge_dict.dplms_ge_dict` and builds the
+        grid-search FOM fits from the model's log-density (iminuit
+        ``log=True``) — substantially faster, results differ at
+        machine-precision level; requires a pygama version with
+        ``use_log_pdf`` support.
     ``--channel`` : str
         Channel identifier; used as the HDF5 group name in *lh5-path*.
     ``--raw-table-name`` : str
@@ -100,17 +115,22 @@ def par_geds_dsp_dplms() -> None:
 
     args = argparser.parse_args()
 
-    dsp_config = Props.read_from(args.processing_chain)
     log = build_log(args.log_config, args.log)
 
+    check_input_files(args.peak_file, "--peak-file")
+    check_input_files(args.inplots, "--inplots")
+    prepare_output_paths(args.lh5_path, args.dsp_pars, args.plot_path)
+
+    dsp_config = Props.read_from(args.processing_chain)
+
     t0 = time.time()
 
     dplms_dict = Props.read_from(args.config_file)
     db_dict = Props.read_from(args.database)
+    require_config_keys(dplms_dict, ["run_dplms"], f"dplms config ({args.config_file})")
 
     if dplms_dict["run_dplms"] is True:
-        with Path(args.fft_raw_filelist).open() as f:
-            fft_files = sorted(f.read().splitlines())
+        fft_files = expand_filelist([args.fft_raw_filelist], "--fft-raw-filelist")
 
         t0 = time.time()
         log.info("\nLoad fft data")
@@ -130,17 +150,20 @@ def par_geds_dsp_dplms() -> None:
 
         log.info("\nRunning event selection")
         peaks_kev = np.array(dplms_dict["peaks_kev"])
-        # kev_widths = [tuple(kev_width) for kev_width in dplms_dict["kev_widths"]]
 
         peaks_rounded = [int(peak) for peak in peaks_kev]
         peaks = lh5.read_as(f"{args.raw_table_name}/peak", args.peak_file, library="np")
+        require_peaks_present(
+            np.unique(peaks),
+            peaks_rounded,
+            f"peak file {args.peak_file} requested by dplms config ({args.config_file})",
+        )
         ids = np.isin(peaks, peaks_rounded)
-        peaks = peaks[ids]
-        # idx_list = [np.where(peaks == peak)[0] for peak in peaks_rounded]
 
         raw_cal = lh5.read(args.raw_table_name, args.peak_file, idx=ids)
         msg = f"Time to run event selection {(time.time() - t1):.2f} s, total events {len(raw_cal)}"
         log.info(msg)
+        del peaks, ids
 
         if isinstance(dsp_config, str | list):
             dsp_config = Props.read_from(dsp_config)
@@ -156,13 +179,12 @@ def par_geds_dsp_dplms() -> None:
         for cut in cut_dict:
             idxs = dsp_fft[cut].nda & idxs
 
-        selected_eidxs = eidxs[: len(dsp_fft)]
-        raw_fft = lh5.read(
-            args.raw_table_name,
-            fft_files,
-            n_rows=dplms_dict["n_baselines"],
-            idx=selected_eidxs[idxs],
-        )
+        # idxs is a boolean mask over the rows already loaded in raw_fft, so
+        # subset in memory instead of re-reading the FFT files from disk
+        # (row-identical: the first read was idx=eidxs capped at n_baselines,
+        # so the re-applied n_rows cap was a no-op)
+        raw_fft = take_table_rows(raw_fft, idxs)
+        del dsp_fft, energies, eidxs, cut_dict, idxs
 
         log.debug("Applied Cuts")
 
@@ -201,17 +223,14 @@ def par_geds_dsp_dplms() -> None:
 
     db_dict.update(out_dict)
 
-    Path(args.lh5_path).parent.mkdir(parents=True, exist_ok=True)
     lh5.write(
         Table(col_dict={"dplms": dplms_pars}),
         name=args.channel,
         lh5_file=args.lh5_path,
         wo_mode="overwrite",
     )
-    Path(args.dsp_pars).parent.mkdir(parents=True, exist_ok=True)
     Props.write_to(args.dsp_pars, convert_dict_np_to_float(db_dict))
 
     if args.plot_path:
-        Path(args.plot_path).parent.mkdir(parents=True, exist_ok=True)
         with Path(args.plot_path).open("wb") as f:
             pkl.dump(inplot_dict, f, protocol=pkl.HIGHEST_PROTOCOL)

{legend_dataflow_scripts-0.3.2 → legend_dataflow_scripts-0.4.0a1}/src/legenddataflowscripts/par/geds/dsp/eopt.py

@@ -19,7 +19,13 @@ from pygama.pargen.dsp_optimize import (
     run_bayesian_optimisation,
 )
 
-from ....utils import build_log
+from ....utils import (
+    build_log,
+    check_input_files,
+    prepare_output_paths,
+    require_config_keys,
+    require_peaks_present,
+)
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 try:
@@ -64,7 +70,11 @@ def par_geds_dsp_eopt() -> None:
         Energy optimisation configuration file(s).  Must contain ``run_eopt``
         (bool), ``peaks`` (list of keV values), ``kev_widths``, ``fom``,
         ``fom_field``, ``fom_err_field``, ``initial_samples``, ``acq_func``,
-        ``batch_size``, and ``n_iter``.
+        ``batch_size``, and ``n_iter``.  Optional: ``use_log_pdf`` (bool,
+        default false) builds the FOM's unbinned fits from the model's
+        log-density (iminuit ``log=True``) — substantially faster, results
+        differ at machine-precision level; requires a pygama version with
+        ``use_log_pdf`` support.
     ``--log-config`` : str, optional
         Logging configuration file.
     ``--raw-table-name`` : str
@@ -107,7 +117,9 @@ def par_geds_dsp_eopt() -> None:
     argparser.add_argument(
         "--final-dsp-pars", help="final_dsp_pars", type=str, required=True
     )
-    argparser.add_argument("--qbb-grid-path", help="qbb_grid_path", type=str)
+    argparser.add_argument(
+        "--qbb-grid-path", help="qbb_grid_path", type=str, required=True
+    )
     argparser.add_argument("--plot-path", help="plot_path", type=str)
 
     argparser.add_argument(
@@ -115,25 +127,31 @@ def par_geds_dsp_eopt() -> None:
     )
     args = argparser.parse_args()
 
-    dsp_config = Props.read_from(args.processing_chain)
     log = build_log(args.log_config, args.log)
 
+    check_input_files(args.peak_file, "--peak-file")
+    check_input_files(args.inplots, "--inplots")
+    prepare_output_paths(args.qbb_grid_path, args.final_dsp_pars, args.plot_path)
+
+    dsp_config = Props.read_from(args.processing_chain)
+
     t0 = time.time()
 
     opt_dict = Props.read_from(args.config_file)
     db_dict = Props.read_from(args.decay_const)
+    require_config_keys(opt_dict, ["run_eopt"], f"eopt config ({args.config_file})")
 
-    if opt_dict.pop("run_eopt") is True:
+    run_eopt = opt_dict.pop("run_eopt") is True
+    if run_eopt:
         peaks_kev = np.array(opt_dict["peaks"])
         kev_widths = [tuple(kev_width) for kev_width in opt_dict["kev_widths"]]
 
         kwarg_dicts_cusp = []
         kwarg_dicts_trap = []
         kwarg_dicts_zac = []
-        for peak in peaks_kev:
-            peak_idx = np.where(peaks_kev == peak)[0][0]
-            kev_width = kev_widths[peak_idx]
-
+        # strict: a length mismatch between the peaks and kev_widths config
+        # lists is a config error and must not silently drop peaks
+        for peak, kev_width in zip(peaks_kev, kev_widths, strict=True):
             kwarg_dicts_cusp.append(
                 {
                     "parameter": "cuspEmax",
@@ -164,6 +182,11 @@ def par_geds_dsp_eopt() -> None:
 
         peaks_rounded = [int(peak) for peak in peaks_kev]
         peaks = lh5.read_as(f"{args.raw_table_name}/peak", args.peak_file, library="np")
+        require_peaks_present(
+            np.unique(peaks),
+            peaks_rounded,
+            f"peak file {args.peak_file} requested by eopt config ({args.config_file})",
+        )
         ids = np.isin(peaks, peaks_rounded)
         peaks = peaks[ids]
         idx_list = [np.where(peaks == peak)[0] for peak in peaks_rounded]
@@ -191,29 +214,42 @@ def par_geds_dsp_eopt() -> None:
         db_dict["etrap"] = {"flat": flat_val}
 
         tb_data.add_column("dt_eff", init_data["dt_eff"])
+        # add_column stores the Array object itself, so dt_eff survives freeing
+        # the rest of the init DSP output (otherwise held for the whole
+        # optimisation)
+        del init_data, full_dt
 
         dsp_config["processors"].pop("dt_eff")
 
         dsp_config["outputs"] = ["zacEmax", "cuspEmax", "trapEmax", "dt_eff"]
 
+        # opt-in: build the unbinned NLL fits from the model's log-density
+        # (iminuit log=True mode) — much faster on large samples, results
+        # differ at machine-precision level. Requires pygama with
+        # use_log_pdf support; older versions silently ignore the key.
+        use_log_pdf = opt_dict.get("use_log_pdf", False)
+
         kwarg_dict = [
             {
                 "peak_dicts": kwarg_dicts_cusp,
                 "ctc_param": "dt_eff",
                 "idx_list": idx_list,
                 "peaks_kev": peaks_kev,
+                "use_log_pdf": use_log_pdf,
             },
             {
                 "peak_dicts": kwarg_dicts_zac,
                 "ctc_param": "dt_eff",
                 "idx_list": idx_list,
                 "peaks_kev": peaks_kev,
+                "use_log_pdf": use_log_pdf,
             },
             {
                 "peak_dicts": kwarg_dicts_trap,
                 "ctc_param": "dt_eff",
                 "idx_list": idx_list,
                 "peaks_kev": peaks_kev,
+                "use_log_pdf": use_log_pdf,
             },
         ]
 
@@ -414,14 +450,12 @@ def par_geds_dsp_eopt() -> None:
         else:
             db_dict.update({"ctc_params": out_alpha_dict})
 
-        Path(args.qbb_grid_path).parent.mkdir(parents=True, exist_ok=True)
         with Path(args.qbb_grid_path).open("wb") as f:
             pkl.dump({"eopt": optimisers}, f, protocol=pkl.HIGHEST_PROTOCOL)
 
     else:
         Path(args.qbb_grid_path).touch()
 
-    Path(args.final_dsp_pars).parent.mkdir(parents=True, exist_ok=True)
     Props.write_to(args.final_dsp_pars, db_dict)
 
     if args.plot_path:
@@ -431,21 +465,22 @@ def par_geds_dsp_eopt() -> None:
         else:
             plot_dict = {}
 
-        plot_dict["trap_optimisation"] = {
-            "kernel_space": bopt_trap.plot(init_samples=sample_x),
-            "acq_space": bopt_trap.plot_acq(init_samples=sample_x),
-        }
+        # the optimiser plots only exist when the optimisation actually ran
+        if run_eopt:
+            plot_dict["trap_optimisation"] = {
+                "kernel_space": bopt_trap.plot(init_samples=sample_x),
+                "acq_space": bopt_trap.plot_acq(init_samples=sample_x),
+            }
 
-        plot_dict["cusp_optimisation"] = {
-            "kernel_space": bopt_cusp.plot(init_samples=sample_x),
-            "acq_space": bopt_cusp.plot_acq(init_samples=sample_x),
-        }
+            plot_dict["cusp_optimisation"] = {
+                "kernel_space": bopt_cusp.plot(init_samples=sample_x),
+                "acq_space": bopt_cusp.plot_acq(init_samples=sample_x),
+            }
 
-        plot_dict["zac_optimisation"] = {
-            "kernel_space": bopt_zac.plot(init_samples=sample_x),
-            "acq_space": bopt_zac.plot_acq(init_samples=sample_x),
-        }
+            plot_dict["zac_optimisation"] = {
+                "kernel_space": bopt_zac.plot(init_samples=sample_x),
+                "acq_space": bopt_zac.plot_acq(init_samples=sample_x),
+            }
 
-        Path(args.plot_path).parent.mkdir(parents=True, exist_ok=True)
         with Path(args.plot_path).open("wb") as w:
             pkl.dump(plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)