legend-dataflow-scripts 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

{legend_dataflow_scripts-0.1.2.dist-info → legend_dataflow_scripts-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.1.2
+Version: 0.1.4
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -23,7 +23,7 @@ Requires-Dist: pygama>=2.1
 Requires-Dist: dspeed>=1.6
 Requires-Dist: pylegendmeta>=1.2.5
 Requires-Dist: legend-pydataobj>=1.11
-Requires-Dist: legend-daq2lh5>=1.5.0
+Requires-Dist: legend-daq2lh5>=1.6.1
 Requires-Dist: pip
 Provides-Extra: test
 Requires-Dist: legend-dataflow-scripts; extra == "test"

{legend_dataflow_scripts-0.1.2.dist-info → legend_dataflow_scripts-0.1.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 legenddataflowscripts/__init__.py,sha256=hlpvTxSBjOyXlZUyOyYx3VwT5LS6zNzhAZnTmfT3NjU,303
-legenddataflowscripts/_version.py,sha256=bSmADqydH8nBu-J4lG8UVuR7hnU_zcwhnSav2oQ0W0A,511
+legenddataflowscripts/_version.py,sha256=hcPkC9vIGgfrKK6ft7ysLT7iOCjpFmCBmyKLmXiaZ1g,511
 legenddataflowscripts/par/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/par/geds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/par/geds/dsp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -7,30 +7,31 @@ legenddataflowscripts/par/geds/dsp/dplms.py,sha256=OOvnN_OcU046zhuyNTGxcmGYLCZ7Y
 legenddataflowscripts/par/geds/dsp/eopt.py,sha256=UklMozVw57X8EhOVjOxwKSXtuHkX45iJ2YMLx69TQps,13680
 legenddataflowscripts/par/geds/dsp/evtsel.py,sha256=qheFqGezjRxW9UNf9DV1n8VuZjCLGZT-WaWyhrfgPJ4,16485
 legenddataflowscripts/par/geds/dsp/nopt.py,sha256=K7R4OBfO0wek03VvhBoYXDP7lURi_Xoph9Kb3iMkfx4,4263
-legenddataflowscripts/par/geds/dsp/pz.py,sha256=6tkXvL1ND-j4ulxfsRJadiNXPWqqadT7k99eti9MK-k,8265
+legenddataflowscripts/par/geds/dsp/pz.py,sha256=sucYUtsgWKYtR-GbmRh5b61EYHqNRMWGckD3DJdHUvs,8273
 legenddataflowscripts/par/geds/dsp/svm.py,sha256=eDneRB_PQZp8Q4n2VheTX3kbu4ufZQ-jnuCCjvtwFpk,826
 legenddataflowscripts/par/geds/dsp/svm_build.py,sha256=8K0NUpQqL2HUWRKjMpM5H-TienVPUzIuKED94ZZnIzA,2227
 legenddataflowscripts/par/geds/hit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-legenddataflowscripts/par/geds/hit/aoe.py,sha256=0reLWzYxau09DKyu_KrywC56njSLOVGAT6QfDBSNfvM,8499
+legenddataflowscripts/par/geds/hit/aoe.py,sha256=4sXRz6UaPD7phihAO3_C42l7fJT8UrhhOZswzO8rAqw,11387
 legenddataflowscripts/par/geds/hit/ecal.py,sha256=tsIL9FP-aXgM2B1K8ygjcsLeNZ9Yv7je84fT-K7m3vQ,26194
-legenddataflowscripts/par/geds/hit/lq.py,sha256=CWasjmdaBNVk7AFNrV-RZ8UcvUzfww7wxvhrRdGSobc,7379
-legenddataflowscripts/par/geds/hit/qc.py,sha256=yT922UcNdw9X0yQ-JqoABeDuT-NMpKE2dvKO7qUGLes,11821
+legenddataflowscripts/par/geds/hit/lq.py,sha256=td0qVvVRwYbhiA-IhAZYLgTdW6xpTeeE3mrSoWeLw2U,11307
+legenddataflowscripts/par/geds/hit/qc.py,sha256=16RWhybvnODv4dUuSkO6EDtPvJJJmeFwFaPTgS9N7cs,12746
 legenddataflowscripts/tier/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/tier/dsp.py,sha256=J_ABEqbQurZHJOg8LV2aporNjZQpPg2bDZG6DkmuAL4,8633
 legenddataflowscripts/tier/hit.py,sha256=-LCfsqYAYPAuN2OKbPuh4G2P5B4uA4YmQGcqUtbZxis,4989
-legenddataflowscripts/utils/__init__.py,sha256=_eYTjqTMgjeQGO4HKmiqVAkyW9fh9FEz-_lQtcM-Lfg,366
+legenddataflowscripts/utils/__init__.py,sha256=NPpxqapio8CeQIbG8gYnz-OPoUbOlDT6tjXV-lJrKWc,426
 legenddataflowscripts/utils/alias_table.py,sha256=e0wRiDcpje8VVr_PlcbkA0kPz0qSGpGX4vJ3tfGefMA,742
 legenddataflowscripts/utils/cfgtools.py,sha256=_1yxw_eJ08AihONhJ9sWet5HQZpOagj8Yg8y9LS3zX4,381
 legenddataflowscripts/utils/convert_np.py,sha256=8q_K3w3jAREG-9CJlErP3Vca0OYTYAYJIK4g_TClvBU,819
 legenddataflowscripts/utils/log.py,sha256=5v8J7PZJHHoSy12w9e9DkYmqdPvYXj6YfVaMbhvX614,2230
+legenddataflowscripts/utils/plot_dict.py,sha256=-wfJC5U9OTE1Asazz64kT5Ta008w6UJmDSsr3YAubRM,449
 legenddataflowscripts/utils/pulser_removal.py,sha256=kuARdp1jf-lsUWcb0_KRDp-ZXzkHNrDCXUc3h7TJm7Q,424
-legenddataflowscripts/workflow/__init__.py,sha256=p-57OklNpVYqwMaLctLO8zMvhpXaTILkd3j4CaS_Juk,394
+legenddataflowscripts/workflow/__init__.py,sha256=JhudKYhBT8bXtX4LCqxQCHzUiITpugAtFxePWEtphC4,474
 legenddataflowscripts/workflow/execenv.py,sha256=O6Z6EH6Yp0JXnwX2wTzduHC9Q9gu-_d1RuuE2pQ5caE,9061
 legenddataflowscripts/workflow/filedb.py,sha256=rbvOcXUxLbHz177QuDIDAL3aysz-bZDjHOiMsRHssZo,3434
 legenddataflowscripts/workflow/pre_compile_catalog.py,sha256=cEK0KXh-ClSE2Bo9MK471o79XG22bMY5r-2tIihtCfk,790
 legenddataflowscripts/workflow/utils.py,sha256=eKE8KIG2ffynZt9fTbI1SVQV85i3aW9GFGh1Nio1iDo,3118
-legend_dataflow_scripts-0.1.2.dist-info/METADATA,sha256=FsY5VWfz1t6KmRXwG2iRBnawVy5_yqy3_rNpT50FOHo,3122
-legend_dataflow_scripts-0.1.2.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-legend_dataflow_scripts-0.1.2.dist-info/entry_points.txt,sha256=B197waSm-orA_ZS-9rkxNDsmOHdCn8CbWodnlqXQKRg,1313
-legend_dataflow_scripts-0.1.2.dist-info/top_level.txt,sha256=s8E2chjJNYUbrN6whFG_VCsJKySFp1IOXLcUefA7DB0,22
-legend_dataflow_scripts-0.1.2.dist-info/RECORD,,
+legend_dataflow_scripts-0.1.4.dist-info/METADATA,sha256=kCV4uTOiqJBCGvMteN0O5b0YGwLCMrZR8DF3LytqCF4,3122
+legend_dataflow_scripts-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+legend_dataflow_scripts-0.1.4.dist-info/entry_points.txt,sha256=B197waSm-orA_ZS-9rkxNDsmOHdCn8CbWodnlqXQKRg,1313
+legend_dataflow_scripts-0.1.4.dist-info/top_level.txt,sha256=s8E2chjJNYUbrN6whFG_VCsJKySFp1IOXLcUefA7DB0,22
+legend_dataflow_scripts-0.1.4.dist-info/RECORD,,

{legend_dataflow_scripts-0.1.2.dist-info → legend_dataflow_scripts-0.1.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

legenddataflowscripts/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.2'
-__version_tuple__ = version_tuple = (0, 1, 2)
+__version__ = version = '0.1.4'
+__version_tuple__ = version_tuple = (0, 1, 4)

legenddataflowscripts/par/geds/dsp/pz.py

@@ -181,14 +181,14 @@ def par_geds_dsp_pz() -> None:
 
             plot_dict = tau.plot_waveforms_after_correction(
                 tb_data,
-                kwarg_dict("wf_pz_field", "wf_pz"),
+                kwarg_dict.get("wf_pz_field", "wf_pz"),
                 norm_param=kwarg_dict.get("norm_param", "pz_mean"),
                 xlim=[0, len(tb_data[kwarg_dict["wf_field"]]["values"].nda[0])],
             )
 
             zoomed = tau.plot_waveforms_after_correction(
                 tb_data,
-                kwarg_dict("wf_pz_field", "wf_pz"),
+                kwarg_dict.get("wf_pz_field", "wf_pz"),
                 norm_param=kwarg_dict.get("norm_param", "pz_mean"),
                 xlim=[400, len(tb_data[kwarg_dict["wf_field"]]["values"].nda[0])],
                 ylim=[0.8, 1.1],

legenddataflowscripts/par/geds/hit/aoe.py

@@ -1,7 +1,11 @@
 from __future__ import annotations
 
 import argparse
+import copy
+import logging
 import pickle as pkl
+import re
+import time
 import warnings
 from pathlib import Path
 
@@ -9,42 +13,172 @@ import numpy as np
 from dbetto import TextDB
 from dbetto.catalog import Props
 from pygama.pargen.AoE_cal import *  # noqa: F403
-from pygama.pargen.AoE_cal import CalAoE, Pol1, SigmaFit, aoe_peak
+from pygama.pargen.AoE_cal import CalAoE
 from pygama.pargen.utils import load_data
 
 from ....utils import (
     build_log,
     convert_dict_np_to_float,
+    fill_plot_dict,
     get_pulser_mask,
 )
 
+log = logging.getLogger(__name__)
+
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 
 
 def get_results_dict(aoe_class):
-    return {
-        "cal_energy_param": aoe_class.cal_energy_param,
-        "dt_param": aoe_class.dt_param,
-        "rt_correction": aoe_class.dt_corr,
-        "1000-1300keV": aoe_class.timecorr_df.to_dict("index"),
-        "correction_fit_results": aoe_class.energy_corr_res_dict,
-        "low_cut": aoe_class.low_cut_val,
-        "high_cut": aoe_class.high_cut_val,
-        "low_side_sfs": aoe_class.low_side_sfs.to_dict("index"),
-        "2_side_sfs": aoe_class.two_side_sfs.to_dict("index"),
-    }
+    result_dict = {}
+    for tstamp in aoe_class.low_side_sfs_by_run:
+        result_dict[tstamp] = {
+            "cal_energy_param": aoe_class.cal_energy_param,
+            "dt_param": aoe_class.dt_param,
+            "rt_correction": aoe_class.dt_corr,
+            "1000-1300keV": aoe_class.timecorr_df.to_dict("index"),
+            "correction_fit_results": aoe_class.energy_corr_res_dict,
+            "low_cut": aoe_class.low_cut_val,
+            "high_cut": aoe_class.high_cut_val,
+            "low_side_sfs": aoe_class.low_side_sfs.to_dict("index"),
+            "2_side_sfs": aoe_class.two_side_sfs.to_dict("index"),
+            "low_side_sfs_by_run": aoe_class.low_side_sfs_by_run[tstamp].to_dict(
+                "index"
+            ),
+            "2_side_sfs_by_run": aoe_class.two_side_sfs_by_run[tstamp].to_dict("index"),
+        }
+    return result_dict
+
+
+def run_aoe_calibration(
+    data,
+    cal_dicts,
+    results_dicts,
+    object_dicts,
+    plot_dicts,
+    config,
+    debug_mode=False,
+):
+    if isinstance(config, str | list):
+        config = Props.read_from(config)
+
+    if config.get("run_aoe", True) is True:
+        if "plot_options" in config:
+            for field, item in config["plot_options"].items():
+                config["plot_options"][field]["function"] = eval(item["function"])
+
+        if "dt_cut" in config and config["dt_cut"] is not None:
+            cut_dict = config["dt_cut"]["cut"]
+            for tstamp in cal_dicts:
+                cal_dicts[tstamp].update(cut_dict)
+
+            exp = cut_dict[next(iter(cut_dict))]["expression"]
+            for key in cut_dict[next(iter(cut_dict))]["parameters"]:
+                exp = re.sub(f"(?<![a-zA-Z0-9]){key}(?![a-zA-Z0-9])", f"@{key}", exp)
+            data[next(iter(cut_dict))] = data.eval(
+                exp, local_dict=cut_dict[next(iter(cut_dict))]["parameters"]
+            )
+
+        try:
+            eres = copy.deepcopy(
+                results_dicts[next(iter(results_dicts))]["partition_ecal"][
+                    config["cal_energy_param"]
+                ]["eres_linear"]
+            )
+
+            def eres_func(x):
+                return eval(eres["expression"], dict(x=x, **eres["parameters"]))
+
+            if np.isnan(eres_func(2000)):
+                raise RuntimeError
+        except (KeyError, RuntimeError):
+            try:
+                eres = copy.deepcopy(
+                    results_dicts[next(iter(results_dicts))]["ecal"][
+                        config["cal_energy_param"]
+                    ]["eres_linear"]
+                )
+
+                def eres_func(x):
+                    return eval(eres["expression"], dict(x=x, **eres["parameters"]))
+
+            except KeyError:
+
+                def eres_func(x):
+                    return x * np.nan
+
+        data["AoE_Uncorr"] = (
+            data[config["current_param"]] / data[config["energy_param"]]
+        )
+
+        start = time.time()
+        log.info("calibrating A/E")
+
+        aoe = CalAoE(
+            cal_dicts=cal_dicts,
+            cal_energy_param=config["cal_energy_param"],
+            eres_func=eres_func,
+            pdf=eval(config.get("pdf", "aoe_peak")),
+            mean_func=eval(config.get("mean_func", "Pol1")),
+            sigma_func=eval(config.get("sigma_func", "SigmaFit")),
+            selection_string=f"{config['cut_field']}&(~is_pulser)",
+            dt_corr=config.get("dt_corr", False),
+            dep_correct=config.get("dep_correct", False),
+            dt_cut=config.get("dt_cut", None),
+            dt_param=config.get("dt_param", 3),
+            high_cut_val=config.get("high_cut_val", 3),
+            compt_bands_width=config.get("debug_mode", 20),
+            debug_mode=debug_mode | config.get("debug_mode", False),
+        )
+        aoe.update_cal_dicts(
+            {
+                "AoE_Uncorr": {
+                    "expression": f"{config['current_param']}/{config['energy_param']}",
+                    "parameters": {},
+                }
+            }
+        )
+        aoe.calibrate(data, "AoE_Uncorr")
 
+        msg = f"A/E calibration completed in {time.time() - start:.2f} seconds"
+        log.info(msg)
 
-def fill_plot_dict(aoe_class, data, plot_options, plot_dict=None):
-    if plot_dict is not None:
-        for key, item in plot_options.items():
-            if item["options"] is not None:
-                plot_dict[key] = item["function"](aoe_class, data, **item["options"])
-            else:
-                plot_dict[key] = item["function"](aoe_class, data)
+        out_dict = get_results_dict(aoe)
+        aoe_plot_dict = fill_plot_dict(aoe, data, config.get("plot_options", None))
+
+        aoe.pdf = aoe.pdf.name
+        # need to change eres func as can't pickle lambdas
+        try:
+            aoe.eres_func = results_dicts[next(iter(results_dicts))]["partition_ecal"][
+                config["cal_energy_param"]
+            ]["eres_linear"]
+        except KeyError:
+            aoe.eres_func = {}
     else:
-        plot_dict = {}
-    return plot_dict
+        out_dict = dict.fromkeys(cal_dicts)
+        aoe_plot_dict = {}
+        aoe = None
+
+    out_result_dicts = {}
+    for tstamp, result_dict in results_dicts.items():
+        out_result_dicts[tstamp] = dict(**result_dict, aoe=out_dict[tstamp])
+
+    out_object_dicts = {}
+    for tstamp, object_dict in object_dicts.items():
+        out_object_dicts[tstamp] = dict(**object_dict, aoe=aoe)
+
+    common_dict = (
+        aoe_plot_dict.pop("common") if "common" in list(aoe_plot_dict) else None
+    )
+    out_plot_dicts = {}
+    for tstamp, plot_dict in plot_dicts.items():
+        if "common" in list(plot_dict) and common_dict is not None:
+            plot_dict["common"].update(common_dict)
+        elif common_dict is not None:
+            plot_dict["common"] = common_dict
+        plot_dict.update({"aoe": aoe_plot_dict})
+        out_plot_dicts[tstamp] = plot_dict
+
+    return cal_dicts, out_result_dicts, out_object_dicts, out_plot_dicts
 
 
 def par_geds_hit_aoe() -> None:
@@ -79,7 +213,7 @@ def par_geds_hit_aoe() -> None:
     configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
     config_dict = configs["snakemake_rules"]["pars_hit_aoecal"]
 
-    log = build_log(config_dict, args.log)
+    build_log(config_dict, args.log)
 
     channel_dict = config_dict["inputs"]["aoecal_config"][args.channel]
     kwarg_dict = Props.read_from(channel_dict)
@@ -91,40 +225,16 @@ def par_geds_hit_aoe() -> None:
     with Path(args.eres_file).open("rb") as o:
         object_dict = pkl.load(o)
 
-    if kwarg_dict["run_aoe"] is True:
-        kwarg_dict.pop("run_aoe")
-
-        pdf = eval(kwarg_dict.pop("pdf")) if "pdf" in kwarg_dict else aoe_peak
-
-        sigma_func = (
-            eval(kwarg_dict.pop("sigma_func"))
-            if "sigma_func" in kwarg_dict
-            else SigmaFit
-        )
-
-        mean_func = (
-            eval(kwarg_dict.pop("mean_func")) if "mean_func" in kwarg_dict else Pol1
-        )
-
-        if "plot_options" in kwarg_dict:
-            for field, item in kwarg_dict["plot_options"].items():
-                kwarg_dict["plot_options"][field]["function"] = eval(item["function"])
-
-        with Path(args.files[0]).open() as f:
-            files = f.read().splitlines()
-        files = sorted(files)
-
-        try:
-            eres = eres_dict[kwarg_dict["cal_energy_param"]]["eres_linear"].copy()
-
-            def eres_func(x):
-                return eval(eres["expression"], dict(x=x, **eres["parameters"]))
-
-        except KeyError:
+    if args.inplots:
+        with Path(args.inplots).open("rb") as r:
+            out_plot_dict = pkl.load(r)
+    else:
+        out_plot_dict = {}
 
-            def eres_func(x):
-                return x * np.nan
+    with Path(args.files[0]).open() as f:
+        files = sorted(f.read().splitlines())
 
+    if kwarg_dict["run_aoe"] is True:
         params = [
             kwarg_dict["current_param"],
             "tp_0_est",
@@ -150,9 +260,11 @@ def par_geds_hit_aoe() -> None:
             args.table_name,
             cal_dict,
             params=params,
-            threshold=kwarg_dict.pop("threshold"),
+            threshold=kwarg_dict["threshold"],
             return_selection_mask=True,
         )
+        msg = f"Loaded {len(data)} events"
+        log.info(msg)
 
         mask = get_pulser_mask(
             pulser_file=args.pulser_file,
@@ -160,61 +272,35 @@ def par_geds_hit_aoe() -> None:
 
         data["is_pulser"] = mask[threshold_mask]
 
-        data["AoE_Uncorr"] = (
-            data[kwarg_dict["current_param"]] / data[kwarg_dict["energy_param"]]
-        )
-        aoe = CalAoE(
-            cal_dicts=cal_dict,
-            cal_energy_param=kwarg_dict["cal_energy_param"],
-            eres_func=eres_func,
-            pdf=pdf,
-            mean_func=mean_func,
-            sigma_func=sigma_func,
-            selection_string=f"{kwarg_dict.pop('cut_field')}&(~is_pulser)",
-            dt_corr=kwarg_dict.get("dt_corr", False),
-            dep_correct=kwarg_dict.get("dep_correct", False),
-            dt_cut=kwarg_dict.get("dt_cut", None),
-            dt_param=kwarg_dict.get("dt_param", 3),
-            high_cut_val=kwarg_dict.get("high_cut_val", 3),
-            compt_bands_width=kwarg_dict.get("debug_mode", 20),
-            debug_mode=args.debug | kwarg_dict.get("debug_mode", False),
-        )
-        aoe.update_cal_dicts(
-            {
-                "AoE_Uncorr": {
-                    "expression": f"{kwarg_dict['current_param']}/{kwarg_dict['energy_param']}",
-                    "parameters": {},
-                }
-            }
-        )
-        aoe.calibrate(data, "AoE_Uncorr")
+        msg = f"{len(data.query('~is_pulser'))}  non pulser events"
+        log.info(msg)
 
-        log.info("Calibrated A/E")
-        out_dict = get_results_dict(aoe)
-        plot_dict = fill_plot_dict(aoe, data, kwarg_dict.get("plot_options", None))
+        with Path(args.files[0]).open() as f:
+            files = f.read().splitlines()
+        files = sorted(files)
 
-        aoe.pdf = aoe.pdf.name
+        data["run_timestamp"] = args.timestamp
 
-        # need to change eres func as can't pickle lambdas
-        try:
-            aoe.eres_func = eres_dict[kwarg_dict["cal_energy_param"]][
-                "eres_linear"
-            ].copy()
-        except KeyError:
-            aoe.eres_func = {}
+        cal_dict, results_dicts, object_dicts, plot_dicts = run_aoe_calibration(
+            data,
+            {args.timestamp: cal_dict},
+            {args.timestamp: eres_dict},
+            {args.timestamp: object_dict},
+            {args.timestamp: out_plot_dict},
+            kwarg_dict,
+            debug_mode=args.debug,
+        )
+        cal_dict = cal_dict[args.timestamp]
+        results_dict = results_dicts[args.timestamp]
+        aoe = object_dicts[args.timestamp]
+        plot_dict = plot_dicts[args.timestamp]
     else:
-        out_dict = {}
-        plot_dict = {}
         aoe = None
+        plot_dict = out_plot_dict
 
     if args.plot_file:
         common_dict = plot_dict.pop("common") if "common" in list(plot_dict) else None
-        if args.inplots:
-            with Path(args.inplots).open("rb") as r:
-                out_plot_dict = pkl.load(r)
-            out_plot_dict.update({"aoe": plot_dict})
-        else:
-            out_plot_dict = {"aoe": plot_dict}
+        out_plot_dict.update({"aoe": plot_dict})
 
         if "common" in list(out_plot_dict) and common_dict is not None:
             out_plot_dict["common"].update(common_dict)
@@ -226,10 +312,9 @@ def par_geds_hit_aoe() -> None:
             pkl.dump(out_plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
 
     Path(args.hit_pars).parent.mkdir(parents=True, exist_ok=True)
-    results_dict = dict(**ecal_dict["results"], aoe=out_dict)
     final_hit_dict = {
         "pars": {"operations": cal_dict},
-        "results": results_dict,
+        "results": dict(**ecal_dict["results"], aoe=results_dict),
     }
 
     final_hit_dict = convert_dict_np_to_float(final_hit_dict)
@@ -237,9 +322,5 @@ def par_geds_hit_aoe() -> None:
     Props.write_to(args.hit_pars, final_hit_dict)
 
     Path(args.aoe_results).parent.mkdir(parents=True, exist_ok=True)
-    final_object_dict = dict(
-        **object_dict,
-        aoe=aoe,
-    )
     with Path(args.aoe_results).open("wb") as w:
-        pkl.dump(final_object_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
+        pkl.dump(dict(**object_dict, aoe=aoe), w, protocol=pkl.HIGHEST_PROTOCOL)

legenddataflowscripts/par/geds/hit/lq.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import argparse
+import copy
+import logging
 import pickle as pkl
+import time
 import warnings
 from pathlib import Path
 
 import numpy as np
+import pandas as pd
 from dbetto import TextDB
 from dbetto.catalog import Props
 from pygama.math.distributions import gaussian
@@ -17,9 +21,11 @@ from pygama.pargen.utils import load_data
 from ....utils import (
     build_log,
     convert_dict_np_to_float,
+    fill_plot_dict,
     get_pulser_mask,
 )
 
+log = logging.getLogger(__name__)
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 
 
@@ -34,16 +40,172 @@ def get_results_dict(lq_class):
     }
 
 
-def fill_plot_dict(lq_class, data, plot_options, plot_dict=None):
-    if plot_dict is not None:
-        for key, item in plot_options.items():
-            if item["options"] is not None:
-                plot_dict[key] = item["function"](lq_class, data, **item["options"])
-            else:
-                plot_dict[key] = item["function"](lq_class, data)
+def lq_calibration(
+    data: pd.DataFrame,
+    cal_dicts: dict,
+    energy_param: str,
+    cal_energy_param: str,
+    dt_param: str,
+    eres_func: callable,
+    cdf: callable = gaussian,
+    selection_string: str = "",
+    plot_options: dict | None = None,
+    debug_mode: bool = False,
+):
+    """Loads in data from the provided files and runs the LQ calibration on said files
+
+    Parameters
+    ----------
+    data: pd.DataFrame
+        A dataframe containing the data used for calibrating LQ
+    cal_dicts: dict
+        A dict of hit-level operations to apply to the data
+    energy_param: string
+        The energy parameter of choice. Used for normalizing the
+        raw lq values
+    cal_energy_param: string
+        The calibrated energy parameter of choice
+    dt_param: string
+        The drift-time parameter of choice
+    eres_func: callable
+        The energy resolution functions
+    cdf: callable
+        The CDF used for the binned fitting of LQ distributions
+    cut_field: string
+        A string of flags to apply to the data when running the calibration
+    plot_options: dict
+        A dict containing the plot functions the user wants to run,and any
+        user options to provide those plot functions
+    Returns
+    -------
+    cal_dicts: dict
+        The user provided dict, updated with hit-level operations for LQ
+    results_dict: dict
+        A dict containing the results of the LQ calibration
+    plot_dict: dict
+        A dict containing all the figures specified by the plot options
+    lq: cal_lq class
+        The cal_lq object used for the LQ calibration
+    """
+
+    lq = LQCal(
+        cal_dicts,
+        cal_energy_param,
+        dt_param,
+        eres_func,
+        cdf,
+        selection_string,
+        debug_mode=debug_mode,
+    )
+
+    data["LQ_Ecorr"] = np.divide(data["lq80"], data[energy_param])
+
+    lq.update_cal_dicts(
+        {
+            "LQ_Ecorr": {
+                "expression": f"lq80/{energy_param}",
+                "parameters": {},
+            }
+        }
+    )
+
+    lq.calibrate(data, "LQ_Ecorr")
+    return cal_dicts, get_results_dict(lq), fill_plot_dict(lq, data, plot_options), lq
+
+
+def run_lq_calibration(
+    data,
+    cal_dicts,
+    results_dicts,
+    object_dicts,
+    plot_dicts,
+    configs,
+    debug_mode=False,
+    # gen_plots=True,
+):
+    if isinstance(configs, str | list):
+        configs = Props.read_from(configs)
+
+    if configs.pop("run_lq") is True:
+        if "plot_options" in configs:
+            for field, item in configs["plot_options"].items():
+                configs["plot_options"][field]["function"] = eval(item["function"])
+
+        try:
+            eres = copy.deepcopy(
+                results_dicts[next(iter(results_dicts))]["partition_ecal"][
+                    configs["cal_energy_param"]
+                ]["eres_linear"]
+            )
+
+            def eres_func(x):
+                return eval(eres["expression"], dict(x=x, **eres["parameters"]))
+
+            if np.isnan(eres_func(2000)):
+                raise RuntimeError
+        except (KeyError, RuntimeError):
+            try:
+                eres = copy.deepcopy(
+                    results_dicts[next(iter(results_dicts))]["ecal"][
+                        configs["cal_energy_param"]
+                    ]["eres_linear"]
+                )
+
+                def eres_func(x):
+                    return eval(eres["expression"], dict(x=x, **eres["parameters"]))
+
+            except KeyError:
+
+                def eres_func(x):
+                    return x * np.nan
+
+        log.info("starting lq calibration")
+        start = time.time()
+        cal_dicts, out_dict, lq_plot_dict, lq_obj = lq_calibration(
+            data,
+            cal_dicts=cal_dicts,
+            energy_param=configs["energy_param"],
+            cal_energy_param=configs["cal_energy_param"],
+            dt_param=configs["dt_param"],
+            eres_func=eres_func,
+            cdf=eval(configs.get("cdf", "gaussian")),
+            selection_string=f"{configs.pop('cut_field')}&(~is_pulser)",
+            plot_options=configs.get("plot_options", None),
+            debug_mode=debug_mode | configs.get("debug_mode", False),
+        )
+        msg = f"lq calibration took {time.time() - start:.2f} seconds"
+        log.info(msg)
+        # need to change eres func as can't pickle lambdas
+        try:
+            lq_obj.eres_func = results_dicts[next(iter(results_dicts))][
+                "partition_ecal"
+            ][configs["cal_energy_param"]]["eres_linear"]
+        except KeyError:
+            lq_obj.eres_func = {}
     else:
-        plot_dict = {}
-    return plot_dict
+        out_dict = dict.fromkeys(cal_dicts)
+        lq_plot_dict = {}
+        lq_obj = None
+
+    out_result_dicts = {}
+    for tstamp, result_dict in results_dicts.items():
+        out_result_dicts[tstamp] = dict(**result_dict, lq=out_dict)
+
+    out_object_dicts = {}
+    for tstamp, object_dict in object_dicts.items():
+        out_object_dicts[tstamp] = dict(**object_dict, lq=lq_obj)
+
+    common_dict = lq_plot_dict.pop("common") if "common" in list(lq_plot_dict) else None
+    out_plot_dicts = {}
+    for tstamp, plot_dict in plot_dicts.items():
+        if "common" in list(plot_dict) and common_dict is not None:
+            plot_dict["common"].update(common_dict)
+        elif common_dict is not None:
+            plot_dict["common"] = common_dict
+        plot_dict.update({"lq": lq_plot_dict})
+        out_plot_dicts[tstamp] = plot_dict
+
+    return cal_dicts, out_result_dicts, out_object_dicts, out_plot_dicts
 
 
 def par_geds_hit_lq() -> None:
@@ -78,7 +240,7 @@ def par_geds_hit_lq() -> None:
     configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
     config_dict = configs["snakemake_rules"]["pars_hit_lqcal"]
 
-    log = build_log(config_dict, args.log)
+    build_log(config_dict, args.log)
 
     channel_dict = config_dict["inputs"]["lqcal_config"][args.channel]
     kwarg_dict = Props.read_from(channel_dict)
@@ -87,33 +249,20 @@ def par_geds_hit_lq() -> None:
     cal_dict = ecal_dict["pars"]["operations"]
     eres_dict = ecal_dict["results"]["ecal"]
 
+    if args.inplots:
+        with Path(args.inplots).open("rb") as r:
+            plot_dict = pkl.load(r)
+    else:
+        plot_dict = {}
+
     with Path(args.eres_file).open("rb") as o:
         object_dict = pkl.load(o)
 
     if kwarg_dict["run_lq"] is True:
-        kwarg_dict.pop("run_lq")
-
-        cdf = eval(kwarg_dict.pop("cdf")) if "cdf" in kwarg_dict else gaussian
-
-        if "plot_options" in kwarg_dict:
-            for field, item in kwarg_dict["plot_options"].items():
-                kwarg_dict["plot_options"][field]["function"] = eval(item["function"])
-
         with Path(args.files[0]).open() as f:
             files = f.read().splitlines()
         files = sorted(files)
 
-        try:
-            eres = eres_dict[kwarg_dict["cal_energy_param"]]["eres_linear"].copy()
-
-            def eres_func(x):
-                return eval(eres["expression"], dict(x=x, **eres["parameters"]))
-
-        except KeyError:
-
-            def eres_func(x):
-                return x * np.nan
-
         params = [
             "lq80",
             "dt_eff",
@@ -132,73 +281,46 @@ def par_geds_hit_lq() -> None:
             return_selection_mask=True,
         )
 
+        msg = f"Loaded {len(data)} events"
+        log.info(msg)
+
         mask = get_pulser_mask(
             pulser_file=args.pulser_file,
         )
 
         data["is_pulser"] = mask[threshold_mask]
 
-        lq = LQCal(
-            cal_dict,
-            kwarg_dict["cal_energy_param"],
-            kwarg_dict["dt_param"],
-            eres_func,
-            cdf,
-            selection_string=f"{kwarg_dict.pop('cut_field')}&(~is_pulser)",
-            debug_mode=args.debug | kwarg_dict.get("debug_mode", False),
-        )
+        msg = f"{len(data.query('~is_pulser'))}  non pulser events"
+        log.info(msg)
 
-        data["LQ_Ecorr"] = np.divide(data["lq80"], data[kwarg_dict["energy_param"]])
+        data["run_timestamp"] = args.timestamp
 
-        lq.update_cal_dicts(
-            {
-                "LQ_Ecorr": {
-                    "expression": f"lq80/{kwarg_dict['energy_param']}",
-                    "parameters": {},
-                }
-            }
+        out_dicts, eres_dicts, plot_dicts, lq_dict = run_lq_calibration(
+            data,
+            cal_dicts={args.timestamp: cal_dict},
+            results_dicts={args.timestamp: eres_dict},
+            object_dicts={args.timestamp: object_dict},
+            plot_dicts={args.timestamp: plot_dict},
+            configs=kwarg_dict,
+            debug_mode=args.debug,
         )
+        cal_dict = out_dicts[args.timestamp]
+        eres_dict = eres_dicts[args.timestamp]
+        plot_dict = plot_dicts[args.timestamp]
+        lq = lq_dict[args.timestamp]
 
-        lq.calibrate(data, "LQ_Ecorr")
-        log.info("Calibrated LQ")
-
-        out_dict = get_results_dict(lq)
-        plot_dict = fill_plot_dict(lq, data, kwarg_dict.get("plot_options", None))
-
-        # need to change eres func as can't pickle lambdas
-        try:
-            lq.eres_func = eres_dict[kwarg_dict["cal_energy_param"]][
-                "eres_linear"
-            ].copy()
-        except KeyError:
-            lq.eres_func = {}
     else:
-        out_dict = {}
-        plot_dict = {}
         lq = None
 
     if args.plot_file:
-        common_dict = plot_dict.pop("common") if "common" in list(plot_dict) else None
-        if args.inplots:
-            with Path(args.inplots).open("rb") as r:
-                out_plot_dict = pkl.load(r)
-            out_plot_dict.update({"lq": plot_dict})
-        else:
-            out_plot_dict = {"lq": plot_dict}
-
-        if "common" in list(out_plot_dict) and common_dict is not None:
-            out_plot_dict["common"].update(common_dict)
-        elif common_dict is not None:
-            out_plot_dict["common"] = common_dict
-
         Path(args.plot_file).parent.mkdir(parents=True, exist_ok=True)
         with Path(args.plot_file).open("wb") as w:
-            pkl.dump(out_plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
+            pkl.dump(plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
 
     final_hit_dict = convert_dict_np_to_float(
         {
             "pars": {"operations": cal_dict},
-            "results": dict(**ecal_dict["results"], lq=out_dict),
+            "results": dict(**ecal_dict["results"], lq=eres_dict),
         }
     )
     Path(args.hit_pars).parent.mkdir(parents=True, exist_ok=True)

legenddataflowscripts/par/geds/hit/qc.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 import argparse
 import json
+import logging
 import pickle as pkl
 import re
+import time
 import warnings
 from pathlib import Path
 
@@ -23,83 +25,23 @@ from ....utils import (
     get_pulser_mask,
 )
 
+log = logging.getLogger(__name__)
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 
 
-def par_geds_hit_qc() -> None:
-    argparser = argparse.ArgumentParser()
-    argparser.add_argument("--cal-files", help="cal_files", nargs="*", type=str)
-    argparser.add_argument("--fft-files", help="fft_files", nargs="*", type=str)
-
-    argparser.add_argument(
-        "--tcm-filelist", help="tcm_filelist", type=str, required=False
-    )
-    argparser.add_argument(
-        "--pulser-file", help="pulser_file", type=str, required=False
-    )
-    argparser.add_argument(
-        "--overwrite-files",
-        help="overwrite_files",
-        type=str,
-        required=False,
-        nargs="*",
-    )
-
-    argparser.add_argument("--configs", help="config", type=str, required=True)
-    argparser.add_argument("--log", help="log_file", type=str)
-
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
-    argparser.add_argument("--table-name", help="table name", type=str, required=True)
-    argparser.add_argument("--tier", help="tier", type=str, default="hit")
-
-    argparser.add_argument("--plot-path", help="plot_path", type=str, required=False)
-    argparser.add_argument("--save-path", help="save_path", type=str)
-    args = argparser.parse_args()
-
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    if args.tier == "hit":
-        config_dict = configs["snakemake_rules"]["pars_hit_qc"]
-    elif args.tier == "pht":
-        config_dict = configs["snakemake_rules"]["pars_pht_qc"]
-    else:
-        msg = f"tier {args.tier} not recognized"
-        raise ValueError(msg)
-
-    log = build_log(config_dict, args.log)
-
-    # get metadata dictionary
-    channel_dict = config_dict["inputs"]["qc_config"][args.channel]
-    kwarg_dict = Props.read_from(channel_dict)
+def build_qc(
+    config,
+    cal_files,
+    fft_files,
+    table_name,
+    overwrite=None,
+    pulser_file=None,
+    build_plots=False,
+):
+    search_name = table_name if table_name[-1] == "/" else table_name + "/"
 
-    if args.overwrite_files:
-        overwrite = Props.read_from(args.overwrite_files)
-        if args.channel in overwrite:
-            overwrite = overwrite[args.channel]["pars"]["operations"]
-        else:
-            overwrite = None
-    else:
-        overwrite = None
-
-    if len(args.fft_files) == 1 and Path(args.fft_files[0]).suffix == ".filelist":
-        with Path(args.fft_files[0]).open() as f:
-            fft_files = f.read().splitlines()
-    else:
-        fft_files = args.fft_files
-
-    if len(args.cal_files) == 1 and Path(args.cal_files[0]).suffix == ".filelist":
-        with Path(args.cal_files[0]).open() as f:
-            cal_files = f.read().splitlines()
-    else:
-        cal_files = args.fft_files
-
-    search_name = (
-        args.table_name if args.table_name[-1] == "/" else args.table_name + "/"
-    )
-
-    kwarg_dict_fft = kwarg_dict["fft_fields"]
-    kwarg_dict_cal = kwarg_dict["cal_fields"]
+    kwarg_dict_fft = config["fft_fields"]
+    kwarg_dict_cal = config["cal_fields"]
 
     cut_fields = get_keys(
         [key.replace(search_name, "") for key in ls(cal_files[0], search_name)],
@@ -110,8 +52,8 @@ def par_geds_hit_qc() -> None:
         kwarg_dict_fft["cut_parameters"],
     )
 
-    if "initial_cal_cuts" in kwarg_dict:
-        init_cal = kwarg_dict["initial_cal_cuts"]
+    if "initial_cal_cuts" in config:
+        init_cal = config["initial_cal_cuts"]
         cut_fields += get_keys(
             [key.replace(search_name, "") for key in ls(cal_files[0], search_name)],
             init_cal["cut_parameters"],
@@ -120,11 +62,14 @@ def par_geds_hit_qc() -> None:
     if len(fft_files) > 0:
         fft_data = load_data(
             fft_files,
-            args.table_name,
+            table_name,
             {},
             [*cut_fields, "t_sat_lo", "timestamp", "trapTmax"],
         )
 
+        msg = f"{len(fft_data)} events"
+        log.info(msg)
+
         discharges = fft_data["t_sat_lo"] > 0
         discharge_timestamps = np.where(fft_data["timestamp"][discharges])[0]
         is_recovering = np.full(len(fft_data), False, dtype=bool)
@@ -139,17 +84,22 @@ def par_geds_hit_qc() -> None:
             )
         fft_data["is_recovering"] = is_recovering
 
+        msg = f"{len(fft_data.query('is_recovering'))} discharge recovery events"
+        log.info(msg)
+
         hit_dict_fft = {}
         plot_dict_fft = {}
         cut_data = fft_data.query("is_recovering==0")
+
         msg = f"cut_data shape: {len(cut_data)}"
         log.debug(msg)
+
         for name, cut in kwarg_dict_fft["cut_parameters"].items():
             cut_dict, cut_plots = generate_cut_classifiers(
                 cut_data,
                 {name: cut},
-                kwarg_dict.get("rounding", 4),
-                display=1 if args.plot_path else 0,
+                config.get("rounding", 4),
+                display=build_plots,
             )
             hit_dict_fft.update(cut_dict)
             plot_dict_fft.update(cut_plots)
@@ -179,6 +129,7 @@ def par_geds_hit_qc() -> None:
     else:
         hit_dict_fft = {}
         plot_dict_fft = {}
+        fft_data = None
 
     if overwrite is not None:
         for name in kwarg_dict_fft["cut_parameters"]:
@@ -189,7 +140,7 @@ def par_geds_hit_qc() -> None:
     # load data in
     data, threshold_mask = load_data(
         cal_files,
-        args.table_name,
+        table_name,
         {},
         [*cut_fields, "timestamp", "trapTmax", "t_sat_lo"],
         threshold=kwarg_dict_cal.get("threshold", 0),
@@ -198,11 +149,14 @@ def par_geds_hit_qc() -> None:
     )
 
     mask = get_pulser_mask(
-        pulser_file=args.pulser_file,
+        pulser_file=pulser_file,
     )
 
     data["is_pulser"] = mask[threshold_mask]
 
+    msg = f"{len(data.query('~is_pulser'))} non pulser events"
+    log.info(msg)
+
     discharges = data["t_sat_lo"] > 0
     discharge_timestamps = np.where(data["timestamp"][discharges])[0]
     is_recovering = np.full(len(data), False, dtype=bool)
@@ -217,19 +171,22 @@ def par_geds_hit_qc() -> None:
         )
     data["is_recovering"] = is_recovering
 
+    msg = f"{len(data.query('is_recovering'))} discharge recovery events"
+    log.info(msg)
+
     rng = np.random.default_rng()
     mask = np.full(len(data.query("~is_pulser & ~is_recovering")), False, dtype=bool)
     mask[
         rng.choice(len(data.query("~is_pulser & ~is_recovering")), 4000, replace=False)
     ] = True
 
-    if "initial_cal_cuts" in kwarg_dict:
-        init_cal = kwarg_dict["initial_cal_cuts"]
+    if "initial_cal_cuts" in config:
+        init_cal = config["initial_cal_cuts"]
         hit_dict_init_cal, plot_dict_init_cal = generate_cut_classifiers(
             data.query("~is_pulser & ~is_recovering")[mask],
             init_cal["cut_parameters"],
             init_cal.get("rounding", 4),
-            display=1 if args.plot_path else 0,
+            display=build_plots,
         )
         ct_mask = np.full(len(data), True, dtype=bool)
         for outname, info in hit_dict_init_cal.items():
@@ -260,8 +217,8 @@ def par_geds_hit_qc() -> None:
     hit_dict_cal, plot_dict_cal = generate_cut_classifiers(
         cal_data,
         kwarg_dict_cal["cut_parameters"],
-        kwarg_dict.get("rounding", 4),
-        display=1 if args.plot_path else 0,
+        config.get("rounding", 4),
+        display=build_plots,
     )
 
     if overwrite is not None:
@@ -273,14 +230,12 @@ def par_geds_hit_qc() -> None:
     hit_dict = {**hit_dict_fft, **hit_dict_init_cal, **hit_dict_cal}
     plot_dict = {**plot_dict_fft, **plot_dict_init_cal, **plot_dict_cal}
 
-    hit_dict = convert_dict_np_to_float(hit_dict)
-
     for outname, info in hit_dict.items():
         # convert to pandas eval
         exp = info["expression"]
         for key in info.get("parameters", None):
             exp = re.sub(f"(?<![a-zA-Z0-9]){key}(?![a-zA-Z0-9])", f"@{key}", exp)
-        if outname not in fft_data:
+        if fft_data is not None and outname not in fft_data:
             fft_data[outname] = fft_data.eval(
                 exp, local_dict=info.get("parameters", None)
             )
@@ -297,28 +252,120 @@ def par_geds_hit_qc() -> None:
                 ((sf_cal) * (1 - sf_cal))
                 / len(data.query("~is_pulser & ~is_recovering"))
             )
-            sf_fft = len(fft_data.query(f"{entry} & ~is_recovering")) / len(
-                fft_data.query("~is_recovering")
-            )
-            sf_fft_err = 100 * np.sqrt(
-                ((sf_fft) * (1 - sf_fft)) / len(fft_data.query("~is_recovering"))
-            )
             sf_cal *= 100
-            sf_fft *= 100
-            msg = f"{entry} cut applied: {sf_cal:.2f}% of events passed the cut for cal data, {sf_fft:.2f}% for fft data"
+            msg = f"{entry} cut applied: {sf_cal:.2f}% of events passed the cut for cal data"
             log.info(msg)
+
             qc_results[entry] = {
                 "sf_cal": sf_cal,
                 "sf_cal_err": sf_cal_err,
-                "sf_fft": sf_fft,
-                "sf_fft_err": sf_fft_err,
             }
-    qc_results = convert_dict_np_to_float(qc_results)
 
-    Path(args.save_path).parent.mkdir(parents=True, exist_ok=True)
-    Props.write_to(
-        args.save_path, {"operations": hit_dict, "results": {"qc": qc_results}}
+            if fft_data is not None:
+                sf_fft = len(fft_data.query(f"{entry} & ~is_recovering")) / len(
+                    fft_data.query("~is_recovering")
+                )
+                sf_fft_err = 100 * np.sqrt(
+                    ((sf_fft) * (1 - sf_fft)) / len(fft_data.query("~is_recovering"))
+                )
+                sf_fft *= 100
+                msg = f"{entry} cut applied: {sf_fft:.2f}% of events passed the cut for fft data"
+
+                log.info(msg)
+
+                qc_results[entry].update(
+                    {
+                        "sf_fft": sf_fft,
+                        "sf_fft_err": sf_fft_err,
+                    }
+                )
+
+    out_dict = convert_dict_np_to_float(
+        {"operations": hit_dict, "results": {"qc": qc_results}}
+    )
+
+    return out_dict, plot_dict
+
+
+def par_geds_hit_qc() -> None:
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--cal-files", help="cal_files", nargs="*", type=str)
+    argparser.add_argument("--fft-files", help="fft_files", nargs="*", type=str)
+
+    argparser.add_argument(
+        "--tcm-filelist", help="tcm_filelist", type=str, required=False
+    )
+    argparser.add_argument(
+        "--pulser-file", help="pulser_file", type=str, required=False
+    )
+    argparser.add_argument(
+        "--overwrite-files",
+        help="overwrite_files",
+        type=str,
+        required=False,
+        nargs="*",
+    )
+
+    argparser.add_argument("--configs", help="config", type=str, required=True)
+    argparser.add_argument("--log", help="log_file", type=str)
+
+    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
+    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
+    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument("--table-name", help="table name", type=str, required=True)
+    argparser.add_argument("--tier", help="tier", type=str, default="hit")
+
+    argparser.add_argument("--plot-path", help="plot_path", type=str, required=False)
+    argparser.add_argument("--save-path", help="save_path", type=str)
+    args = argparser.parse_args()
+
+    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
+    config_dict = configs["snakemake_rules"]["pars_hit_qc"]
+
+    build_log(config_dict, args.log)
+
+    # get metadata dictionary
+    channel_dict = config_dict["inputs"]["qc_config"][args.channel]
+    kwarg_dict = Props.read_from(channel_dict)
+
+    if args.overwrite_files:
+        overwrite = Props.read_from(args.overwrite_files)
+        if args.channel in overwrite:
+            overwrite = overwrite[args.channel]["pars"]["operations"]
+        else:
+            overwrite = None
+    else:
+        overwrite = None
+
+    if len(args.fft_files) == 1 and Path(args.fft_files[0]).suffix == ".filelist":
+        with Path(args.fft_files[0]).open() as f:
+            fft_files = f.read().splitlines()
+    else:
+        fft_files = args.fft_files
+
+    if len(args.cal_files) == 1 and Path(args.cal_files[0]).suffix == ".filelist":
+        with Path(args.cal_files[0]).open() as f:
+            cal_files = f.read().splitlines()
+    else:
+        cal_files = args.fft_files
+
+    start = time.time()
+    log.info("starting qc")
+
+    out_dict, plot_dict = build_qc(
+        config=kwarg_dict,
+        cal_files=cal_files,
+        fft_files=fft_files,
+        table_name=args.table_name,
+        overwrite=overwrite,
+        pulser_file=args.pulser_file,
+        build_plots=int(bool(args.plot_path)),
     )
+    msg = f"qc took {time.time() - start:.2f} seconds"
+    log.info(msg)
+
+    Path(args.save_path).parent.mkdir(parents=True, exist_ok=True)
+    Props.write_to(args.save_path, out_dict)
 
     if args.plot_path:
         Path(args.plot_path).parent.mkdir(parents=True, exist_ok=True)

legenddataflowscripts/utils/__init__.py

@@ -4,12 +4,14 @@ from .alias_table import alias_table
 from .cfgtools import get_channel_config
 from .convert_np import convert_dict_np_to_float
 from .log import build_log
+from .plot_dict import fill_plot_dict
 from .pulser_removal import get_pulser_mask
 
 __all__ = [
     "alias_table",
     "build_log",
     "convert_dict_np_to_float",
+    "fill_plot_dict",
     "get_channel_config",
     "get_pulser_mask",
 ]

legenddataflowscripts/utils/plot_dict.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+def fill_plot_dict(plot_class, data, plot_options, plot_dict=None):
+    if plot_dict is None:
+        plot_dict = {}
+    else:
+        for key, item in plot_options.items():
+            if item["options"] is not None:
+                plot_dict[key] = item["function"](plot_class, data, **item["options"])
+            else:
+                plot_dict[key] = item["function"](plot_class, data)
+    return plot_dict

legenddataflowscripts/workflow/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from .execenv import execenv_prefix, execenv_pyexe
+from .pre_compile_catalog import pre_compile_catalog
 from .utils import (
     as_ro,
     set_last_rule_name,
@@ -13,6 +14,7 @@ __all__ = [
     "as_ro",
     "execenv_prefix",
     "execenv_pyexe",
+    "pre_compile_catalog",
     "set_last_rule_name",
     "subst_vars",
     "subst_vars_impl",