legend-dataflow-scripts 0.1.8__tar.gz → 0.2.1__tar.gz

{legend_dataflow_scripts-0.1.8/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.1.8
+Version: 0.2.1
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.1.8
+Version: 0.2.1
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.8'
-__version_tuple__ = version_tuple = (0, 1, 8)
+__version__ = version = '0.2.1'
+__version_tuple__ = version_tuple = (0, 2, 1)
 
-__commit_id__ = commit_id = 'g2c31549ca'
+__commit_id__ = commit_id = 'g68aa434fe'

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/dplms.py

@@ -7,7 +7,6 @@ from pathlib import Path
 
 import numpy as np
 import pygama.math.distributions as pmd  # noqa: F401
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo import Array, Table, lh5
 from pygama.pargen.dplms_ge_dict import dplms_ge_dict
@@ -23,11 +22,21 @@ def par_geds_dsp_dplms() -> None:
     argparser.add_argument("--database", help="database", type=str, required=True)
 
     argparser.add_argument("--log", help="log_file", type=str)
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
+    argparser.add_argument(
+        "--processing-chain",
+        help="Processing chain config",
+        type=str,
+        nargs="*",
+        required=True,
+    )
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -38,18 +47,13 @@ def par_geds_dsp_dplms() -> None:
 
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_dplms"]
-
-    log = build_log(config_dict, args.log)
-
-    configs = TextDB(args.configs).on(args.timestamp, system=args.datatype)
-    dsp_config = config_dict["inputs"]["proc_chain"][args.channel]
+    dsp_config = Props.read_from(args.processing_chain)
+    log = build_log(args.log_config, args.log)
 
-    dplms_json = config_dict["inputs"]["dplms_pars"][args.channel]
-    dplms_dict = Props.read_from(dplms_json)
+    t0 = time.time()
 
-    db_dict = Props.read_from(args.database)
+    dplms_dict = Props.read_from(args.config_file)
+    db_dict = Props.read_from(args.decay_const)
 
     if dplms_dict["run_dplms"] is True:
         with Path(args.fft_raw_filelist).open() as f:

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/eopt.py

@@ -9,7 +9,6 @@ from pathlib import Path
 import numpy as np
 import pygama.pargen.energy_optimisation as om  # noqa: F401
 import sklearn.gaussian_process.kernels as ker
-from dbetto import TextDB
 from dbetto.catalog import Props
 from dspeed.units import unit_registry as ureg
 from lgdo import lh5
@@ -37,11 +36,21 @@ def par_geds_dsp_eopt() -> None:
     argparser.add_argument("--inplots", help="in_plot_path", type=str)
 
     argparser.add_argument("--log", help="log_file", type=str)
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument(
+        "--processing-chain",
+        help="Processing chain config",
+        type=str,
+        nargs="*",
+        required=True,
+    )
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -57,17 +66,12 @@ def par_geds_dsp_eopt() -> None:
     )
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_eopt"]
-
-    log = build_log(config_dict, args.log)
+    dsp_config = Props.read_from(args.processing_chain)
+    log = build_log(args.log_config, args.log)
 
     t0 = time.time()
 
-    dsp_config = config_dict["inputs"]["processing_chain"][args.channel]
-    opt_json = config_dict["inputs"]["optimiser_config"][args.channel]
-
-    opt_dict = Props.read_from(opt_json)
+    opt_dict = Props.read_from(args.config_file)
     db_dict = Props.read_from(args.decay_const)
 
     if opt_dict.pop("run_eopt") is True:
@@ -121,9 +125,6 @@ def par_geds_dsp_eopt() -> None:
         msg = f"Data Loaded in {(t1 - t0) / 60} minutes"
         log.info(msg)
 
-        if isinstance(dsp_config, str | list):
-            dsp_config = Props.read_from(dsp_config)
-
         dsp_config["outputs"] = ["tp_99", "tp_0_est", "dt_eff"]
 
         init_data = run_one_dsp(tb_data, dsp_config, db_dict=db_dict, verbosity=0)

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/evtsel.py

@@ -11,7 +11,6 @@ import lgdo
 import numpy as np
 import pygama.math.histogram as pgh
 import pygama.pargen.energy_cal as pgc
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo import lh5
 from pygama.pargen.data_cleaning import generate_cuts, get_keys
@@ -83,7 +82,7 @@ def get_out_data(
 
 def par_geds_dsp_evtsel() -> None:
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str)
+    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str, nargs="*")
     argparser.add_argument(
         "--pulser-file", help="pulser-file", type=str, required=False
     )
@@ -97,15 +96,25 @@ def par_geds_dsp_evtsel() -> None:
         help="raw calibration curve file(s)",
         type=str,
         nargs="*",
-        required=True,
+        required=False,
     )
 
     argparser.add_argument("--log", help="log_file", type=str)
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument(
+        "--processing-chain",
+        help="Processing chain config",
+        type=str,
+        nargs="*",
+        required=True,
+    )
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -113,18 +122,14 @@ def par_geds_dsp_evtsel() -> None:
     argparser.add_argument("--peak-file", help="peak_file", type=str, required=True)
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_peak_selection"]
-
-    log = build_log(config_dict, args.log)
+    log = build_log(args.log_config, args.log)
 
     sto = lh5.LH5Store()
     t0 = time.time()
 
-    dsp_config = config_dict["inputs"]["processing_chain"][args.channel]
-    peak_json = config_dict["inputs"]["peak_config"][args.channel]
+    dsp_config = Props.read_from(args.processing_chain)
 
-    peak_dict = Props.read_from(peak_json)
+    peak_dict = Props.read_from(args.config_file)
     db_dict = Props.read_from(args.decay_const)
 
     Path(args.peak_file).parent.mkdir(parents=True, exist_ok=True)
@@ -134,13 +139,17 @@ def par_geds_dsp_evtsel() -> None:
     if peak_dict.pop("run_selection") is True:
         log.debug("Starting peak selection")
 
-        with Path(args.raw_filelist).open() as f:
-            files = f.read().splitlines()
-        raw_files = sorted(files)
+        if (
+            isinstance(args.raw_filelist, list)
+            and args.raw_filelist[0].split(".")[-1] == "filelist"
+        ):
+            files = args.raw_filelist[0]
+            with Path(files).open() as f:
+                files = f.read().splitlines()
+        else:
+            files = args.raw_filelist
 
-        raw_dict = Props.read_from(args.raw_cal_curve)[args.channel]["pars"][
-            "operations"
-        ]
+        raw_files = sorted(files)
 
         peaks_kev = peak_dict["peaks"]
         kev_widths = peak_dict["kev_widths"]
@@ -181,6 +190,34 @@ def par_geds_dsp_evtsel() -> None:
                 False,
             )
 
+        if args.raw_cal_curve:
+            raw_dict = Props.read_from(args.raw_cal_curve)[args.channel]["pars"][
+                "operations"
+            ]
+        else:
+            E_uncal = tb.daqenergy.nda
+            E_uncal = E_uncal[E_uncal > 200]
+            guess_keV = 2620 / np.nanpercentile(E_uncal, 99)  # usual simple guess
+
+            # daqenergy is an int so use integer binning (dx used to be bugged as output so switched to nbins)
+
+            hpge_cal = pgc.HPGeCalibration(
+                "daqenergy",
+                peaks_kev,
+                guess_keV,
+                0,
+                uncal_is_int=True,
+            )
+
+            hpge_cal.hpge_find_energy_peaks(E_uncal, etol_kev=5)
+            roughpars = hpge_cal.pars
+            raw_dict = {
+                "daqenergy_cal": {
+                    "expression": "daqenergy*a",
+                    "parameters": {"a": round(float(roughpars[1]), 5)},
+                }
+            }
+
         for outname, info in raw_dict.items():
             outcol = tb.eval(info["expression"], info.get("parameters", None))
             tb.add_column(outname, outcol)

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/nopt.py

@@ -7,7 +7,6 @@ from pathlib import Path
 
 import numpy as np
 import pygama.pargen.noise_optimization as pno
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo import lh5
 from pygama.pargen.data_cleaning import generate_cuts, get_cut_indexes
@@ -22,12 +21,22 @@ def par_geds_dsp_nopt() -> None:
     argparser.add_argument("--database", help="database", type=str, required=True)
     argparser.add_argument("--inplots", help="inplots", type=str)
 
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
     argparser.add_argument("--log", help="log_file", type=str)
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument(
+        "--processing-chain",
+        help="Processing chain config",
+        type=str,
+        nargs="*",
+        required=True,
+    )
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -37,18 +46,13 @@ def par_geds_dsp_nopt() -> None:
 
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_nopt"]
-
-    log = build_log(config_dict, args.log)
+    dsp_config = Props.read_from(args.processing_chain)
+    log = build_log(args.log_config, args.log)
 
     t0 = time.time()
 
-    dsp_config = config_dict["inputs"]["processing_chain"][args.channel]
-    opt_json = config_dict["inputs"]["optimiser_config"][args.channel]
-
-    opt_dict = Props.read_from(opt_json)
-    db_dict = Props.read_from(args.database)
+    opt_dict = Props.read_from(args.config_file)
+    db_dict = Props.read_from(args.decay_const)
 
     if opt_dict.pop("run_nopt") is True:
         with Path(args.raw_filelist).open() as f:
@@ -81,9 +85,6 @@ def par_geds_dsp_nopt() -> None:
         msg = f"... {len(tb_data)} baselines after cuts"
         log.info(msg)
 
-        if isinstance(dsp_config, str | list):
-            dsp_config = Props.read_from(dsp_config)
-
         if args.plot_path:
             out_dict, plot_dict = pno.noise_optimization(
                 tb_data,

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/pz.py

@@ -6,7 +6,6 @@ import pickle as pkl
 from pathlib import Path
 
 import numpy as np
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo import lh5
 from pygama.pargen.data_cleaning import get_cut_indexes
@@ -28,9 +27,20 @@ def par_geds_dsp_pz() -> None:
         "-p", "--no-pulse", help="no pulser present", action="store_true"
     )
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument(
+        "--processing-chain",
+        help="Processing chain config",
+        type=str,
+        nargs="*",
+        required=True,
+    )
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -46,19 +56,13 @@ def par_geds_dsp_pz() -> None:
     argparser.add_argument("--pz-files", help="input files", nargs="*", type=str)
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_tau"]
-
-    log = build_log(config_dict, args.log)
-
-    channel_dict = config_dict["inputs"]["processing_chain"][args.channel]
-    kwarg_dict = config_dict["inputs"]["tau_config"][args.channel]
-
-    kwarg_dict = Props.read_from(kwarg_dict)
+    log = build_log(args.log_config, args.log)
+    kwarg_dict = Props.read_from(args.config_file)
 
     if kwarg_dict["run_tau"] is True:
-        dsp_config = Props.read_from(channel_dict)
+        dsp_config = Props.read_from(args.processing_chain)
         kwarg_dict.pop("run_tau")
+        input_file = []
         if args.pz_files is not None and len(args.pz_files) > 0:
             if (
                 isinstance(args.pz_files, list)
@@ -156,7 +160,7 @@ def par_geds_dsp_pz() -> None:
             tau.get_single_decay_constant(
                 tb_data, kwarg_dict.get("slope_param", "tail_slope")
             )
-            msg = f"Found tau: {tau.output_dict['pz']['tau']}+- {tau.output_dict['pz']['tau_err']}"
+            msg = f"Found tau: {tau.output_dict['pz']['tau1']}+- {tau.output_dict['pz']['tau1_err']}"
             log.debug(msg)
         elif kwarg_dict["mode"] == "double":
             tau.get_dpz_decay_constants(

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/dsp/svm_build.py

@@ -4,7 +4,6 @@ import argparse
 import pickle as pkl
 from pathlib import Path
 
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo import lh5
 from sklearn.svm import SVC
@@ -15,10 +14,10 @@ from ....utils import build_log
 def par_geds_dsp_svm_build() -> None:
     argparser = argparse.ArgumentParser()
     argparser.add_argument("--log", help="log file", type=str)
-    argparser.add_argument("--configs", help="config file", type=str)
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
 
     argparser.add_argument(
         "--output-file", help="output SVM file", type=str, required=True
@@ -31,10 +30,7 @@ def par_geds_dsp_svm_build() -> None:
     )
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_dsp_build_svm"]
-
-    log = build_log(config_dict, args.log)
+    log = build_log(args.log_config, args.log)
 
     if args.train_data is not None and len(args.train_data) > 0:
         # Load files

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/hit/aoe.py

@@ -10,7 +10,6 @@ import warnings
 from pathlib import Path
 
 import numpy as np
-from dbetto import TextDB
 from dbetto.catalog import Props
 from pygama.pargen.AoE_cal import *  # noqa: F403
 from pygama.pargen.AoE_cal import CalAoE
@@ -195,12 +194,23 @@ def par_geds_hit_aoe() -> None:
     argparser.add_argument("--eres-file", help="eres_file", type=str, required=True)
     argparser.add_argument("--inplots", help="in_plot_path", type=str, required=False)
 
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
+    argparser.add_argument(
+        "--timestamp",
+        help="timestamp",
+        type=str,
+        required=False,
+        default="20000101T000000Z",
+    )
+
     argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
     argparser.add_argument("--table-name", help="table name", type=str, required=True)
 
     argparser.add_argument("--plot-file", help="plot_file", type=str, required=False)
@@ -210,13 +220,8 @@ def par_geds_hit_aoe() -> None:
     argparser.add_argument("-d", "--debug", help="debug_mode", action="store_true")
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_hit_aoecal"]
-
-    build_log(config_dict, args.log)
-
-    channel_dict = config_dict["inputs"]["aoecal_config"][args.channel]
-    kwarg_dict = Props.read_from(channel_dict)
+    log = build_log(args.log_config, args.log)
+    kwarg_dict = Props.read_from(args.config_file)
 
     ecal_dict = Props.read_from(args.ecal_file)
     cal_dict = ecal_dict["pars"]
@@ -298,19 +303,10 @@ def par_geds_hit_aoe() -> None:
         aoe = None
         plot_dict = out_plot_dict
         results_dict = {}
-
     if args.plot_file:
-        common_dict = plot_dict.pop("common") if "common" in list(plot_dict) else None
-        out_plot_dict.update({"aoe": plot_dict})
-
-        if "common" in list(out_plot_dict) and common_dict is not None:
-            out_plot_dict["common"].update(common_dict)
-        elif common_dict is not None:
-            out_plot_dict["common"] = common_dict
-
         Path(args.plot_file).parent.mkdir(parents=True, exist_ok=True)
         with Path(args.plot_file).open("wb") as w:
-            pkl.dump(out_plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
+            pkl.dump(plot_dict, w, protocol=pkl.HIGHEST_PROTOCOL)
 
     Path(args.hit_pars).parent.mkdir(parents=True, exist_ok=True)
     final_hit_dict = {

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/hit/ecal.py

@@ -12,9 +12,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pygama.math.distributions as pgf
 import pygama.math.histogram as pgh
-from dbetto import TextDB
 from dbetto.catalog import Props
-from legendmeta import LegendMetadata
 from lgdo import lh5
 from matplotlib.colors import LogNorm
 from pygama.math.distributions import nb_poly
@@ -460,16 +458,21 @@ def par_geds_hit_ecal() -> None:
     argparser.add_argument("--in-hit-dict", help="in_hit_dict", required=False)
     argparser.add_argument("--inplot-dict", help="inplot_dict", required=False)
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
-    argparser.add_argument("--table-name", help="table name", type=str, required=True)
+    argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
 
-    argparser.add_argument("--tier", help="tier", type=str, default="hit")
-    argparser.add_argument("--configs", help="config", type=str, required=True)
-    argparser.add_argument("--metadata", help="metadata path", type=str, required=True)
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
 
-    argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument(
+        "--det-status", help="detector status", type=str, default="on", required=False
+    )
+
+    argparser.add_argument("--table-name", help="table name", type=str, required=True)
+    argparser.add_argument("--channel", help="channel", type=str)
 
     argparser.add_argument("--plot-path", help="plot_path", type=str, required=False)
     argparser.add_argument("--save-path", help="save_path", type=str)
@@ -478,22 +481,7 @@ def par_geds_hit_ecal() -> None:
     argparser.add_argument("-d", "--debug", help="debug_mode", action="store_true")
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]
-    if args.tier == "hit":
-        config_dict = config_dict["pars_hit_ecal"]
-    elif args.tier == "pht":
-        config_dict = config_dict["pars_pht_ecal"]
-    else:
-        msg = "invalid tier"
-        raise ValueError(msg)
-
-    log = build_log(config_dict, args.log)
-
-    chmap = LegendMetadata(args.metadata).channelmap(
-        args.timestamp, system=args.datatype
-    )
-    det_status = chmap[args.channel]["analysis"]["usability"]
+    log = build_log(args.log_config, args.log)
 
     if args.in_hit_dict:
         hit_dict = Props.read_from(args.in_hit_dict)
@@ -508,10 +496,12 @@ def par_geds_hit_ecal() -> None:
 
     database_dic = Props.read_from(db_files)
 
-    hit_dict.update(database_dic[args.channel]["ctc_params"])
+    if args.channel and args.channel in database_dic:
+        database_dic = database_dic[args.channel]
 
-    channel_dict = config_dict["inputs"]["ecal_config"][args.channel]
-    kwarg_dict = Props.read_from(channel_dict)
+    hit_dict.update(database_dic["ctc_params"])
+
+    kwarg_dict = Props.read_from(args.config_file)
 
     # convert plot functions from strings to functions and split off baseline and common plots
     for field, item in kwarg_dict["plot_options"].items():
@@ -605,7 +595,7 @@ def par_geds_hit_ecal() -> None:
             debug_mode=kwarg_dict.get("debug_mode", False) | args.debug,
         )
         full_object_dict[cal_energy_param].hpge_get_energy_peaks(
-            e_uncal, etol_kev=5 if det_status == "on" else 20
+            e_uncal, etol_kev=5 if args.det_status == "on" else 20
         )
         if 2614.511 not in full_object_dict[cal_energy_param].peaks_kev:
             full_object_dict[cal_energy_param] = HPGeCalibration(
@@ -616,10 +606,10 @@ def par_geds_hit_ecal() -> None:
                 debug_mode=kwarg_dict.get("debug_mode", False),
             )
             full_object_dict[cal_energy_param].hpge_get_energy_peaks(
-                e_uncal, etol_kev=5 if det_status == "on" else 30, n_sigma=2
+                e_uncal, etol_kev=5 if args.det_status == "on" else 30, n_sigma=2
             )
         got_peaks_kev = full_object_dict[cal_energy_param].peaks_kev.copy()
-        if det_status != "on":
+        if args.det_status != "on":
             full_object_dict[cal_energy_param].hpge_cal_energy_peak_tops(
                 e_uncal,
                 peaks_kev=got_peaks_kev,
@@ -633,7 +623,7 @@ def par_geds_hit_ecal() -> None:
             tail_weight=kwarg_dict.get("tail_weight", 0),
             n_events=kwarg_dict.get("n_events", None),
             allowed_p_val=kwarg_dict.get("p_val", 0),
-            update_cal_pars=bool(det_status == "on"),
+            update_cal_pars=bool(args.det_status == "on"),
             bin_width_kev=0.5,
         )
         full_object_dict[cal_energy_param].hpge_fit_energy_peaks(
@@ -680,31 +670,6 @@ def par_geds_hit_ecal() -> None:
         hit_dict.update(
             {cal_energy_param: full_object_dict[cal_energy_param].gen_pars_dict()}
         )
-        if "copy_calibration" in kwarg_dict:
-            for copy_cal_param, _copy_to_cal_param in kwarg_dict[
-                "copy_calibration"
-            ].items():
-                if copy_cal_param not in full_object_dict:
-                    msg = f"copy_calibration parameter {copy_cal_param} not found in full_object_dict"
-                    raise ValueError(msg)
-                if isinstance(_copy_to_cal_param, str):
-                    copy_to_cal_param = [_copy_to_cal_param]
-                else:
-                    copy_to_cal_param = _copy_to_cal_param
-                for cal_par in copy_to_cal_param:
-                    if cal_par in full_object_dict:
-                        msg = f"copy_calibration parameter {cal_par} already exists in full_object_dict"
-                        raise ValueError(msg)
-                    copy_dict = {cal_par: full_object_dict[cal_par].gen_pars_dict()}
-                    copy_dict["expression"] = copy_dict[cal_par]["expression"].replace(
-                        copy_cal_param, cal_par
-                    )
-                    hit_dict.update({cal_par: copy_dict[cal_par]})
-        if "extra_blocks" in kwarg_dict:
-            if isinstance(kwarg_dict["extra_blocks"], dict):
-                kwarg_dict["extra_blocks"] = [kwarg_dict["extra_blocks"]]
-            for extra_block in kwarg_dict["extra_blocks"]:
-                hit_dict.update(extra_block)
 
         if args.plot_path:
             param_plot_dict = {}
@@ -753,7 +718,7 @@ def par_geds_hit_ecal() -> None:
             peak_dict["parameters"] = peak_dict["parameters"].to_dict()
             peak_dict["uncertainties"] = peak_dict["uncertainties"].to_dict()
 
-        if det_status != "on":
+        if args.det_status != "on":
             for peak_dict in (
                 full_object_dict[cal_energy_param]
                 .results["hpge_cal_energy_peak_tops"]["peak_parameters"]
@@ -763,6 +728,29 @@ def par_geds_hit_ecal() -> None:
                 peak_dict["parameters"] = peak_dict["parameters"].to_dict()
                 peak_dict["uncertainties"] = peak_dict["uncertainties"].to_dict()
 
+    if "copy_calibration" in kwarg_dict:
+        for cal_par, copy_dict in kwarg_dict["copy_calibration"].items():
+            copy_cal_param = copy_dict["copy_param"]
+            new_input_param = copy_dict["new_input_param"]
+            old_input_param = copy_dict["old_input_param"]
+            if copy_cal_param not in full_object_dict:
+                msg = f"copy_calibration parameter {copy_cal_param} not found in full_object_dict"
+                raise ValueError(msg)
+            if cal_par in full_object_dict:
+                msg = f"copy_calibration parameter {cal_par} already exists in full_object_dict"
+                raise ValueError(msg)
+            new_dict = {cal_par: copy.deepcopy(hit_dict[copy_cal_param])}
+            new_dict[cal_par]["expression"] = new_dict[cal_par]["expression"].replace(
+                old_input_param, new_input_param
+            )
+            hit_dict.update({cal_par: new_dict[cal_par]})
+
+    if "extra_blocks" in kwarg_dict:
+        if isinstance(kwarg_dict["extra_blocks"], dict):
+            kwarg_dict["extra_blocks"] = [kwarg_dict["extra_blocks"]]
+        for extra_block in kwarg_dict["extra_blocks"]:
+            hit_dict.update(extra_block)
+
     if "monitoring_parameters" in kwarg_dict:
         monitor_dict = monitor_parameters(
             files, args.table_name, kwarg_dict["monitoring_parameters"]

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/hit/lq.py

@@ -10,7 +10,6 @@ from pathlib import Path
 
 import numpy as np
 import pandas as pd
-from dbetto import TextDB
 from dbetto.catalog import Props
 from pygama.math.distributions import gaussian
 from pygama.pargen.AoE_cal import *  # noqa: F403
@@ -222,13 +221,23 @@ def par_geds_hit_lq() -> None:
     argparser.add_argument("--eres-file", help="eres_file", type=str, required=True)
     argparser.add_argument("--inplots", help="in_plot_path", type=str, required=False)
 
-    argparser.add_argument("--configs", help="configs", type=str, required=True)
     argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
     argparser.add_argument("--table-name", help="table name", type=str, required=True)
+    argparser.add_argument(
+        "--timestamp",
+        help="timestamp",
+        type=str,
+        required=False,
+        default="20000101T000000Z",
+    )
 
     argparser.add_argument("--plot-file", help="plot_file", type=str, required=False)
     argparser.add_argument("--hit-pars", help="hit_pars", type=str)
@@ -237,13 +246,8 @@ def par_geds_hit_lq() -> None:
     argparser.add_argument("-d", "--debug", help="debug_mode", action="store_true")
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_hit_lqcal"]
-
-    build_log(config_dict, args.log)
-
-    channel_dict = config_dict["inputs"]["lqcal_config"][args.channel]
-    kwarg_dict = Props.read_from(channel_dict)
+    build_log(args.log_config, args.log)
+    kwarg_dict = Props.read_from(args.config_file)
 
     ecal_dict = Props.read_from(args.ecal_file)
     cal_dict = ecal_dict["pars"]["operations"]
@@ -295,7 +299,7 @@ def par_geds_hit_lq() -> None:
 
         data["run_timestamp"] = args.timestamp
 
-        out_dicts, results_dicts, plot_dicts, lq_dict = run_lq_calibration(
+        out_dicts, results_dicts, lq_dict, plot_dicts = run_lq_calibration(
             data,
             cal_dicts={args.timestamp: cal_dict},
             results_dicts={args.timestamp: eres_dict},

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/par/geds/hit/qc.py

@@ -10,7 +10,6 @@ import warnings
 from pathlib import Path
 
 import numpy as np
-from dbetto import TextDB
 from dbetto.catalog import Props
 from lgdo.lh5 import ls
 from pygama.pargen.data_cleaning import (
@@ -305,28 +304,27 @@ def par_geds_hit_qc() -> None:
         required=False,
         nargs="*",
     )
+    argparser.add_argument("--channel", help="channel", type=str)
 
-    argparser.add_argument("--configs", help="config", type=str, required=True)
     argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument(
+        "--log-config", help="Log config file", type=str, required=False, default={}
+    )
+
+    argparser.add_argument(
+        "--config-file", help="Config file", type=str, nargs="*", required=True
+    )
 
-    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
-    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
-    argparser.add_argument("--channel", help="Channel", type=str, required=True)
     argparser.add_argument("--table-name", help="table name", type=str, required=True)
-    argparser.add_argument("--tier", help="tier", type=str, default="hit")
 
     argparser.add_argument("--plot-path", help="plot_path", type=str, required=False)
     argparser.add_argument("--save-path", help="save_path", type=str)
     args = argparser.parse_args()
 
-    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
-    config_dict = configs["snakemake_rules"]["pars_hit_qc"]
-
-    build_log(config_dict, args.log)
+    build_log(args.log_config, args.log)
 
     # get metadata dictionary
-    channel_dict = config_dict["inputs"]["qc_config"][args.channel]
-    kwarg_dict = Props.read_from(channel_dict)
+    kwarg_dict = Props.read_from(args.config_file)
 
     if args.overwrite_files:
         overwrite = Props.read_from(args.overwrite_files)
@@ -347,7 +345,7 @@ def par_geds_hit_qc() -> None:
         with Path(args.cal_files[0]).open() as f:
             cal_files = f.read().splitlines()
     else:
-        cal_files = args.fft_files
+        cal_files = args.cal_files
 
     start = time.time()
     log.info("starting qc")

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/utils/log.py

@@ -39,6 +39,8 @@ def build_log(
     log_file
         The path to the log file.
     """
+    if isinstance(config_dict, str):
+        config_dict = {"options": {"logging": config_dict}}
     if "logging" in config_dict["options"]:
         log_config = config_dict["options"]["logging"]
         log_config = Props.read_from(log_config)

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/utils/plot_dict.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 def fill_plot_dict(plot_class, data, plot_options, plot_dict=None):
     if plot_dict is None:
         plot_dict = {}
-    else:
+    if plot_options is not None:
         for key, item in plot_options.items():
             if item["options"] is not None:
                 plot_dict[key] = item["function"](plot_class, data, **item["options"])

{legend_dataflow_scripts-0.1.8 → legend_dataflow_scripts-0.2.1}/src/legenddataflowscripts/workflow/utils.py

@@ -49,7 +49,8 @@ def subst_vars(
     if use_env:
         combined_var_values = dict(iter(os.environ.items()))
         combined_var_values.update(copy.copy(var_values))
-    subst_vars_impl(props, combined_var_values, ignore_missing)
+
+    return subst_vars_impl(props, combined_var_values, ignore_missing)
 
 
 def subst_vars_in_snakemake_config(workflow, config):
@@ -65,10 +66,11 @@ def subst_vars_in_snakemake_config(workflow, config):
         use_env=True,
         ignore_missing=False,
     )
-    if "system" in config:
-        config["execenv"] = config["execenv"][config["system"]]
-    else:
-        config["execenv"] = config["execenv"]["bare"]
+    if "execenv" in config:
+        if "system" in config:
+            config["execenv"] = config["execenv"][config["system"]]
+        else:
+            config["execenv"] = config["execenv"]["bare"]
 
 
 def set_last_rule_name(workflow, new_name):