legend-dataflow-scripts 0.2.0__tar.gz → 0.2.2__tar.gz

{legend_dataflow_scripts-0.2.0/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -23,7 +23,6 @@ Requires-Dist: pygama>=2.1
 Requires-Dist: dspeed>=1.6
 Requires-Dist: pylegendmeta>=1.2.5
 Requires-Dist: legend-pydataobj>=1.11
-Requires-Dist: legend-daq2lh5>=1.6.1
 Requires-Dist: pip
 Provides-Extra: test
 Requires-Dist: legend-dataflow-scripts; extra == "test"

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/pyproject.toml

@@ -56,7 +56,6 @@ dependencies = [
     "dspeed>=1.6",
     "pylegendmeta>=1.2.5",
     "legend-pydataobj>=1.11",
-    "legend-daq2lh5>=1.6.1",
     "pip",
 ]
 # "legend-daq2lh5 @  file:///${PROJECT_ROOT}/software/python/src/legend-daq2lh5

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -23,7 +23,6 @@ Requires-Dist: pygama>=2.1
 Requires-Dist: dspeed>=1.6
 Requires-Dist: pylegendmeta>=1.2.5
 Requires-Dist: legend-pydataobj>=1.11
-Requires-Dist: legend-daq2lh5>=1.6.1
 Requires-Dist: pip
 Provides-Extra: test
 Requires-Dist: legend-dataflow-scripts; extra == "test"

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legend_dataflow_scripts.egg-info/requires.txt

@@ -4,7 +4,6 @@ pygama>=2.1
 dspeed>=1.6
 pylegendmeta>=1.2.5
 legend-pydataobj>=1.11
-legend-daq2lh5>=1.6.1
 pip
 
 [dev]

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)
 
-__commit_id__ = commit_id = 'g39085207f'
+__commit_id__ = commit_id = 'g512d89c35'

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/dsp/dplms.py

@@ -37,6 +37,7 @@ def par_geds_dsp_dplms() -> None:
         "--config-file", help="Config file", type=str, nargs="*", required=True
     )
 
+    argparser.add_argument("--channel", help="channel", type=str, required=True)
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -53,7 +54,7 @@ def par_geds_dsp_dplms() -> None:
     t0 = time.time()
 
     dplms_dict = Props.read_from(args.config_file)
-    db_dict = Props.read_from(args.decay_const)
+    db_dict = Props.read_from(args.database)
 
     if dplms_dict["run_dplms"] is True:
         with Path(args.fft_raw_filelist).open() as f:

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/dsp/eopt.py

@@ -131,10 +131,10 @@ def par_geds_dsp_eopt() -> None:
         full_dt = (init_data["tp_99"].nda - init_data["tp_0_est"].nda)[idx_list[-1]]
         flat_val = np.ceil(1.1 * np.nanpercentile(full_dt, 99) / 100) / 10
 
-        if flat_val < 1.0:
-            flat_val = 1.0
-        elif flat_val > 4:
-            flat_val = 4
+        if flat_val < opt_dict.get("min_flat_value", 1):
+            flat_val = opt_dict.get("min_flat_value", 1)
+        elif flat_val > opt_dict.get("max_flat_value", 4):
+            flat_val = opt_dict.get("max_flat_value", 4)
         flat_val = f"{flat_val}*us"
 
         db_dict["cusp"] = {"flat": flat_val}
@@ -311,7 +311,7 @@ def par_geds_dsp_eopt() -> None:
 
         optimisers = [bopt_cusp, bopt_zac, bopt_trap]
 
-        out_param_dict, out_results_list = run_bayesian_optimisation(
+        out_param_dict, _ = run_bayesian_optimisation(
             tb_data,
             dsp_config,
             [fom],

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/dsp/evtsel.py

@@ -82,7 +82,7 @@ def get_out_data(
 
 def par_geds_dsp_evtsel() -> None:
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str)
+    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str, nargs="*")
     argparser.add_argument(
         "--pulser-file", help="pulser-file", type=str, required=False
     )
@@ -133,14 +133,19 @@ def par_geds_dsp_evtsel() -> None:
     db_dict = Props.read_from(args.decay_const)
 
     Path(args.peak_file).parent.mkdir(parents=True, exist_ok=True)
-    rng = np.random.default_rng()
-    rand_num = f"{rng.integers(0, 99999):05d}"
-    temp_output = f"{args.peak_file}.{rand_num}"
     if peak_dict.pop("run_selection") is True:
         log.debug("Starting peak selection")
 
-        with Path(args.raw_filelist).open() as f:
-            files = f.read().splitlines()
+        if (
+            isinstance(args.raw_filelist, list)
+            and args.raw_filelist[0].split(".")[-1] == "filelist"
+        ):
+            files = args.raw_filelist[0]
+            with Path(files).open() as f:
+                files = f.read().splitlines()
+        else:
+            files = args.raw_filelist
+
         raw_files = sorted(files)
 
         peaks_kev = peak_dict["peaks"]
@@ -158,8 +163,10 @@ def par_geds_dsp_evtsel() -> None:
         if lh5_path[-1] != "/":
             lh5_path += "/"
 
+        energy_field = peak_dict.get("energy_param", "daqenergy")
+
         tb = lh5.read(
-            lh5_path, raw_files, field_mask=["daqenergy", "t_sat_lo", "timestamp"]
+            lh5_path, raw_files, field_mask=[energy_field, "t_sat_lo", "timestamp"]
         )
 
         if args.no_pulse is False:
@@ -187,14 +194,14 @@ def par_geds_dsp_evtsel() -> None:
                 "operations"
             ]
         else:
-            E_uncal = tb.daqenergy.nda
+            E_uncal = tb[energy_field].nda
             E_uncal = E_uncal[E_uncal > 200]
             guess_keV = 2620 / np.nanpercentile(E_uncal, 99)  # usual simple guess
 
             # daqenergy is an int so use integer binning (dx used to be bugged as output so switched to nbins)
 
             hpge_cal = pgc.HPGeCalibration(
-                "daqenergy",
+                energy_field,
                 peaks_kev,
                 guess_keV,
                 0,
@@ -205,7 +212,7 @@ def par_geds_dsp_evtsel() -> None:
             roughpars = hpge_cal.pars
             raw_dict = {
                 "daqenergy_cal": {
-                    "expression": "daqenergy*a",
+                    "expression": f"{energy_field}*a",
                     "parameters": {"a": round(float(roughpars[1]), 5)},
                 }
             }
@@ -377,7 +384,7 @@ def par_geds_dsp_evtsel() -> None:
                             lh5.write(
                                 out_tbl,
                                 name=lh5_path,
-                                lh5_file=temp_output,
+                                lh5_file=args.peak_file,
                                 wo_mode="a",
                             )
                             peak_dict["obj_buf"] = None
@@ -408,7 +415,7 @@ def par_geds_dsp_evtsel() -> None:
                             lh5.write(
                                 out_tbl,
                                 name=lh5_path,
-                                lh5_file=temp_output,
+                                lh5_file=args.peak_file,
                                 wo_mode="a",
                             )
                             peak_dict["obj_buf"] = None
@@ -423,7 +430,6 @@ def par_geds_dsp_evtsel() -> None:
                                 log.debug(msg)
 
     else:
-        Path(temp_output).touch()
+        Path(args.peak_file).touch()
     msg = f"event selection completed in {time.time() - t0} seconds"
     log.debug(msg)
-    Path(temp_output).rename(args.peak_file)

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/dsp/nopt.py

@@ -52,7 +52,7 @@ def par_geds_dsp_nopt() -> None:
     t0 = time.time()
 
     opt_dict = Props.read_from(args.config_file)
-    db_dict = Props.read_from(args.decay_const)
+    db_dict = Props.read_from(args.database)
 
     if opt_dict.pop("run_nopt") is True:
         with Path(args.raw_filelist).open() as f:

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/dsp/pz.py

@@ -21,7 +21,6 @@ from ....utils import (
 
 def par_geds_dsp_pz() -> None:
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("--configs", help="configs path", type=str, required=True)
     argparser.add_argument("--log", help="log file", type=str)
     argparser.add_argument(
         "-p", "--no-pulse", help="no pulser present", action="store_true"

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/par/geds/hit/ecal.py

@@ -71,7 +71,7 @@ def plot_2614_timemap(
             norm=LogNorm(),
         )
 
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )
@@ -125,7 +125,7 @@ def plot_pulser_timemap(
             norm=LogNorm(),
         )
         plt.ylim([mean - n_spread * spread, mean + n_spread * spread])
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )
@@ -264,7 +264,7 @@ def bin_survival_fraction(
         data.query(selection_string)[cal_energy_param],
         bins=np.arange(erange[0], erange[1] + dx, dx),
     )
-    counts_fail, bins_fail, _ = pgh.get_hist(
+    counts_fail, _, _ = pgh.get_hist(
         data.query(f"(~{cut_field})&(~{pulser_field})")[cal_energy_param],
         bins=np.arange(erange[0], erange[1] + dx, dx),
     )
@@ -303,7 +303,7 @@ def plot_baseline_timemap(
         norm=LogNorm(),
     )
 
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/workflow/filedb.py

@@ -42,6 +42,11 @@ def build_filedb() -> None:
         ignore = []
 
     fdb = FileDB(config, scan=False)
+    try:
+        fdb.scan_files([args.scan_path])
+    except Exception as e:
+        msg = f"error when building {args.output} from {args.scan_path}"
+        raise RuntimeError(msg) from e
     fdb.scan_files([args.scan_path])
     fdb.scan_tables_columns(dir_files_conform=True)
 
@@ -86,7 +91,7 @@ def build_filedb() -> None:
         if (
             (loc_timestamps == default).all() or not found
         ) and row.raw_file not in ignore:
-            msg = "something went wrong! no valid first timestamp found. Likely: the file is empty"
+            msg = "something went wrong! no valid first timestamp found. Likely: the file {row.raw_file} is empty"
             raise RuntimeError(msg)
 
         timestamps[i] = np.min(loc_timestamps)
@@ -97,7 +102,7 @@ def build_filedb() -> None:
         if (
             timestamps[i] < 0 or timestamps[i] > 4102444800
         ) and row.raw_file not in ignore:
-            msg = f"something went wrong! timestamp {timestamps[i]} does not make sense"
+            msg = f"something went wrong! timestamp {timestamps[i]} does not make sense in {row.raw_file}"
             raise RuntimeError(msg)
 
     fdb.df["first_timestamp"] = timestamps

{legend_dataflow_scripts-0.2.0 → legend_dataflow_scripts-0.2.2}/src/legenddataflowscripts/workflow/utils.py

@@ -49,7 +49,8 @@ def subst_vars(
     if use_env:
         combined_var_values = dict(iter(os.environ.items()))
         combined_var_values.update(copy.copy(var_values))
-    subst_vars_impl(props, combined_var_values, ignore_missing)
+
+    return subst_vars_impl(props, combined_var_values, ignore_missing)
 
 
 def subst_vars_in_snakemake_config(workflow, config):
@@ -65,10 +66,11 @@ def subst_vars_in_snakemake_config(workflow, config):
         use_env=True,
         ignore_missing=False,
     )
-    if "system" in config:
-        config["execenv"] = config["execenv"][config["system"]]
-    else:
-        config["execenv"] = config["execenv"]["bare"]
+    if "execenv" in config:
+        if "system" in config:
+            config["execenv"] = config["execenv"][config["system"]]
+        else:
+            config["execenv"] = config["execenv"]["bare"]
 
 
 def set_last_rule_name(workflow, new_name):