legend-dataflow-scripts 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

{legend_dataflow_scripts-0.2.0.dist-info → legend_dataflow_scripts-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -23,7 +23,6 @@ Requires-Dist: pygama>=2.1
 Requires-Dist: dspeed>=1.6
 Requires-Dist: pylegendmeta>=1.2.5
 Requires-Dist: legend-pydataobj>=1.11
-Requires-Dist: legend-daq2lh5>=1.6.1
 Requires-Dist: pip
 Provides-Extra: test
 Requires-Dist: legend-dataflow-scripts; extra == "test"

{legend_dataflow_scripts-0.2.0.dist-info → legend_dataflow_scripts-0.2.2.dist-info}/RECORD

@@ -1,18 +1,18 @@
 legenddataflowscripts/__init__.py,sha256=hlpvTxSBjOyXlZUyOyYx3VwT5LS6zNzhAZnTmfT3NjU,303
-legenddataflowscripts/_version.py,sha256=Dg8AmJomLVpjKL6prJylOONZAPRtB86LOce7dorQS_A,704
+legenddataflowscripts/_version.py,sha256=o3ZTescp-19Z9cvBGq9dQnbppljgzdUYUf98Nov0spY,704
 legenddataflowscripts/par/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/par/geds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/par/geds/dsp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-legenddataflowscripts/par/geds/dsp/dplms.py,sha256=GjOPuBuLPtzBEkfmLMSkxSSeBBsY7-wy0rY3jLjAJv0,5298
-legenddataflowscripts/par/geds/dsp/eopt.py,sha256=MP_prgRL4Aqj_iNIpxHks2v4NY6B_iWh3SgcxQFhixk,13506
-legenddataflowscripts/par/geds/dsp/evtsel.py,sha256=E0jJIuqaywIidr4K-PPMFlctZ9oumuSmiX5zKDPHVlA,17144
-legenddataflowscripts/par/geds/dsp/nopt.py,sha256=vHwlHbmBlozVHO1u89FwAx8c_kazTsxMqZGpbyjN-1M,3980
-legenddataflowscripts/par/geds/dsp/pz.py,sha256=epYHSAqz13QLjxfnm38zH1Caaih5efuwNlgCipHD5Do,8148
+legenddataflowscripts/par/geds/dsp/dplms.py,sha256=GIMQo_qiI-ga5zsdf3oDnxjDzw5T6gqVCQob-Mf2Pvw,5376
+legenddataflowscripts/par/geds/dsp/eopt.py,sha256=epnkSddyzI0sXwPyA8gJxoRumGubv3tQpa2J_uRusV8,13615
+legenddataflowscripts/par/geds/dsp/evtsel.py,sha256=tyF5sCovm22w0Rdd9vD8NyzM2ym8AGO8IR9oKQFYAWA,17296
+legenddataflowscripts/par/geds/dsp/nopt.py,sha256=l3Z5RqPM4my6xSTG78zmjQIorCwGLMP-ipTbNOauWFY,3977
+legenddataflowscripts/par/geds/dsp/pz.py,sha256=l9nc3MmSjghmYOR3LBbbGZMnHc52MFy5F7EOnYAPqvc,8062
 legenddataflowscripts/par/geds/dsp/svm.py,sha256=eDneRB_PQZp8Q4n2VheTX3kbu4ufZQ-jnuCCjvtwFpk,826
 legenddataflowscripts/par/geds/dsp/svm_build.py,sha256=w5-vT6rXmpl7V9rdkfc7_g6GTzn86i41tHkIT-3f5YI,1931
 legenddataflowscripts/par/geds/hit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 legenddataflowscripts/par/geds/hit/aoe.py,sha256=jAH0Rh3JCnV67vhv9xUZtPdGB8ADJXgG9Lo8t8YQjqs,10841
-legenddataflowscripts/par/geds/hit/ecal.py,sha256=wg8TGgdYt1Pvh-hpvNX4qliwPvxgSlHBsReQ86rQ3aY,27032
+legenddataflowscripts/par/geds/hit/ecal.py,sha256=j8Z90r9UP5Hn-cawf2lvDL8tF2Uvfp4rDjlM4sGbM-M,27009
 legenddataflowscripts/par/geds/hit/lq.py,sha256=vXgK83RlJJ4UUjQQJWfmFGIbT0AEP3EaLALM5LhvZ6s,11133
 legenddataflowscripts/par/geds/hit/qc.py,sha256=vr6j5sRTvjmDmErW2uUteCPsj5qfBrpn7ssvLKzov4M,12408
 legenddataflowscripts/tier/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,11 +27,11 @@ legenddataflowscripts/utils/plot_dict.py,sha256=6f2ZB8J1GNAGfldQjgl1gkKMDcqPo1W7
 legenddataflowscripts/utils/pulser_removal.py,sha256=kuARdp1jf-lsUWcb0_KRDp-ZXzkHNrDCXUc3h7TJm7Q,424
 legenddataflowscripts/workflow/__init__.py,sha256=JhudKYhBT8bXtX4LCqxQCHzUiITpugAtFxePWEtphC4,474
 legenddataflowscripts/workflow/execenv.py,sha256=qTG4N9ovEPxA0QtqG0wWUIuK50BZIcYvpVlpy-XgxPw,9257
-legenddataflowscripts/workflow/filedb.py,sha256=rbvOcXUxLbHz177QuDIDAL3aysz-bZDjHOiMsRHssZo,3434
+legenddataflowscripts/workflow/filedb.py,sha256=6scz8DjdvbXs8OXOy2-6BJWVSALQzmy5cLLn-OsdXsU,3656
 legenddataflowscripts/workflow/pre_compile_catalog.py,sha256=cEK0KXh-ClSE2Bo9MK471o79XG22bMY5r-2tIihtCfk,790
-legenddataflowscripts/workflow/utils.py,sha256=eKE8KIG2ffynZt9fTbI1SVQV85i3aW9GFGh1Nio1iDo,3118
-legend_dataflow_scripts-0.2.0.dist-info/METADATA,sha256=lTH14CoB_S_IUgAPggbXmI1mN35E0hvEjdK_YGWUpLw,3122
-legend_dataflow_scripts-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-legend_dataflow_scripts-0.2.0.dist-info/entry_points.txt,sha256=B197waSm-orA_ZS-9rkxNDsmOHdCn8CbWodnlqXQKRg,1313
-legend_dataflow_scripts-0.2.0.dist-info/top_level.txt,sha256=s8E2chjJNYUbrN6whFG_VCsJKySFp1IOXLcUefA7DB0,22
-legend_dataflow_scripts-0.2.0.dist-info/RECORD,,
+legenddataflowscripts/workflow/utils.py,sha256=VVCsj7wNaqV6sw2Xnk_xykhVv3BKTX4hqQtKE4UUayg,3170
+legend_dataflow_scripts-0.2.2.dist-info/METADATA,sha256=9n2aNwe2pg_qGG5v3e7ly_yQNWELnCpLRe3PC2ygsEA,3085
+legend_dataflow_scripts-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+legend_dataflow_scripts-0.2.2.dist-info/entry_points.txt,sha256=B197waSm-orA_ZS-9rkxNDsmOHdCn8CbWodnlqXQKRg,1313
+legend_dataflow_scripts-0.2.2.dist-info/top_level.txt,sha256=s8E2chjJNYUbrN6whFG_VCsJKySFp1IOXLcUefA7DB0,22
+legend_dataflow_scripts-0.2.2.dist-info/RECORD,,

legenddataflowscripts/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)
 
 __commit_id__ = commit_id = None

legenddataflowscripts/par/geds/dsp/dplms.py

@@ -37,6 +37,7 @@ def par_geds_dsp_dplms() -> None:
         "--config-file", help="Config file", type=str, nargs="*", required=True
     )
 
+    argparser.add_argument("--channel", help="channel", type=str, required=True)
     argparser.add_argument(
         "--raw-table-name", help="raw table name", type=str, required=True
     )
@@ -53,7 +54,7 @@ def par_geds_dsp_dplms() -> None:
     t0 = time.time()
 
     dplms_dict = Props.read_from(args.config_file)
-    db_dict = Props.read_from(args.decay_const)
+    db_dict = Props.read_from(args.database)
 
     if dplms_dict["run_dplms"] is True:
         with Path(args.fft_raw_filelist).open() as f:

legenddataflowscripts/par/geds/dsp/eopt.py

@@ -131,10 +131,10 @@ def par_geds_dsp_eopt() -> None:
         full_dt = (init_data["tp_99"].nda - init_data["tp_0_est"].nda)[idx_list[-1]]
         flat_val = np.ceil(1.1 * np.nanpercentile(full_dt, 99) / 100) / 10
 
-        if flat_val < 1.0:
-            flat_val = 1.0
-        elif flat_val > 4:
-            flat_val = 4
+        if flat_val < opt_dict.get("min_flat_value", 1):
+            flat_val = opt_dict.get("min_flat_value", 1)
+        elif flat_val > opt_dict.get("max_flat_value", 4):
+            flat_val = opt_dict.get("max_flat_value", 4)
         flat_val = f"{flat_val}*us"
 
         db_dict["cusp"] = {"flat": flat_val}
@@ -311,7 +311,7 @@ def par_geds_dsp_eopt() -> None:
 
         optimisers = [bopt_cusp, bopt_zac, bopt_trap]
 
-        out_param_dict, out_results_list = run_bayesian_optimisation(
+        out_param_dict, _ = run_bayesian_optimisation(
             tb_data,
             dsp_config,
             [fom],

legenddataflowscripts/par/geds/dsp/evtsel.py

@@ -82,7 +82,7 @@ def get_out_data(
 
 def par_geds_dsp_evtsel() -> None:
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str)
+    argparser.add_argument("--raw-filelist", help="raw_filelist", type=str, nargs="*")
     argparser.add_argument(
         "--pulser-file", help="pulser-file", type=str, required=False
     )
@@ -133,14 +133,19 @@ def par_geds_dsp_evtsel() -> None:
     db_dict = Props.read_from(args.decay_const)
 
     Path(args.peak_file).parent.mkdir(parents=True, exist_ok=True)
-    rng = np.random.default_rng()
-    rand_num = f"{rng.integers(0, 99999):05d}"
-    temp_output = f"{args.peak_file}.{rand_num}"
     if peak_dict.pop("run_selection") is True:
         log.debug("Starting peak selection")
 
-        with Path(args.raw_filelist).open() as f:
-            files = f.read().splitlines()
+        if (
+            isinstance(args.raw_filelist, list)
+            and args.raw_filelist[0].split(".")[-1] == "filelist"
+        ):
+            files = args.raw_filelist[0]
+            with Path(files).open() as f:
+                files = f.read().splitlines()
+        else:
+            files = args.raw_filelist
+
         raw_files = sorted(files)
 
         peaks_kev = peak_dict["peaks"]
@@ -158,8 +163,10 @@ def par_geds_dsp_evtsel() -> None:
         if lh5_path[-1] != "/":
             lh5_path += "/"
 
+        energy_field = peak_dict.get("energy_param", "daqenergy")
+
         tb = lh5.read(
-            lh5_path, raw_files, field_mask=["daqenergy", "t_sat_lo", "timestamp"]
+            lh5_path, raw_files, field_mask=[energy_field, "t_sat_lo", "timestamp"]
         )
 
         if args.no_pulse is False:
@@ -187,14 +194,14 @@ def par_geds_dsp_evtsel() -> None:
                 "operations"
             ]
         else:
-            E_uncal = tb.daqenergy.nda
+            E_uncal = tb[energy_field].nda
             E_uncal = E_uncal[E_uncal > 200]
             guess_keV = 2620 / np.nanpercentile(E_uncal, 99)  # usual simple guess
 
             # daqenergy is an int so use integer binning (dx used to be bugged as output so switched to nbins)
 
             hpge_cal = pgc.HPGeCalibration(
-                "daqenergy",
+                energy_field,
                 peaks_kev,
                 guess_keV,
                 0,
@@ -205,7 +212,7 @@ def par_geds_dsp_evtsel() -> None:
             roughpars = hpge_cal.pars
             raw_dict = {
                 "daqenergy_cal": {
-                    "expression": "daqenergy*a",
+                    "expression": f"{energy_field}*a",
                     "parameters": {"a": round(float(roughpars[1]), 5)},
                 }
             }
@@ -377,7 +384,7 @@ def par_geds_dsp_evtsel() -> None:
                             lh5.write(
                                 out_tbl,
                                 name=lh5_path,
-                                lh5_file=temp_output,
+                                lh5_file=args.peak_file,
                                 wo_mode="a",
                             )
                             peak_dict["obj_buf"] = None
@@ -408,7 +415,7 @@ def par_geds_dsp_evtsel() -> None:
                             lh5.write(
                                 out_tbl,
                                 name=lh5_path,
-                                lh5_file=temp_output,
+                                lh5_file=args.peak_file,
                                 wo_mode="a",
                             )
                             peak_dict["obj_buf"] = None
@@ -423,7 +430,6 @@ def par_geds_dsp_evtsel() -> None:
                                 log.debug(msg)
 
     else:
-        Path(temp_output).touch()
+        Path(args.peak_file).touch()
     msg = f"event selection completed in {time.time() - t0} seconds"
     log.debug(msg)
-    Path(temp_output).rename(args.peak_file)

legenddataflowscripts/par/geds/dsp/nopt.py

@@ -52,7 +52,7 @@ def par_geds_dsp_nopt() -> None:
     t0 = time.time()
 
     opt_dict = Props.read_from(args.config_file)
-    db_dict = Props.read_from(args.decay_const)
+    db_dict = Props.read_from(args.database)
 
     if opt_dict.pop("run_nopt") is True:
         with Path(args.raw_filelist).open() as f:

legenddataflowscripts/par/geds/dsp/pz.py

@@ -21,7 +21,6 @@ from ....utils import (
 
 def par_geds_dsp_pz() -> None:
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("--configs", help="configs path", type=str, required=True)
     argparser.add_argument("--log", help="log file", type=str)
     argparser.add_argument(
         "-p", "--no-pulse", help="no pulser present", action="store_true"

legenddataflowscripts/par/geds/hit/ecal.py

@@ -71,7 +71,7 @@ def plot_2614_timemap(
             norm=LogNorm(),
         )
 
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )
@@ -125,7 +125,7 @@ def plot_pulser_timemap(
             norm=LogNorm(),
         )
         plt.ylim([mean - n_spread * spread, mean + n_spread * spread])
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )
@@ -264,7 +264,7 @@ def bin_survival_fraction(
         data.query(selection_string)[cal_energy_param],
         bins=np.arange(erange[0], erange[1] + dx, dx),
     )
-    counts_fail, bins_fail, _ = pgh.get_hist(
+    counts_fail, _, _ = pgh.get_hist(
         data.query(f"(~{cut_field})&(~{pulser_field})")[cal_energy_param],
         bins=np.arange(erange[0], erange[1] + dx, dx),
     )
@@ -303,7 +303,7 @@ def plot_baseline_timemap(
         norm=LogNorm(),
     )
 
-    ticks, labels = plt.xticks()
+    ticks, _ = plt.xticks()
     plt.xlabel(
         f"Time starting : {datetime.utcfromtimestamp(ticks[0]).strftime('%d/%m/%y %H:%M')}"
     )

legenddataflowscripts/workflow/filedb.py

@@ -42,6 +42,11 @@ def build_filedb() -> None:
         ignore = []
 
     fdb = FileDB(config, scan=False)
+    try:
+        fdb.scan_files([args.scan_path])
+    except Exception as e:
+        msg = f"error when building {args.output} from {args.scan_path}"
+        raise RuntimeError(msg) from e
     fdb.scan_files([args.scan_path])
     fdb.scan_tables_columns(dir_files_conform=True)
 
@@ -86,7 +91,7 @@ def build_filedb() -> None:
         if (
             (loc_timestamps == default).all() or not found
         ) and row.raw_file not in ignore:
-            msg = "something went wrong! no valid first timestamp found. Likely: the file is empty"
+            msg = "something went wrong! no valid first timestamp found. Likely: the file {row.raw_file} is empty"
             raise RuntimeError(msg)
 
         timestamps[i] = np.min(loc_timestamps)
@@ -97,7 +102,7 @@ def build_filedb() -> None:
         if (
             timestamps[i] < 0 or timestamps[i] > 4102444800
         ) and row.raw_file not in ignore:
-            msg = f"something went wrong! timestamp {timestamps[i]} does not make sense"
+            msg = f"something went wrong! timestamp {timestamps[i]} does not make sense in {row.raw_file}"
             raise RuntimeError(msg)
 
     fdb.df["first_timestamp"] = timestamps

legenddataflowscripts/workflow/utils.py

@@ -49,7 +49,8 @@ def subst_vars(
     if use_env:
         combined_var_values = dict(iter(os.environ.items()))
         combined_var_values.update(copy.copy(var_values))
-    subst_vars_impl(props, combined_var_values, ignore_missing)
+
+    return subst_vars_impl(props, combined_var_values, ignore_missing)
 
 
 def subst_vars_in_snakemake_config(workflow, config):
@@ -65,10 +66,11 @@ def subst_vars_in_snakemake_config(workflow, config):
         use_env=True,
         ignore_missing=False,
     )
-    if "system" in config:
-        config["execenv"] = config["execenv"][config["system"]]
-    else:
-        config["execenv"] = config["execenv"]["bare"]
+    if "execenv" in config:
+        if "system" in config:
+            config["execenv"] = config["execenv"][config["system"]]
+        else:
+            config["execenv"] = config["execenv"]["bare"]
 
 
 def set_last_rule_name(workflow, new_name):