legend-dataflow-scripts 0.1.4__tar.gz → 0.1.6__tar.gz

{legend_dataflow_scripts-0.1.4/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.1.4
+Version: 0.1.6
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6/src/legend_dataflow_scripts.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend-dataflow-scripts
-Version: 0.1.4
+Version: 0.1.6
 Summary: Python package for the processing scripts for LEGEND-200 data
 Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.4'
-__version_tuple__ = version_tuple = (0, 1, 4)
+__version__ = version = '0.1.6'
+__version_tuple__ = version_tuple = (0, 1, 6)

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/par/geds/dsp/eopt.py

@@ -23,7 +23,10 @@ from pygama.pargen.dsp_optimize import (
 from ....utils import build_log
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
-warnings.filterwarnings(action="ignore", category=np.exceptions.RankWarning)
+try:
+    warnings.filterwarnings(action="ignore", category=np.exceptions.RankWarning)
+except AttributeError:  # np < 2
+    warnings.filterwarnings(action="ignore", category=np.RankWarning)
 
 
 def par_geds_dsp_eopt() -> None:

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/par/geds/hit/aoe.py

@@ -297,6 +297,7 @@ def par_geds_hit_aoe() -> None:
     else:
         aoe = None
         plot_dict = out_plot_dict
+        results_dict = {}
 
     if args.plot_file:
         common_dict = plot_dict.pop("common") if "common" in list(plot_dict) else None

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/par/geds/hit/ecal.py

@@ -33,7 +33,10 @@ mpl.use("agg")
 sto = lh5.LH5Store()
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
-warnings.filterwarnings(action="ignore", category=np.exceptions.RankWarning)
+try:
+    warnings.filterwarnings(action="ignore", category=np.exceptions.RankWarning)
+except AttributeError:  # np < 2
+    warnings.filterwarnings(action="ignore", category=np.RankWarning)
 
 
 def plot_2614_timemap(
@@ -485,7 +488,7 @@ def par_geds_hit_ecal() -> None:
         msg = "invalid tier"
         raise ValueError(msg)
 
-    build_log(config_dict, args.log)
+    log = build_log(config_dict, args.log)
 
     chmap = LegendMetadata(args.metadata).channelmap(
         args.timestamp, system=args.datatype
@@ -575,14 +578,23 @@ def par_geds_hit_ecal() -> None:
     ):
         e_uncal = data.query(selection_string)[energy_param].to_numpy()
 
-        hist, bins, bar = pgh.get_hist(
-            e_uncal[
-                (e_uncal > np.nanpercentile(e_uncal, 95))
-                & (e_uncal < np.nanpercentile(e_uncal, 99.9))
-            ],
-            dx=1,
-            range=[np.nanpercentile(e_uncal, 95), np.nanpercentile(e_uncal, 99.9)],
-        )
+        if len(e_uncal) > 0:
+            if isinstance(e_uncal[0], np.ndarray | list):
+                e_uncal = np.concatenate([arr for arr in e_uncal if len(arr) > 0])
+            hist, bins, _ = pgh.get_hist(
+                e_uncal[
+                    (e_uncal > np.nanpercentile(e_uncal, 95))
+                    & (e_uncal < np.nanpercentile(e_uncal, 99.9))
+                ],
+                dx=1,
+                range=[
+                    np.nanpercentile(e_uncal, 95),
+                    np.nanpercentile(e_uncal, 99.9),
+                ],
+            )
+        else:
+            msg = f"e_uncal should not be empty! energy_param: {energy_param}"
+            raise ValueError(msg)
 
         guess = 2614.511 / bins[np.nanargmax(hist)]
         full_object_dict[cal_energy_param] = HPGeCalibration(
@@ -644,8 +656,21 @@ def par_geds_hit_ecal() -> None:
             interp_energy_kev={"Qbb": 2039.0},
         )
 
+        energy = data[energy_param].to_numpy()
+        if isinstance(energy[0], np.ndarray | list):
+            energy = np.concatenate(energy)
+
+        if len(energy) < len(data):
+            log.warning("len(energy) and len(data) are not the same")
+            energy = np.pad(
+                energy, (0, len(data) - len(energy)), constant_values=np.nan
+            )
+
+        if len(data) < len(energy):
+            energy = energy[: len(data)]
+
         data[cal_energy_param] = nb_poly(
-            data[energy_param].to_numpy(), full_object_dict[cal_energy_param].pars
+            energy, full_object_dict[cal_energy_param].pars
         )
 
         results_dict[cal_energy_param] = get_results_dict(
@@ -655,18 +680,32 @@ def par_geds_hit_ecal() -> None:
         hit_dict.update(
             {cal_energy_param: full_object_dict[cal_energy_param].gen_pars_dict()}
         )
-        if "ctc" in cal_energy_param:
-            no_ctc_dict = full_object_dict[cal_energy_param].gen_pars_dict()
-            no_ctc_dict["expression"] = no_ctc_dict["expression"].replace("_ctc", "")
-            hit_dict.update({cal_energy_param.replace("ctc", "noctc"): no_ctc_dict})
-            hit_dict.update(
-                {
-                    cal_energy_param.replace("_ctc", ""): {
-                        "expression": f"where({cal_energy_param.replace('ctc', 'noctc')}>{kwarg_dict.get('dt_theshold_kev', 100)}, {cal_energy_param}, {cal_energy_param.replace('ctc', 'noctc')})",
-                        "parameters": {},
-                    }
-                }
-            )
+        if "copy_calibration" in kwarg_dict:
+            for copy_cal_param, _copy_to_cal_param in kwarg_dict[
+                "copy_calibration"
+            ].items():
+                if copy_cal_param not in full_object_dict:
+                    msg = f"copy_calibration parameter {copy_cal_param} not found in full_object_dict"
+                    raise ValueError(msg)
+                if isinstance(_copy_to_cal_param, str):
+                    copy_to_cal_param = [_copy_to_cal_param]
+                else:
+                    copy_to_cal_param = _copy_to_cal_param
+                for cal_par in copy_to_cal_param:
+                    if cal_par in full_object_dict:
+                        msg = f"copy_calibration parameter {cal_par} already exists in full_object_dict"
+                        raise ValueError(msg)
+                    copy_dict = {cal_par: full_object_dict[cal_par].gen_pars_dict()}
+                    copy_dict["expression"] = copy_dict[cal_par]["expression"].replace(
+                        copy_cal_param, cal_par
+                    )
+                    hit_dict.update({cal_par: copy_dict[cal_par]})
+        if "extra_blocks" in kwarg_dict:
+            if isinstance(kwarg_dict["extra_blocks"], dict):
+                kwarg_dict["extra_blocks"] = [kwarg_dict["extra_blocks"]]
+            for extra_block in kwarg_dict["extra_blocks"]:
+                hit_dict.update(extra_block)
+
         if args.plot_path:
             param_plot_dict = {}
             if ~np.isnan(full_object_dict[cal_energy_param].pars).all():

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/par/geds/hit/lq.py

@@ -295,7 +295,7 @@ def par_geds_hit_lq() -> None:
 
         data["run_timestamp"] = args.timestamp
 
-        out_dicts, eres_dicts, plot_dicts, lq_dict = run_lq_calibration(
+        out_dicts, results_dicts, plot_dicts, lq_dict = run_lq_calibration(
             data,
             cal_dicts={args.timestamp: cal_dict},
             results_dicts={args.timestamp: eres_dict},
@@ -305,12 +305,13 @@ def par_geds_hit_lq() -> None:
             debug_mode=args.debug,
         )
         cal_dict = out_dicts[args.timestamp]
-        eres_dict = eres_dicts[args.timestamp]
+        results_dict = results_dicts[args.timestamp]
         plot_dict = plot_dicts[args.timestamp]
         lq = lq_dict[args.timestamp]
 
     else:
         lq = None
+        results_dict = {}
 
     if args.plot_file:
         Path(args.plot_file).parent.mkdir(parents=True, exist_ok=True)
@@ -320,7 +321,7 @@ def par_geds_hit_lq() -> None:
     final_hit_dict = convert_dict_np_to_float(
         {
             "pars": {"operations": cal_dict},
-            "results": dict(**ecal_dict["results"], lq=eres_dict),
+            "results": dict(**ecal_dict["results"], lq=results_dict),
         }
     )
     Path(args.hit_pars).parent.mkdir(parents=True, exist_ok=True)

legend_dataflow_scripts-0.1.6/src/legenddataflowscripts/utils/alias_table.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import h5py
+
+
+def convert_parents_to_structs(h5group):
+    if h5group.parent.name != "/" and len(h5group.parent.attrs) == 0:
+        h5group.parent.attrs.update(
+            {"datatype": "struct{" + h5group.name.split("/")[-1] + "}"}
+        )
+    elif (
+        len(h5group.parent.attrs) > 0
+        and h5group.name.split("/")[-1] not in h5group.parent.attrs["datatype"]
+    ):
+        h5group.parent.attrs.update(
+            {
+                "datatype": h5group.parent.attrs["datatype"][:-1]
+                + ","
+                + h5group.name.split("/")[-1]
+                + "}"
+            }
+        )
+    else:
+        return
+    convert_parents_to_structs(h5group.parent)
+    return
+
+
+def alias_table(file: str | Path, mapping: str):
+    """
+    Create an alias table for the given file and mapping.
+
+    Args:
+        file (str): Path to the input file.
+        mapping (dict): Mapping of current table name and alias table name.
+
+    """
+    if isinstance(mapping, str):
+        mapping = json.loads(mapping)
+    if isinstance(mapping, list):
+        for m in mapping:
+            alias_table(file, m)
+    with h5py.File(file, "a") as f:
+        for raw_id, alias in mapping.items():
+            if raw_id in f:
+                if isinstance(alias, list | tuple):
+                    for a in alias:
+                        f[a] = f[raw_id]
+                        convert_parents_to_structs(f[a])
+                else:
+                    f[alias] = f[raw_id]
+                    convert_parents_to_structs(f[alias])

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/utils/convert_np.py

@@ -24,7 +24,7 @@ def convert_dict_np_to_float(dic: dict) -> dict:
             convert_dict_np_to_float(value)
         elif isinstance(value, np.float32 | np.float64):
             dic[key] = float(value)
-        elif isinstance(dic[key], Sequence):
+        elif isinstance(dic[key], Sequence) and not isinstance(dic[key], str):
             dic[key] = [
                 float(x) if isinstance(x, np.float32 | np.float64) else x for x in value
             ]

{legend_dataflow_scripts-0.1.4 → legend_dataflow_scripts-0.1.6}/src/legenddataflowscripts/workflow/execenv.py

@@ -316,10 +316,16 @@ def cmdexec(args) -> None:
     )
     config_dict["execenv"] = config_dict["execenv"][args.system]
 
+    exe_path = Path(config_dict.paths.install).resolve() / "bin"
+
     cmd_prefix, cmd_env = execenv_prefix(config_dict, as_string=False)
     cmd_expr = [*cmd_prefix, *args.command]
 
     msg = "running: " + _execenv2str(cmd_expr, cmd_env)
     log.debug(msg)
 
-    subprocess.run(cmd_expr, env=os.environ | cmd_env, check=True)
+    env_dict = os.environ | cmd_env
+    env_dict["PATH"] = (
+        f"{exe_path}:{env_dict['PATH']}"  # prepend the virtualenv bin dir
+    )
+    subprocess.run(cmd_expr, env=env_dict, check=True)

legend_dataflow_scripts-0.1.4/src/legenddataflowscripts/utils/alias_table.py

@@ -1,28 +0,0 @@
-from __future__ import annotations
-
-import json
-
-import h5py
-
-
-def alias_table(file, mapping):
-    """
-    Create an alias table for the given file and mapping.
-
-    Args:
-        file (str): Path to the input file.
-        mapping (dict): Mapping of current table name and alias table name.
-
-    Returns:
-        dict: A dictionary containing the alias table.
-    """
-    if isinstance(mapping, str):
-        mapping = json.loads(mapping)
-    with h5py.File(file, "a") as f:
-        for raw_id, alias in mapping.items():
-            if raw_id in f:
-                if isinstance(alias, list | tuple):
-                    for a in alias:
-                        f[a] = f[raw_id]
-                else:
-                    f[alias] = f[raw_id]