legend-dataflow-scripts 0.2.3__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {legend_dataflow_scripts-0.2.3/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.3.0}/PKG-INFO +4 -4
  2. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/pyproject.toml +3 -3
  3. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0/src/legend_dataflow_scripts.egg-info}/PKG-INFO +4 -4
  4. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/requires.txt +3 -3
  5. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/_version.py +24 -0
  6. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/dplms.py +86 -23
  7. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/eopt.py +53 -4
  8. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/evtsel.py +104 -6
  9. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/nopt.py +41 -3
  10. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/pz.py +44 -2
  11. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/par/geds/dsp/svm.py +52 -0
  12. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/svm_build.py +30 -0
  13. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/aoe.py +129 -6
  14. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/ecal.py +341 -4
  15. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/lq.py +145 -32
  16. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/qc.py +101 -9
  17. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/dsp.py +107 -4
  18. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/hit.py +65 -0
  19. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/alias_table.py +85 -0
  20. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/cfgtools.py +38 -0
  21. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/log.py +170 -0
  22. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/plot_dict.py +39 -0
  23. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/pulser_removal.py +32 -0
  24. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/execenv.py +80 -21
  25. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/filedb.py +35 -0
  26. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/pre_compile_catalog.py +20 -0
  27. legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/workflow/utils.py +214 -0
  28. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/_version.py +0 -34
  29. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/par/geds/dsp/svm.py +0 -28
  30. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/utils/alias_table.py +0 -56
  31. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/utils/cfgtools.py +0 -14
  32. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/utils/log.py +0 -82
  33. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/utils/plot_dict.py +0 -13
  34. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/utils/pulser_removal.py +0 -16
  35. legend_dataflow_scripts-0.2.3/src/legenddataflowscripts/workflow/utils.py +0 -115
  36. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/LICENSE +0 -0
  37. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/README.md +0 -0
  38. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/setup.cfg +0 -0
  39. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/SOURCES.txt +0 -0
  40. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/dependency_links.txt +0 -0
  41. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/entry_points.txt +0 -0
  42. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/not-zip-safe +0 -0
  43. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/top_level.txt +0 -0
  44. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/__init__.py +0 -0
  45. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/__init__.py +0 -0
  46. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/__init__.py +0 -0
  47. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/__init__.py +0 -0
  48. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/__init__.py +0 -0
  49. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/__init__.py +0 -0
  50. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/utils/__init__.py +0 -0
  51. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/utils/convert_np.py +0 -0
  52. {legend_dataflow_scripts-0.2.3 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: legend-dataflow-scripts
3
- Version: 0.2.3
3
+ Version: 0.3.0
4
4
  Summary: Python package for the processing scripts for LEGEND-200 data
5
5
  Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
6
6
  Maintainer: The LEGEND Collaboration
@@ -19,10 +19,10 @@ Requires-Python: >=3.11
19
19
  Description-Content-Type: text/markdown
20
20
  Requires-Dist: colorlog
21
21
  Requires-Dist: dbetto>=1.2.3
22
- Requires-Dist: pygama>=2.1
23
- Requires-Dist: dspeed>=1.6
22
+ Requires-Dist: pygama>=2.3.8
23
+ Requires-Dist: dspeed>=2.0
24
24
  Requires-Dist: pylegendmeta>=1.2.5
25
- Requires-Dist: legend-pydataobj>=1.11
25
+ Requires-Dist: legend-pydataobj>=1.16
26
26
  Requires-Dist: pip
27
27
  Provides-Extra: test
28
28
  Requires-Dist: legend-dataflow-scripts; extra == "test"
@@ -52,10 +52,10 @@ dynamic = ["version"]
52
52
  dependencies = [
53
53
  "colorlog",
54
54
  "dbetto>=1.2.3",
55
- "pygama>=2.1",
56
- "dspeed>=1.6",
55
+ "pygama>=2.3.8",
56
+ "dspeed>=2.0",
57
57
  "pylegendmeta>=1.2.5",
58
- "legend-pydataobj>=1.11",
58
+ "legend-pydataobj>=1.16",
59
59
  "pip",
60
60
  ]
61
61
  # "legend-daq2lh5 @ file:///${PROJECT_ROOT}/software/python/src/legend-daq2lh5
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: legend-dataflow-scripts
3
- Version: 0.2.3
3
+ Version: 0.3.0
4
4
  Summary: Python package for the processing scripts for LEGEND-200 data
5
5
  Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
6
6
  Maintainer: The LEGEND Collaboration
@@ -19,10 +19,10 @@ Requires-Python: >=3.11
19
19
  Description-Content-Type: text/markdown
20
20
  Requires-Dist: colorlog
21
21
  Requires-Dist: dbetto>=1.2.3
22
- Requires-Dist: pygama>=2.1
23
- Requires-Dist: dspeed>=1.6
22
+ Requires-Dist: pygama>=2.3.8
23
+ Requires-Dist: dspeed>=2.0
24
24
  Requires-Dist: pylegendmeta>=1.2.5
25
- Requires-Dist: legend-pydataobj>=1.11
25
+ Requires-Dist: legend-pydataobj>=1.16
26
26
  Requires-Dist: pip
27
27
  Provides-Extra: test
28
28
  Requires-Dist: legend-dataflow-scripts; extra == "test"
@@ -1,9 +1,9 @@
1
1
  colorlog
2
2
  dbetto>=1.2.3
3
- pygama>=2.1
4
- dspeed>=1.6
3
+ pygama>=2.3.8
4
+ dspeed>=2.0
5
5
  pylegendmeta>=1.2.5
6
- legend-pydataobj>=1.11
6
+ legend-pydataobj>=1.16
7
7
  pip
8
8
 
9
9
  [dev]
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.3.0'
22
+ __version_tuple__ = version_tuple = (0, 3, 0)
23
+
24
+ __commit_id__ = commit_id = 'gfec5983ae'
@@ -9,12 +9,63 @@ import numpy as np
9
9
  import pygama.math.distributions as pmd # noqa: F401
10
10
  from dbetto.catalog import Props
11
11
  from lgdo import Array, Table, lh5
12
+ from pygama.pargen.data_cleaning import generate_cuts
12
13
  from pygama.pargen.dplms_ge_dict import dplms_ge_dict
14
+ from pygama.pargen.dsp_optimize import run_one_dsp
13
15
 
14
16
  from ....utils import build_log, convert_dict_np_to_float
15
17
 
16
18
 
17
19
  def par_geds_dsp_dplms() -> None:
20
+ """Compute DPLMS optimal filter coefficients for HPGe detectors.
21
+
22
+ CLI entry point registered as ``par-geds-dsp-dplms``. The *Discrete
23
+ Prolate Laplacian Maximum Signal* (DPLMS) filter is constructed from two
24
+ data samples:
25
+
26
+ 1. **FFT baselines** - low-energy (``daqenergy <= 10`` ADC) events from
27
+ dedicated FFT run files used to characterise the noise power spectrum.
28
+ 2. **Calibration peak events** - gamma-line events read from the
29
+ *peak-file* produced by :func:`par_geds_dsp_evtsel`.
30
+
31
+ The coefficients are computed by
32
+ :func:`pygama.pargen.dplms_ge_dict.dplms_ge_dict` and written to an LH5
33
+ file at *lh5-path* (so that the DSP chain can load them with
34
+ ``loadlh5(...)``). The DSP parameter database is updated with the filter
35
+ configuration and written to *dsp-pars*.
36
+
37
+ Notes
38
+ -----
39
+ **Command-line arguments**
40
+
41
+ ``--fft-raw-filelist`` : str
42
+ Path to a text file listing the FFT raw LH5 input files.
43
+ ``--peak-file`` : str
44
+ LH5 file containing preselected calibration peak events.
45
+ ``--inplots`` : str, optional
46
+ Existing pickle plot file to update with DPLMS plots.
47
+ ``--database`` : str
48
+ Path to the existing DSP parameter database (JSON/YAML).
49
+ ``--log`` : str, optional
50
+ Path to the log file.
51
+ ``--log-config`` : str, optional
52
+ Logging configuration file.
53
+ ``--processing-chain`` : list of str
54
+ Processing chain configuration file(s).
55
+ ``--config-file`` : list of str
56
+ DPLMS configuration file(s). Must contain ``run_dplms`` (bool),
57
+ ``n_baselines`` (int), and ``peaks_kev`` (list).
58
+ ``--channel`` : str
59
+ Channel identifier; used as the HDF5 group name in *lh5-path*.
60
+ ``--raw-table-name`` : str
61
+ LH5 table path within the raw and peak files.
62
+ ``--dsp-pars`` : str
63
+ Output path for the updated DSP parameter database (JSON/YAML).
64
+ ``--lh5-path`` : str
65
+ Output LH5 file path for the DPLMS filter coefficients.
66
+ ``--plot-path`` : str, optional
67
+ Output path for diagnostic plots (pickle).
68
+ """
18
69
  argparser = argparse.ArgumentParser()
19
70
  argparser.add_argument("--fft-raw-filelist", help="fft_raw_filelist", type=str)
20
71
  argparser.add_argument("--peak-file", help="tcm_filelist", type=str, required=True)
@@ -65,12 +116,12 @@ def par_geds_dsp_dplms() -> None:
65
116
  energies = lh5.read_as(
66
117
  f"{args.raw_table_name}/daqenergy", fft_files, library="np"
67
118
  )
68
- idxs = np.where(energies == 0)[0]
119
+ eidxs = np.where(energies <= 10)[0]
69
120
  raw_fft = lh5.read(
70
121
  args.raw_table_name,
71
122
  fft_files,
72
123
  n_rows=dplms_dict["n_baselines"],
73
- idx=idxs,
124
+ idx=eidxs,
74
125
  )
75
126
  t1 = time.time()
76
127
  msg = f"Time to load fft data {(t1 - t0):.2f} s, total events {len(raw_fft)}"
@@ -93,30 +144,43 @@ def par_geds_dsp_dplms() -> None:
93
144
  if isinstance(dsp_config, str | list):
94
145
  dsp_config = Props.read_from(dsp_config)
95
146
 
147
+ dsp_fft = run_one_dsp(raw_fft, dsp_config, db_dict=db_dict)
148
+
149
+ cut_dict = generate_cuts(dsp_fft, cut_dict=dplms_dict["bls_cut_pars"])
150
+ log.debug("Cuts are %s", cut_dict)
151
+ idxs = np.full(len(dsp_fft), True, dtype=bool)
152
+ for outname, info in cut_dict.items():
153
+ outcol = dsp_fft.eval(info["expression"], info.get("parameters", None))
154
+ dsp_fft.add_column(outname, outcol)
155
+ for cut in cut_dict:
156
+ idxs = dsp_fft[cut].nda & idxs
157
+
158
+ selected_eidxs = eidxs[: len(dsp_fft)]
159
+ raw_fft = lh5.read(
160
+ args.raw_table_name,
161
+ fft_files,
162
+ n_rows=dplms_dict["n_baselines"],
163
+ idx=selected_eidxs[idxs],
164
+ )
165
+
166
+ log.debug("Applied Cuts")
167
+
168
+ out_dict, plot_dict = dplms_ge_dict(
169
+ raw_fft,
170
+ raw_cal,
171
+ dsp_config,
172
+ db_dict,
173
+ dplms_dict,
174
+ fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
175
+ display=1 if args.plot_path else 0,
176
+ )
96
177
  if args.plot_path:
97
- out_dict, plot_dict = dplms_ge_dict(
98
- raw_fft,
99
- raw_cal,
100
- dsp_config,
101
- db_dict,
102
- dplms_dict,
103
- fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
104
- display=1,
105
- )
106
178
  if args.inplots:
107
179
  with Path(args.inplots).open("rb") as r:
108
180
  inplot_dict = pkl.load(r)
109
181
  inplot_dict.update({"dplms": plot_dict})
110
-
111
- else:
112
- out_dict = dplms_ge_dict(
113
- raw_fft,
114
- raw_cal,
115
- dsp_config,
116
- db_dict,
117
- dplms_dict,
118
- fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
119
- )
182
+ else:
183
+ inplot_dict = {"dplms": plot_dict}
120
184
 
121
185
  coeffs = out_dict["dplms"].pop("coefficients")
122
186
  dplms_pars = Table(col_dict={"coefficients": Array(coeffs)})
@@ -128,7 +192,7 @@ def par_geds_dsp_dplms() -> None:
128
192
  else:
129
193
  out_dict = {}
130
194
  dplms_pars = Table(col_dict={"coefficients": Array([])})
131
- if args.inplots:
195
+ if args.plot_path and args.inplots:
132
196
  with Path(args.inplots).open("rb") as r:
133
197
  inplot_dict = pkl.load(r)
134
198
  else:
@@ -143,7 +207,6 @@ def par_geds_dsp_dplms() -> None:
143
207
  lh5_file=args.lh5_path,
144
208
  wo_mode="overwrite",
145
209
  )
146
-
147
210
  Path(args.dsp_pars).parent.mkdir(parents=True, exist_ok=True)
148
211
  Props.write_to(args.dsp_pars, convert_dict_np_to_float(db_dict))
149
212
 
@@ -10,13 +10,13 @@ import numpy as np
10
10
  import pygama.pargen.energy_optimisation as om # noqa: F401
11
11
  import sklearn.gaussian_process.kernels as ker
12
12
  from dbetto.catalog import Props
13
+ from dspeed import build_dsp
13
14
  from dspeed.units import unit_registry as ureg
14
15
  from lgdo import lh5
15
16
  from pygama.math.distributions import hpge_peak
16
17
  from pygama.pargen.dsp_optimize import (
17
18
  BayesianOptimizer,
18
19
  run_bayesian_optimisation,
19
- run_one_dsp,
20
20
  )
21
21
 
22
22
  from ....utils import build_log
@@ -29,6 +29,55 @@ except AttributeError: # np < 2
29
29
 
30
30
 
31
31
  def par_geds_dsp_eopt() -> None:
32
+ """Optimise DSP filter parameters for energy resolution using Bayesian optimisation.
33
+
34
+ CLI entry point registered as ``par-geds-dsp-eopt``. Reads calibration
35
+ peak events from an LH5 file produced by :func:`par_geds_dsp_evtsel` and
36
+ jointly optimises the shaping parameters (``sigma`` for CUSP and ZAC
37
+ filters, ``rise`` for the trapezoidal filter) and the charge-trapping
38
+ correction coefficient (``alpha``) for each filter type.
39
+
40
+ Optimisation uses a Gaussian-process surrogate model
41
+ (:class:`pygama.pargen.dsp_optimize.BayesianOptimizer`) with a configurable
42
+ acquisition function. Initial samples are evaluated first; subsequent
43
+ iterations are selected by the acquisition function. The optimal filter
44
+ parameters and CTC coefficients are written to *final-dsp-pars*, and the
45
+ trained optimiser objects are serialised to *qbb-grid-path*.
46
+
47
+ Notes
48
+ -----
49
+ **Command-line arguments**
50
+
51
+ ``--peak-file`` : str
52
+ LH5 file containing preselected calibration peak events (output of
53
+ ``par-geds-dsp-evtsel``).
54
+ ``--decay-const`` : str
55
+ JSON/YAML file with the current DSP parameter database (PZ constants,
56
+ etc.).
57
+ ``--inplots`` : str, optional
58
+ Existing pickle plot file to update with new optimisation plots.
59
+ ``--log`` : str, optional
60
+ Path to the log file.
61
+ ``--processing-chain`` : list of str
62
+ Processing chain configuration file(s).
63
+ ``--config-file`` : list of str
64
+ Energy optimisation configuration file(s). Must contain ``run_eopt``
65
+ (bool), ``peaks`` (list of keV values), ``kev_widths``, ``fom``,
66
+ ``fom_field``, ``fom_err_field``, ``initial_samples``, ``acq_func``,
67
+ ``batch_size``, and ``n_iter``.
68
+ ``--log-config`` : str, optional
69
+ Logging configuration file.
70
+ ``--raw-table-name`` : str
71
+ LH5 table path within the peak file.
72
+ ``--final-dsp-pars`` : str
73
+ Output path for the optimised DSP parameter database (JSON/YAML).
74
+ ``--qbb-grid-path`` : str
75
+ Output path for the serialised optimiser objects (pickle).
76
+ ``--plot-path`` : str, optional
77
+ Output path for diagnostic plots (pickle).
78
+ ``--plot-save-path`` : str, optional
79
+ Alternative save path for plots.
80
+ """
32
81
  argparser = argparse.ArgumentParser()
33
82
 
34
83
  argparser.add_argument("--peak-file", help="tcm_filelist", type=str, required=True)
@@ -127,7 +176,7 @@ def par_geds_dsp_eopt() -> None:
127
176
 
128
177
  dsp_config["outputs"] = ["tp_99", "tp_0_est", "dt_eff"]
129
178
 
130
- init_data = run_one_dsp(tb_data, dsp_config, db_dict=db_dict, verbosity=0)
179
+ init_data = build_dsp(raw_in=tb_data, dsp_config=dsp_config, database=db_dict)
131
180
  full_dt = (init_data["tp_99"].nda - init_data["tp_0_est"].nda)[idx_list[-1]]
132
181
  flat_val = np.ceil(1.1 * np.nanpercentile(full_dt, 99) / 100) / 10
133
182
 
@@ -193,7 +242,7 @@ def par_geds_dsp_eopt() -> None:
193
242
  msg = f"Initialising values {i + 1} : {db_dict}"
194
243
  log.info(msg)
195
244
 
196
- tb_out = run_one_dsp(tb_data, dsp_config, db_dict=db_dict, verbosity=0)
245
+ tb_out = build_dsp(raw_in=tb_data, dsp_config=dsp_config, database=db_dict)
197
246
 
198
247
  res = fom(tb_out, kwarg_dict[0])
199
248
  results_cusp.append(res)
@@ -367,7 +416,7 @@ def par_geds_dsp_eopt() -> None:
367
416
 
368
417
  Path(args.qbb_grid_path).parent.mkdir(parents=True, exist_ok=True)
369
418
  with Path(args.qbb_grid_path).open("wb") as f:
370
- pkl.dump(optimisers, f)
419
+ pkl.dump({"eopt": optimisers}, f, protocol=pkl.HIGHEST_PROTOCOL)
371
420
 
372
421
  else:
373
422
  Path(args.qbb_grid_path).touch()
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import argparse
4
4
  import json
5
+ import sys
5
6
  import time
6
7
  import warnings
7
8
  from bisect import bisect_left
@@ -12,9 +13,9 @@ import numpy as np
12
13
  import pygama.math.histogram as pgh
13
14
  import pygama.pargen.energy_cal as pgc
14
15
  from dbetto.catalog import Props
16
+ from dspeed import build_dsp
15
17
  from lgdo import lh5
16
18
  from pygama.pargen.data_cleaning import generate_cuts, get_keys
17
- from pygama.pargen.dsp_optimize import run_one_dsp
18
19
 
19
20
  from ....utils import build_log, get_pulser_mask
20
21
 
@@ -33,6 +34,50 @@ def get_out_data(
33
34
  final_cut_field="is_valid_cal",
34
35
  energy_param="trapTmax",
35
36
  ):
37
+ """Apply cuts and assemble a calibration output table for a single gamma peak.
38
+
39
+ Evaluates the expressions in *cut_dict* on *dsp_data* and *raw_dict* on
40
+ *raw_data*, selects events within the energy window
41
+ [*e_lower_lim*, *e_upper_lim*] that also pass the *final_cut_field* flag,
42
+ and returns a :class:`lgdo.Table` containing both the raw waveforms and
43
+ key reconstructed quantities.
44
+
45
+ Parameters
46
+ ----------
47
+ raw_data : lgdo.Table
48
+ Raw-tier data buffer for the current batch of events.
49
+ dsp_data : lgdo.Table
50
+ DSP-tier data for the same batch.
51
+ cut_dict : dict
52
+ Mapping of ``{column_name: {"expression": str, "parameters": dict}}``
53
+ defining quality-cut columns to add to *dsp_data*.
54
+ e_lower_lim : float
55
+ Lower ADC bound for energy window selection.
56
+ e_upper_lim : float
57
+ Upper ADC bound for energy window selection.
58
+ ecal_pars : float
59
+ ADC-to-keV conversion factor applied to ``trapTmax``.
60
+ raw_dict : dict
61
+ Mapping of ``{column_name: {"expression": str, "parameters": dict}}``
62
+ defining derived columns to add to *raw_data* (e.g. calibrated DAQ
63
+ energy).
64
+ peak : float
65
+ Nominal gamma-line energy in keV stored as a label column.
66
+ final_cut_field : str
67
+ Boolean column name in *dsp_data* used as the final event selection
68
+ mask. Defaults to ``"is_valid_cal"``.
69
+ energy_param : str
70
+ DSP energy parameter used for the window selection. Defaults to
71
+ ``"trapTmax"``.
72
+
73
+ Returns
74
+ -------
75
+ out_tbl : lgdo.Table
76
+ Table of selected events containing windowed and pre-summed waveforms
77
+ plus scalar quantities (timestamp, baseline, energies, peak label).
78
+ n_events : int
79
+ Number of events in *out_tbl*.
80
+ """
36
81
  for outname, info in cut_dict.items():
37
82
  outcol = dsp_data.eval(info["expression"], info.get("parameters", None))
38
83
  dsp_data.add_column(outname, outcol)
@@ -81,6 +126,49 @@ def get_out_data(
81
126
 
82
127
 
83
128
  def par_geds_dsp_evtsel() -> None:
129
+ """Select calibration peak events for DSP parameter optimisation.
130
+
131
+ CLI entry point registered as ``par-geds-dsp-evtsel``. Scans raw LH5
132
+ files, applies pulser and discharge-recovery masks, performs a rough
133
+ energy calibration (or uses a provided calibration curve), and collects
134
+ events near each specified gamma-line energy. For each peak the data are
135
+ run through the DSP chain to derive quality cuts, and the surviving
136
+ waveforms together with reconstructed quantities are written to *peak-file*
137
+ in LH5 format for consumption by :func:`par_geds_dsp_eopt` and
138
+ :func:`par_geds_dsp_dplms`.
139
+
140
+ Notes
141
+ -----
142
+ **Command-line arguments**
143
+
144
+ ``--raw-filelist`` : list of str
145
+ Raw LH5 input file(s) or a single ``.filelist`` file.
146
+ ``--pulser-file`` : str, optional
147
+ Path to the pulser mask file.
148
+ ``-p`` / ``--no-pulse``
149
+ Flag indicating that no pulser is present.
150
+ ``--decay-const`` : str
151
+ JSON/YAML file with the current DSP parameter database.
152
+ ``--raw-cal-curve`` : list of str, optional
153
+ Pre-computed raw calibration curve file(s). When provided, the
154
+ ``--channel`` argument is also required.
155
+ ``--channel`` : str, optional (required with ``--raw-cal-curve``)
156
+ Channel identifier used to extract the calibration curve.
157
+ ``--log`` : str, optional
158
+ Path to the log file.
159
+ ``--processing-chain`` : list of str
160
+ Processing chain configuration file(s).
161
+ ``--config-file`` : list of str
162
+ Event selection configuration file(s). Must contain ``run_selection``
163
+ (bool), ``peaks`` (list of keV), ``kev_widths``, ``cut_parameters``,
164
+ ``n_events``, and ``final_cut_field``.
165
+ ``--log-config`` : str, optional
166
+ Logging configuration file.
167
+ ``--raw-table-name`` : str
168
+ LH5 table path within the raw file.
169
+ ``--peak-file`` : str
170
+ Output LH5 file path for selected peak events.
171
+ """
84
172
  argparser = argparse.ArgumentParser()
85
173
  argparser.add_argument("--raw-filelist", help="raw_filelist", type=str, nargs="*")
86
174
  argparser.add_argument(
@@ -99,6 +187,12 @@ def par_geds_dsp_evtsel() -> None:
99
187
  required=False,
100
188
  )
101
189
 
190
+ argparser.add_argument(
191
+ "--channel",
192
+ type=str,
193
+ help="Channel to process; required if --raw-cal-curve is set",
194
+ required="--raw-cal-curve" in sys.argv,
195
+ )
102
196
  argparser.add_argument("--log", help="log_file", type=str)
103
197
 
104
198
  argparser.add_argument(
@@ -247,7 +341,7 @@ def par_geds_dsp_evtsel() -> None:
247
341
  ]
248
342
 
249
343
  log.debug("Processing data")
250
- tb_data = run_one_dsp(input_data, dsp_config, db_dict=db_dict)
344
+ tb_data = build_dsp(raw_in=input_data, dsp_config=dsp_config, database=db_dict)
251
345
 
252
346
  if cut_parameters is not None:
253
347
  cut_dict = generate_cuts(tb_data, cut_parameters)
@@ -297,8 +391,10 @@ def par_geds_dsp_evtsel() -> None:
297
391
  peak_dict["obj_buf_start"] += n_rows_read_i
298
392
  if peak_dict["n_rows_read"] >= 10000 or file == raw_files[-1]:
299
393
  if "e_lower_lim" not in peak_dict:
300
- tb_out = run_one_dsp(
301
- peak_dict["obj_buf"], dsp_config, db_dict=db_dict
394
+ tb_out = build_dsp(
395
+ raw_in=peak_dict["obj_buf"],
396
+ dsp_config=dsp_config,
397
+ database=db_dict,
302
398
  )
303
399
  energy = tb_out[energy_parameter].nda
304
400
 
@@ -396,8 +492,10 @@ def par_geds_dsp_evtsel() -> None:
396
492
  peak_dict["obj_buf"] is not None
397
493
  and len(peak_dict["obj_buf"]) > 0
398
494
  ):
399
- tb_out = run_one_dsp(
400
- peak_dict["obj_buf"], dsp_config, db_dict=db_dict
495
+ tb_out = build_dsp(
496
+ raw_in=peak_dict["obj_buf"],
497
+ dsp_config=dsp_config,
498
+ database=db_dict,
401
499
  )
402
500
  out_tbl, n_wfs = get_out_data(
403
501
  peak_dict["obj_buf"],
@@ -8,14 +8,52 @@ from pathlib import Path
8
8
  import numpy as np
9
9
  import pygama.pargen.noise_optimization as pno
10
10
  from dbetto.catalog import Props
11
+ from dspeed import build_dsp
11
12
  from lgdo import lh5
12
13
  from pygama.pargen.data_cleaning import generate_cuts, get_cut_indexes
13
- from pygama.pargen.dsp_optimize import run_one_dsp
14
14
 
15
15
  from ....utils import build_log
16
16
 
17
17
 
18
18
  def par_geds_dsp_nopt() -> None:
19
+ """Optimise DSP filter parameters for noise performance.
20
+
21
+ CLI entry point registered as ``par-geds-dsp-nopt``. Selects low-energy
22
+ baseline events (``daqenergy ≤ 10`` ADC) from raw LH5 files, applies
23
+ quality cuts derived from the preliminary DSP output, and passes the clean
24
+ baseline sample to :func:`pygama.pargen.noise_optimization.noise_optimization`
25
+ to determine optimal filter shaping parameters for noise rejection.
26
+
27
+ The resulting noise-optimisation parameters are merged with the existing
28
+ DSP parameter database and written to *dsp-pars*. Optionally, diagnostic
29
+ plots are serialised to *plot-path*.
30
+
31
+ Notes
32
+ -----
33
+ **Command-line arguments**
34
+
35
+ ``--raw-filelist`` : str
36
+ Path to a text file listing the raw LH5 input files (one per line).
37
+ ``--database`` : str
38
+ Path to the existing DSP parameter database (JSON/YAML).
39
+ ``--inplots`` : str, optional
40
+ Existing pickle plot file to update with new noise-optimisation plots.
41
+ ``--log`` : str, optional
42
+ Path to the log file.
43
+ ``--processing-chain`` : list of str
44
+ Processing chain configuration file(s).
45
+ ``--config-file`` : list of str
46
+ Noise optimisation configuration file(s). Must contain ``run_nopt``
47
+ (bool), ``n_events`` (int), and ``cut_pars``.
48
+ ``--log-config`` : str, optional
49
+ Logging configuration file.
50
+ ``--raw-table-name`` : str
51
+ LH5 table path within the raw file.
52
+ ``--dsp-pars`` : str
53
+ Output path for the merged DSP parameter database (JSON/YAML).
54
+ ``--plot-path`` : str, optional
55
+ Output path for diagnostic plots (pickle).
56
+ """
19
57
  argparser = argparse.ArgumentParser()
20
58
  argparser.add_argument("--raw-filelist", help="raw_filelist", type=str)
21
59
  argparser.add_argument("--database", help="database", type=str, required=True)
@@ -63,7 +101,7 @@ def par_geds_dsp_nopt() -> None:
63
101
  energies = lh5.read_as(
64
102
  f"{args.raw_table_name}/daqenergy", raw_files, library="np"
65
103
  )
66
- idxs = np.where(energies == 0)[0]
104
+ idxs = np.where(energies <= 10)[0]
67
105
  tb_data = lh5.read(
68
106
  args.raw_table_name, raw_files, n_rows=opt_dict["n_events"], idx=idxs
69
107
  )
@@ -73,7 +111,7 @@ def par_geds_dsp_nopt() -> None:
73
111
 
74
112
  msg = f"Select baselines {len(tb_data)}"
75
113
  log.info(msg)
76
- dsp_data = run_one_dsp(tb_data, dsp_config)
114
+ dsp_data = build_dsp(raw_in=tb_data, dsp_config=dsp_config)
77
115
  cut_dict = generate_cuts(dsp_data, cut_dict=opt_dict.pop("cut_pars"))
78
116
  cut_idxs = get_cut_indexes(dsp_data, cut_dict)
79
117
  tb_data = lh5.read(
@@ -7,9 +7,9 @@ from pathlib import Path
7
7
 
8
8
  import numpy as np
9
9
  from dbetto.catalog import Props
10
+ from dspeed import build_dsp
10
11
  from lgdo import lh5
11
12
  from pygama.pargen.data_cleaning import get_cut_indexes
12
- from pygama.pargen.dsp_optimize import run_one_dsp
13
13
  from pygama.pargen.pz_correct import PZCorrect
14
14
 
15
15
  from ....utils import (
@@ -20,6 +20,48 @@ from ....utils import (
20
20
 
21
21
 
22
22
  def par_geds_dsp_pz() -> None:
23
+ """Determine the pole-zero (PZ) decay constant(s) for HPGe waveforms.
24
+
25
+ CLI entry point registered as ``par-geds-dsp-pz``. Selects high-energy
26
+ events from raw LH5 files (above an ADC threshold, excluding pulser and
27
+ discharge-recovery events), processes them through a preliminary DSP chain,
28
+ and uses :class:`pygama.pargen.pz_correct.PZCorrect` to fit either a
29
+ single- or double-exponential decay model to the waveform tails.
30
+
31
+ The fitted time constant(s) are written to *output-file* in JSON/YAML
32
+ format. Optionally, diagnostic waveform and slope plots are serialised to
33
+ a pickle file at *plot-path*.
34
+
35
+ Notes
36
+ -----
37
+ **Command-line arguments**
38
+
39
+ ``--processing-chain`` : list of str
40
+ Processing chain configuration file(s).
41
+ ``--config-file`` : list of str
42
+ PZ calibration configuration file(s). Must contain ``run_tau``
43
+ (bool), ``threshold`` (ADC), ``n_events`` (int), ``wf_field`` (str),
44
+ and ``mode`` (``"single"`` or ``"double"``).
45
+ ``--log-config`` : str, optional
46
+ Logging configuration file.
47
+ ``--raw-table-name`` : str
48
+ LH5 table path within the raw file (e.g. ``ch1057600/raw``).
49
+ ``--plot-path`` : str, optional
50
+ Output path for the diagnostic pickle file.
51
+ ``--output-file`` : str
52
+ Output path for the fitted PZ parameters (JSON/YAML).
53
+ ``--pulser-file`` : str, optional
54
+ Path to the pulser mask file.
55
+ ``-p`` / ``--no-pulse``
56
+ Flag indicating that no pulser is present; skips pulser masking.
57
+ ``--raw-files`` : list of str
58
+ Raw LH5 input file(s).
59
+ ``--pz-files`` : list of str, optional
60
+ Alternative input file(s) used instead of ``--raw-files`` when
61
+ present (e.g. preselected PZ calibration events).
62
+ ``--log`` : str, optional
63
+ Path to the log file.
64
+ """
23
65
  argparser = argparse.ArgumentParser()
24
66
  argparser.add_argument("--log", help="log file", type=str)
25
67
  argparser.add_argument(
@@ -134,7 +176,7 @@ def par_geds_dsp_pz() -> None:
134
176
  if "frac" in dsp_config["outputs"]:
135
177
  dsp_config_optimise_removed["outputs"].remove("frac")
136
178
 
137
- tb_out = run_one_dsp(tb_data, dsp_config_optimise_removed)
179
+ tb_out = build_dsp(raw_in=tb_data, dsp_config=dsp_config_optimise_removed)
138
180
  log.debug("Processed Data")
139
181
  cut_parameters = kwarg_dict.get("cut_parameters", None)
140
182
  if cut_parameters is not None: