legend-dataflow-scripts 0.2.4__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {legend_dataflow_scripts-0.2.4/src/legend_dataflow_scripts.egg-info → legend_dataflow_scripts-0.3.0}/PKG-INFO +4 -4
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/pyproject.toml +3 -3
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0/src/legend_dataflow_scripts.egg-info}/PKG-INFO +4 -4
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/requires.txt +3 -3
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/_version.py +24 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/dplms.py +86 -23
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/eopt.py +53 -4
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/evtsel.py +104 -6
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/nopt.py +41 -3
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/pz.py +44 -2
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/par/geds/dsp/svm.py +52 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/svm_build.py +30 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/aoe.py +122 -1
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/ecal.py +335 -1
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/lq.py +139 -29
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/qc.py +86 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/dsp.py +107 -4
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/hit.py +65 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/alias_table.py +85 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/cfgtools.py +38 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/log.py +170 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/plot_dict.py +39 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/utils/pulser_removal.py +32 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/execenv.py +80 -21
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/filedb.py +35 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/pre_compile_catalog.py +20 -0
- legend_dataflow_scripts-0.3.0/src/legenddataflowscripts/workflow/utils.py +214 -0
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/_version.py +0 -34
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/par/geds/dsp/svm.py +0 -28
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/utils/alias_table.py +0 -56
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/utils/cfgtools.py +0 -14
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/utils/log.py +0 -82
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/utils/plot_dict.py +0 -13
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/utils/pulser_removal.py +0 -16
- legend_dataflow_scripts-0.2.4/src/legenddataflowscripts/workflow/utils.py +0 -115
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/LICENSE +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/README.md +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/setup.cfg +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/SOURCES.txt +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/dependency_links.txt +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/entry_points.txt +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/not-zip-safe +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legend_dataflow_scripts.egg-info/top_level.txt +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/dsp/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/par/geds/hit/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/tier/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/utils/__init__.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/utils/convert_np.py +0 -0
- {legend_dataflow_scripts-0.2.4 → legend_dataflow_scripts-0.3.0}/src/legenddataflowscripts/workflow/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: legend-dataflow-scripts
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Python package for the processing scripts for LEGEND-200 data
|
|
5
5
|
Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
|
|
6
6
|
Maintainer: The LEGEND Collaboration
|
|
@@ -19,10 +19,10 @@ Requires-Python: >=3.11
|
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
Requires-Dist: colorlog
|
|
21
21
|
Requires-Dist: dbetto>=1.2.3
|
|
22
|
-
Requires-Dist: pygama>=2.
|
|
23
|
-
Requires-Dist: dspeed>=
|
|
22
|
+
Requires-Dist: pygama>=2.3.8
|
|
23
|
+
Requires-Dist: dspeed>=2.0
|
|
24
24
|
Requires-Dist: pylegendmeta>=1.2.5
|
|
25
|
-
Requires-Dist: legend-pydataobj>=1.
|
|
25
|
+
Requires-Dist: legend-pydataobj>=1.16
|
|
26
26
|
Requires-Dist: pip
|
|
27
27
|
Provides-Extra: test
|
|
28
28
|
Requires-Dist: legend-dataflow-scripts; extra == "test"
|
|
@@ -52,10 +52,10 @@ dynamic = ["version"]
|
|
|
52
52
|
dependencies = [
|
|
53
53
|
"colorlog",
|
|
54
54
|
"dbetto>=1.2.3",
|
|
55
|
-
"pygama>=2.
|
|
56
|
-
"dspeed>=
|
|
55
|
+
"pygama>=2.3.8",
|
|
56
|
+
"dspeed>=2.0",
|
|
57
57
|
"pylegendmeta>=1.2.5",
|
|
58
|
-
"legend-pydataobj>=1.
|
|
58
|
+
"legend-pydataobj>=1.16",
|
|
59
59
|
"pip",
|
|
60
60
|
]
|
|
61
61
|
# "legend-daq2lh5 @ file:///${PROJECT_ROOT}/software/python/src/legend-daq2lh5
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: legend-dataflow-scripts
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Python package for the processing scripts for LEGEND-200 data
|
|
5
5
|
Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
|
|
6
6
|
Maintainer: The LEGEND Collaboration
|
|
@@ -19,10 +19,10 @@ Requires-Python: >=3.11
|
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
Requires-Dist: colorlog
|
|
21
21
|
Requires-Dist: dbetto>=1.2.3
|
|
22
|
-
Requires-Dist: pygama>=2.
|
|
23
|
-
Requires-Dist: dspeed>=
|
|
22
|
+
Requires-Dist: pygama>=2.3.8
|
|
23
|
+
Requires-Dist: dspeed>=2.0
|
|
24
24
|
Requires-Dist: pylegendmeta>=1.2.5
|
|
25
|
-
Requires-Dist: legend-pydataobj>=1.
|
|
25
|
+
Requires-Dist: legend-pydataobj>=1.16
|
|
26
26
|
Requires-Dist: pip
|
|
27
27
|
Provides-Extra: test
|
|
28
28
|
Requires-Dist: legend-dataflow-scripts; extra == "test"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.3.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'gfec5983ae'
|
|
@@ -9,12 +9,63 @@ import numpy as np
|
|
|
9
9
|
import pygama.math.distributions as pmd # noqa: F401
|
|
10
10
|
from dbetto.catalog import Props
|
|
11
11
|
from lgdo import Array, Table, lh5
|
|
12
|
+
from pygama.pargen.data_cleaning import generate_cuts
|
|
12
13
|
from pygama.pargen.dplms_ge_dict import dplms_ge_dict
|
|
14
|
+
from pygama.pargen.dsp_optimize import run_one_dsp
|
|
13
15
|
|
|
14
16
|
from ....utils import build_log, convert_dict_np_to_float
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
def par_geds_dsp_dplms() -> None:
|
|
20
|
+
"""Compute DPLMS optimal filter coefficients for HPGe detectors.
|
|
21
|
+
|
|
22
|
+
CLI entry point registered as ``par-geds-dsp-dplms``. The *Discrete
|
|
23
|
+
Prolate Laplacian Maximum Signal* (DPLMS) filter is constructed from two
|
|
24
|
+
data samples:
|
|
25
|
+
|
|
26
|
+
1. **FFT baselines** - low-energy (``daqenergy <= 10`` ADC) events from
|
|
27
|
+
dedicated FFT run files used to characterise the noise power spectrum.
|
|
28
|
+
2. **Calibration peak events** - gamma-line events read from the
|
|
29
|
+
*peak-file* produced by :func:`par_geds_dsp_evtsel`.
|
|
30
|
+
|
|
31
|
+
The coefficients are computed by
|
|
32
|
+
:func:`pygama.pargen.dplms_ge_dict.dplms_ge_dict` and written to an LH5
|
|
33
|
+
file at *lh5-path* (so that the DSP chain can load them with
|
|
34
|
+
``loadlh5(...)``). The DSP parameter database is updated with the filter
|
|
35
|
+
configuration and written to *dsp-pars*.
|
|
36
|
+
|
|
37
|
+
Notes
|
|
38
|
+
-----
|
|
39
|
+
**Command-line arguments**
|
|
40
|
+
|
|
41
|
+
``--fft-raw-filelist`` : str
|
|
42
|
+
Path to a text file listing the FFT raw LH5 input files.
|
|
43
|
+
``--peak-file`` : str
|
|
44
|
+
LH5 file containing preselected calibration peak events.
|
|
45
|
+
``--inplots`` : str, optional
|
|
46
|
+
Existing pickle plot file to update with DPLMS plots.
|
|
47
|
+
``--database`` : str
|
|
48
|
+
Path to the existing DSP parameter database (JSON/YAML).
|
|
49
|
+
``--log`` : str, optional
|
|
50
|
+
Path to the log file.
|
|
51
|
+
``--log-config`` : str, optional
|
|
52
|
+
Logging configuration file.
|
|
53
|
+
``--processing-chain`` : list of str
|
|
54
|
+
Processing chain configuration file(s).
|
|
55
|
+
``--config-file`` : list of str
|
|
56
|
+
DPLMS configuration file(s). Must contain ``run_dplms`` (bool),
|
|
57
|
+
``n_baselines`` (int), and ``peaks_kev`` (list).
|
|
58
|
+
``--channel`` : str
|
|
59
|
+
Channel identifier; used as the HDF5 group name in *lh5-path*.
|
|
60
|
+
``--raw-table-name`` : str
|
|
61
|
+
LH5 table path within the raw and peak files.
|
|
62
|
+
``--dsp-pars`` : str
|
|
63
|
+
Output path for the updated DSP parameter database (JSON/YAML).
|
|
64
|
+
``--lh5-path`` : str
|
|
65
|
+
Output LH5 file path for the DPLMS filter coefficients.
|
|
66
|
+
``--plot-path`` : str, optional
|
|
67
|
+
Output path for diagnostic plots (pickle).
|
|
68
|
+
"""
|
|
18
69
|
argparser = argparse.ArgumentParser()
|
|
19
70
|
argparser.add_argument("--fft-raw-filelist", help="fft_raw_filelist", type=str)
|
|
20
71
|
argparser.add_argument("--peak-file", help="tcm_filelist", type=str, required=True)
|
|
@@ -65,12 +116,12 @@ def par_geds_dsp_dplms() -> None:
|
|
|
65
116
|
energies = lh5.read_as(
|
|
66
117
|
f"{args.raw_table_name}/daqenergy", fft_files, library="np"
|
|
67
118
|
)
|
|
68
|
-
|
|
119
|
+
eidxs = np.where(energies <= 10)[0]
|
|
69
120
|
raw_fft = lh5.read(
|
|
70
121
|
args.raw_table_name,
|
|
71
122
|
fft_files,
|
|
72
123
|
n_rows=dplms_dict["n_baselines"],
|
|
73
|
-
idx=
|
|
124
|
+
idx=eidxs,
|
|
74
125
|
)
|
|
75
126
|
t1 = time.time()
|
|
76
127
|
msg = f"Time to load fft data {(t1 - t0):.2f} s, total events {len(raw_fft)}"
|
|
@@ -93,30 +144,43 @@ def par_geds_dsp_dplms() -> None:
|
|
|
93
144
|
if isinstance(dsp_config, str | list):
|
|
94
145
|
dsp_config = Props.read_from(dsp_config)
|
|
95
146
|
|
|
147
|
+
dsp_fft = run_one_dsp(raw_fft, dsp_config, db_dict=db_dict)
|
|
148
|
+
|
|
149
|
+
cut_dict = generate_cuts(dsp_fft, cut_dict=dplms_dict["bls_cut_pars"])
|
|
150
|
+
log.debug("Cuts are %s", cut_dict)
|
|
151
|
+
idxs = np.full(len(dsp_fft), True, dtype=bool)
|
|
152
|
+
for outname, info in cut_dict.items():
|
|
153
|
+
outcol = dsp_fft.eval(info["expression"], info.get("parameters", None))
|
|
154
|
+
dsp_fft.add_column(outname, outcol)
|
|
155
|
+
for cut in cut_dict:
|
|
156
|
+
idxs = dsp_fft[cut].nda & idxs
|
|
157
|
+
|
|
158
|
+
selected_eidxs = eidxs[: len(dsp_fft)]
|
|
159
|
+
raw_fft = lh5.read(
|
|
160
|
+
args.raw_table_name,
|
|
161
|
+
fft_files,
|
|
162
|
+
n_rows=dplms_dict["n_baselines"],
|
|
163
|
+
idx=selected_eidxs[idxs],
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
log.debug("Applied Cuts")
|
|
167
|
+
|
|
168
|
+
out_dict, plot_dict = dplms_ge_dict(
|
|
169
|
+
raw_fft,
|
|
170
|
+
raw_cal,
|
|
171
|
+
dsp_config,
|
|
172
|
+
db_dict,
|
|
173
|
+
dplms_dict,
|
|
174
|
+
fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
|
|
175
|
+
display=1 if args.plot_path else 0,
|
|
176
|
+
)
|
|
96
177
|
if args.plot_path:
|
|
97
|
-
out_dict, plot_dict = dplms_ge_dict(
|
|
98
|
-
raw_fft,
|
|
99
|
-
raw_cal,
|
|
100
|
-
dsp_config,
|
|
101
|
-
db_dict,
|
|
102
|
-
dplms_dict,
|
|
103
|
-
fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
|
|
104
|
-
display=1,
|
|
105
|
-
)
|
|
106
178
|
if args.inplots:
|
|
107
179
|
with Path(args.inplots).open("rb") as r:
|
|
108
180
|
inplot_dict = pkl.load(r)
|
|
109
181
|
inplot_dict.update({"dplms": plot_dict})
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
out_dict = dplms_ge_dict(
|
|
113
|
-
raw_fft,
|
|
114
|
-
raw_cal,
|
|
115
|
-
dsp_config,
|
|
116
|
-
db_dict,
|
|
117
|
-
dplms_dict,
|
|
118
|
-
fom_func=eval(dplms_dict.get("fom_func", "pmd.gauss_on_step")),
|
|
119
|
-
)
|
|
182
|
+
else:
|
|
183
|
+
inplot_dict = {"dplms": plot_dict}
|
|
120
184
|
|
|
121
185
|
coeffs = out_dict["dplms"].pop("coefficients")
|
|
122
186
|
dplms_pars = Table(col_dict={"coefficients": Array(coeffs)})
|
|
@@ -128,7 +192,7 @@ def par_geds_dsp_dplms() -> None:
|
|
|
128
192
|
else:
|
|
129
193
|
out_dict = {}
|
|
130
194
|
dplms_pars = Table(col_dict={"coefficients": Array([])})
|
|
131
|
-
if args.inplots:
|
|
195
|
+
if args.plot_path and args.inplots:
|
|
132
196
|
with Path(args.inplots).open("rb") as r:
|
|
133
197
|
inplot_dict = pkl.load(r)
|
|
134
198
|
else:
|
|
@@ -143,7 +207,6 @@ def par_geds_dsp_dplms() -> None:
|
|
|
143
207
|
lh5_file=args.lh5_path,
|
|
144
208
|
wo_mode="overwrite",
|
|
145
209
|
)
|
|
146
|
-
|
|
147
210
|
Path(args.dsp_pars).parent.mkdir(parents=True, exist_ok=True)
|
|
148
211
|
Props.write_to(args.dsp_pars, convert_dict_np_to_float(db_dict))
|
|
149
212
|
|
|
@@ -10,13 +10,13 @@ import numpy as np
|
|
|
10
10
|
import pygama.pargen.energy_optimisation as om # noqa: F401
|
|
11
11
|
import sklearn.gaussian_process.kernels as ker
|
|
12
12
|
from dbetto.catalog import Props
|
|
13
|
+
from dspeed import build_dsp
|
|
13
14
|
from dspeed.units import unit_registry as ureg
|
|
14
15
|
from lgdo import lh5
|
|
15
16
|
from pygama.math.distributions import hpge_peak
|
|
16
17
|
from pygama.pargen.dsp_optimize import (
|
|
17
18
|
BayesianOptimizer,
|
|
18
19
|
run_bayesian_optimisation,
|
|
19
|
-
run_one_dsp,
|
|
20
20
|
)
|
|
21
21
|
|
|
22
22
|
from ....utils import build_log
|
|
@@ -29,6 +29,55 @@ except AttributeError: # np < 2
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
def par_geds_dsp_eopt() -> None:
|
|
32
|
+
"""Optimise DSP filter parameters for energy resolution using Bayesian optimisation.
|
|
33
|
+
|
|
34
|
+
CLI entry point registered as ``par-geds-dsp-eopt``. Reads calibration
|
|
35
|
+
peak events from an LH5 file produced by :func:`par_geds_dsp_evtsel` and
|
|
36
|
+
jointly optimises the shaping parameters (``sigma`` for CUSP and ZAC
|
|
37
|
+
filters, ``rise`` for the trapezoidal filter) and the charge-trapping
|
|
38
|
+
correction coefficient (``alpha``) for each filter type.
|
|
39
|
+
|
|
40
|
+
Optimisation uses a Gaussian-process surrogate model
|
|
41
|
+
(:class:`pygama.pargen.dsp_optimize.BayesianOptimizer`) with a configurable
|
|
42
|
+
acquisition function. Initial samples are evaluated first; subsequent
|
|
43
|
+
iterations are selected by the acquisition function. The optimal filter
|
|
44
|
+
parameters and CTC coefficients are written to *final-dsp-pars*, and the
|
|
45
|
+
trained optimiser objects are serialised to *qbb-grid-path*.
|
|
46
|
+
|
|
47
|
+
Notes
|
|
48
|
+
-----
|
|
49
|
+
**Command-line arguments**
|
|
50
|
+
|
|
51
|
+
``--peak-file`` : str
|
|
52
|
+
LH5 file containing preselected calibration peak events (output of
|
|
53
|
+
``par-geds-dsp-evtsel``).
|
|
54
|
+
``--decay-const`` : str
|
|
55
|
+
JSON/YAML file with the current DSP parameter database (PZ constants,
|
|
56
|
+
etc.).
|
|
57
|
+
``--inplots`` : str, optional
|
|
58
|
+
Existing pickle plot file to update with new optimisation plots.
|
|
59
|
+
``--log`` : str, optional
|
|
60
|
+
Path to the log file.
|
|
61
|
+
``--processing-chain`` : list of str
|
|
62
|
+
Processing chain configuration file(s).
|
|
63
|
+
``--config-file`` : list of str
|
|
64
|
+
Energy optimisation configuration file(s). Must contain ``run_eopt``
|
|
65
|
+
(bool), ``peaks`` (list of keV values), ``kev_widths``, ``fom``,
|
|
66
|
+
``fom_field``, ``fom_err_field``, ``initial_samples``, ``acq_func``,
|
|
67
|
+
``batch_size``, and ``n_iter``.
|
|
68
|
+
``--log-config`` : str, optional
|
|
69
|
+
Logging configuration file.
|
|
70
|
+
``--raw-table-name`` : str
|
|
71
|
+
LH5 table path within the peak file.
|
|
72
|
+
``--final-dsp-pars`` : str
|
|
73
|
+
Output path for the optimised DSP parameter database (JSON/YAML).
|
|
74
|
+
``--qbb-grid-path`` : str
|
|
75
|
+
Output path for the serialised optimiser objects (pickle).
|
|
76
|
+
``--plot-path`` : str, optional
|
|
77
|
+
Output path for diagnostic plots (pickle).
|
|
78
|
+
``--plot-save-path`` : str, optional
|
|
79
|
+
Alternative save path for plots.
|
|
80
|
+
"""
|
|
32
81
|
argparser = argparse.ArgumentParser()
|
|
33
82
|
|
|
34
83
|
argparser.add_argument("--peak-file", help="tcm_filelist", type=str, required=True)
|
|
@@ -127,7 +176,7 @@ def par_geds_dsp_eopt() -> None:
|
|
|
127
176
|
|
|
128
177
|
dsp_config["outputs"] = ["tp_99", "tp_0_est", "dt_eff"]
|
|
129
178
|
|
|
130
|
-
init_data =
|
|
179
|
+
init_data = build_dsp(raw_in=tb_data, dsp_config=dsp_config, database=db_dict)
|
|
131
180
|
full_dt = (init_data["tp_99"].nda - init_data["tp_0_est"].nda)[idx_list[-1]]
|
|
132
181
|
flat_val = np.ceil(1.1 * np.nanpercentile(full_dt, 99) / 100) / 10
|
|
133
182
|
|
|
@@ -193,7 +242,7 @@ def par_geds_dsp_eopt() -> None:
|
|
|
193
242
|
msg = f"Initialising values {i + 1} : {db_dict}"
|
|
194
243
|
log.info(msg)
|
|
195
244
|
|
|
196
|
-
tb_out =
|
|
245
|
+
tb_out = build_dsp(raw_in=tb_data, dsp_config=dsp_config, database=db_dict)
|
|
197
246
|
|
|
198
247
|
res = fom(tb_out, kwarg_dict[0])
|
|
199
248
|
results_cusp.append(res)
|
|
@@ -367,7 +416,7 @@ def par_geds_dsp_eopt() -> None:
|
|
|
367
416
|
|
|
368
417
|
Path(args.qbb_grid_path).parent.mkdir(parents=True, exist_ok=True)
|
|
369
418
|
with Path(args.qbb_grid_path).open("wb") as f:
|
|
370
|
-
pkl.dump(optimisers, f)
|
|
419
|
+
pkl.dump({"eopt": optimisers}, f, protocol=pkl.HIGHEST_PROTOCOL)
|
|
371
420
|
|
|
372
421
|
else:
|
|
373
422
|
Path(args.qbb_grid_path).touch()
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
4
|
import json
|
|
5
|
+
import sys
|
|
5
6
|
import time
|
|
6
7
|
import warnings
|
|
7
8
|
from bisect import bisect_left
|
|
@@ -12,9 +13,9 @@ import numpy as np
|
|
|
12
13
|
import pygama.math.histogram as pgh
|
|
13
14
|
import pygama.pargen.energy_cal as pgc
|
|
14
15
|
from dbetto.catalog import Props
|
|
16
|
+
from dspeed import build_dsp
|
|
15
17
|
from lgdo import lh5
|
|
16
18
|
from pygama.pargen.data_cleaning import generate_cuts, get_keys
|
|
17
|
-
from pygama.pargen.dsp_optimize import run_one_dsp
|
|
18
19
|
|
|
19
20
|
from ....utils import build_log, get_pulser_mask
|
|
20
21
|
|
|
@@ -33,6 +34,50 @@ def get_out_data(
|
|
|
33
34
|
final_cut_field="is_valid_cal",
|
|
34
35
|
energy_param="trapTmax",
|
|
35
36
|
):
|
|
37
|
+
"""Apply cuts and assemble a calibration output table for a single gamma peak.
|
|
38
|
+
|
|
39
|
+
Evaluates the expressions in *cut_dict* on *dsp_data* and *raw_dict* on
|
|
40
|
+
*raw_data*, selects events within the energy window
|
|
41
|
+
[*e_lower_lim*, *e_upper_lim*] that also pass the *final_cut_field* flag,
|
|
42
|
+
and returns a :class:`lgdo.Table` containing both the raw waveforms and
|
|
43
|
+
key reconstructed quantities.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
raw_data : lgdo.Table
|
|
48
|
+
Raw-tier data buffer for the current batch of events.
|
|
49
|
+
dsp_data : lgdo.Table
|
|
50
|
+
DSP-tier data for the same batch.
|
|
51
|
+
cut_dict : dict
|
|
52
|
+
Mapping of ``{column_name: {"expression": str, "parameters": dict}}``
|
|
53
|
+
defining quality-cut columns to add to *dsp_data*.
|
|
54
|
+
e_lower_lim : float
|
|
55
|
+
Lower ADC bound for energy window selection.
|
|
56
|
+
e_upper_lim : float
|
|
57
|
+
Upper ADC bound for energy window selection.
|
|
58
|
+
ecal_pars : float
|
|
59
|
+
ADC-to-keV conversion factor applied to ``trapTmax``.
|
|
60
|
+
raw_dict : dict
|
|
61
|
+
Mapping of ``{column_name: {"expression": str, "parameters": dict}}``
|
|
62
|
+
defining derived columns to add to *raw_data* (e.g. calibrated DAQ
|
|
63
|
+
energy).
|
|
64
|
+
peak : float
|
|
65
|
+
Nominal gamma-line energy in keV stored as a label column.
|
|
66
|
+
final_cut_field : str
|
|
67
|
+
Boolean column name in *dsp_data* used as the final event selection
|
|
68
|
+
mask. Defaults to ``"is_valid_cal"``.
|
|
69
|
+
energy_param : str
|
|
70
|
+
DSP energy parameter used for the window selection. Defaults to
|
|
71
|
+
``"trapTmax"``.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
out_tbl : lgdo.Table
|
|
76
|
+
Table of selected events containing windowed and pre-summed waveforms
|
|
77
|
+
plus scalar quantities (timestamp, baseline, energies, peak label).
|
|
78
|
+
n_events : int
|
|
79
|
+
Number of events in *out_tbl*.
|
|
80
|
+
"""
|
|
36
81
|
for outname, info in cut_dict.items():
|
|
37
82
|
outcol = dsp_data.eval(info["expression"], info.get("parameters", None))
|
|
38
83
|
dsp_data.add_column(outname, outcol)
|
|
@@ -81,6 +126,49 @@ def get_out_data(
|
|
|
81
126
|
|
|
82
127
|
|
|
83
128
|
def par_geds_dsp_evtsel() -> None:
|
|
129
|
+
"""Select calibration peak events for DSP parameter optimisation.
|
|
130
|
+
|
|
131
|
+
CLI entry point registered as ``par-geds-dsp-evtsel``. Scans raw LH5
|
|
132
|
+
files, applies pulser and discharge-recovery masks, performs a rough
|
|
133
|
+
energy calibration (or uses a provided calibration curve), and collects
|
|
134
|
+
events near each specified gamma-line energy. For each peak the data are
|
|
135
|
+
run through the DSP chain to derive quality cuts, and the surviving
|
|
136
|
+
waveforms together with reconstructed quantities are written to *peak-file*
|
|
137
|
+
in LH5 format for consumption by :func:`par_geds_dsp_eopt` and
|
|
138
|
+
:func:`par_geds_dsp_dplms`.
|
|
139
|
+
|
|
140
|
+
Notes
|
|
141
|
+
-----
|
|
142
|
+
**Command-line arguments**
|
|
143
|
+
|
|
144
|
+
``--raw-filelist`` : list of str
|
|
145
|
+
Raw LH5 input file(s) or a single ``.filelist`` file.
|
|
146
|
+
``--pulser-file`` : str, optional
|
|
147
|
+
Path to the pulser mask file.
|
|
148
|
+
``-p`` / ``--no-pulse``
|
|
149
|
+
Flag indicating that no pulser is present.
|
|
150
|
+
``--decay-const`` : str
|
|
151
|
+
JSON/YAML file with the current DSP parameter database.
|
|
152
|
+
``--raw-cal-curve`` : list of str, optional
|
|
153
|
+
Pre-computed raw calibration curve file(s). When provided, the
|
|
154
|
+
``--channel`` argument is also required.
|
|
155
|
+
``--channel`` : str, optional (required with ``--raw-cal-curve``)
|
|
156
|
+
Channel identifier used to extract the calibration curve.
|
|
157
|
+
``--log`` : str, optional
|
|
158
|
+
Path to the log file.
|
|
159
|
+
``--processing-chain`` : list of str
|
|
160
|
+
Processing chain configuration file(s).
|
|
161
|
+
``--config-file`` : list of str
|
|
162
|
+
Event selection configuration file(s). Must contain ``run_selection``
|
|
163
|
+
(bool), ``peaks`` (list of keV), ``kev_widths``, ``cut_parameters``,
|
|
164
|
+
``n_events``, and ``final_cut_field``.
|
|
165
|
+
``--log-config`` : str, optional
|
|
166
|
+
Logging configuration file.
|
|
167
|
+
``--raw-table-name`` : str
|
|
168
|
+
LH5 table path within the raw file.
|
|
169
|
+
``--peak-file`` : str
|
|
170
|
+
Output LH5 file path for selected peak events.
|
|
171
|
+
"""
|
|
84
172
|
argparser = argparse.ArgumentParser()
|
|
85
173
|
argparser.add_argument("--raw-filelist", help="raw_filelist", type=str, nargs="*")
|
|
86
174
|
argparser.add_argument(
|
|
@@ -99,6 +187,12 @@ def par_geds_dsp_evtsel() -> None:
|
|
|
99
187
|
required=False,
|
|
100
188
|
)
|
|
101
189
|
|
|
190
|
+
argparser.add_argument(
|
|
191
|
+
"--channel",
|
|
192
|
+
type=str,
|
|
193
|
+
help="Channel to process; required if --raw-cal-curve is set",
|
|
194
|
+
required="--raw-cal-curve" in sys.argv,
|
|
195
|
+
)
|
|
102
196
|
argparser.add_argument("--log", help="log_file", type=str)
|
|
103
197
|
|
|
104
198
|
argparser.add_argument(
|
|
@@ -247,7 +341,7 @@ def par_geds_dsp_evtsel() -> None:
|
|
|
247
341
|
]
|
|
248
342
|
|
|
249
343
|
log.debug("Processing data")
|
|
250
|
-
tb_data =
|
|
344
|
+
tb_data = build_dsp(raw_in=input_data, dsp_config=dsp_config, database=db_dict)
|
|
251
345
|
|
|
252
346
|
if cut_parameters is not None:
|
|
253
347
|
cut_dict = generate_cuts(tb_data, cut_parameters)
|
|
@@ -297,8 +391,10 @@ def par_geds_dsp_evtsel() -> None:
|
|
|
297
391
|
peak_dict["obj_buf_start"] += n_rows_read_i
|
|
298
392
|
if peak_dict["n_rows_read"] >= 10000 or file == raw_files[-1]:
|
|
299
393
|
if "e_lower_lim" not in peak_dict:
|
|
300
|
-
tb_out =
|
|
301
|
-
peak_dict["obj_buf"],
|
|
394
|
+
tb_out = build_dsp(
|
|
395
|
+
raw_in=peak_dict["obj_buf"],
|
|
396
|
+
dsp_config=dsp_config,
|
|
397
|
+
database=db_dict,
|
|
302
398
|
)
|
|
303
399
|
energy = tb_out[energy_parameter].nda
|
|
304
400
|
|
|
@@ -396,8 +492,10 @@ def par_geds_dsp_evtsel() -> None:
|
|
|
396
492
|
peak_dict["obj_buf"] is not None
|
|
397
493
|
and len(peak_dict["obj_buf"]) > 0
|
|
398
494
|
):
|
|
399
|
-
tb_out =
|
|
400
|
-
peak_dict["obj_buf"],
|
|
495
|
+
tb_out = build_dsp(
|
|
496
|
+
raw_in=peak_dict["obj_buf"],
|
|
497
|
+
dsp_config=dsp_config,
|
|
498
|
+
database=db_dict,
|
|
401
499
|
)
|
|
402
500
|
out_tbl, n_wfs = get_out_data(
|
|
403
501
|
peak_dict["obj_buf"],
|
|
@@ -8,14 +8,52 @@ from pathlib import Path
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
import pygama.pargen.noise_optimization as pno
|
|
10
10
|
from dbetto.catalog import Props
|
|
11
|
+
from dspeed import build_dsp
|
|
11
12
|
from lgdo import lh5
|
|
12
13
|
from pygama.pargen.data_cleaning import generate_cuts, get_cut_indexes
|
|
13
|
-
from pygama.pargen.dsp_optimize import run_one_dsp
|
|
14
14
|
|
|
15
15
|
from ....utils import build_log
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def par_geds_dsp_nopt() -> None:
|
|
19
|
+
"""Optimise DSP filter parameters for noise performance.
|
|
20
|
+
|
|
21
|
+
CLI entry point registered as ``par-geds-dsp-nopt``. Selects low-energy
|
|
22
|
+
baseline events (``daqenergy ≤ 10`` ADC) from raw LH5 files, applies
|
|
23
|
+
quality cuts derived from the preliminary DSP output, and passes the clean
|
|
24
|
+
baseline sample to :func:`pygama.pargen.noise_optimization.noise_optimization`
|
|
25
|
+
to determine optimal filter shaping parameters for noise rejection.
|
|
26
|
+
|
|
27
|
+
The resulting noise-optimisation parameters are merged with the existing
|
|
28
|
+
DSP parameter database and written to *dsp-pars*. Optionally, diagnostic
|
|
29
|
+
plots are serialised to *plot-path*.
|
|
30
|
+
|
|
31
|
+
Notes
|
|
32
|
+
-----
|
|
33
|
+
**Command-line arguments**
|
|
34
|
+
|
|
35
|
+
``--raw-filelist`` : str
|
|
36
|
+
Path to a text file listing the raw LH5 input files (one per line).
|
|
37
|
+
``--database`` : str
|
|
38
|
+
Path to the existing DSP parameter database (JSON/YAML).
|
|
39
|
+
``--inplots`` : str, optional
|
|
40
|
+
Existing pickle plot file to update with new noise-optimisation plots.
|
|
41
|
+
``--log`` : str, optional
|
|
42
|
+
Path to the log file.
|
|
43
|
+
``--processing-chain`` : list of str
|
|
44
|
+
Processing chain configuration file(s).
|
|
45
|
+
``--config-file`` : list of str
|
|
46
|
+
Noise optimisation configuration file(s). Must contain ``run_nopt``
|
|
47
|
+
(bool), ``n_events`` (int), and ``cut_pars``.
|
|
48
|
+
``--log-config`` : str, optional
|
|
49
|
+
Logging configuration file.
|
|
50
|
+
``--raw-table-name`` : str
|
|
51
|
+
LH5 table path within the raw file.
|
|
52
|
+
``--dsp-pars`` : str
|
|
53
|
+
Output path for the merged DSP parameter database (JSON/YAML).
|
|
54
|
+
``--plot-path`` : str, optional
|
|
55
|
+
Output path for diagnostic plots (pickle).
|
|
56
|
+
"""
|
|
19
57
|
argparser = argparse.ArgumentParser()
|
|
20
58
|
argparser.add_argument("--raw-filelist", help="raw_filelist", type=str)
|
|
21
59
|
argparser.add_argument("--database", help="database", type=str, required=True)
|
|
@@ -63,7 +101,7 @@ def par_geds_dsp_nopt() -> None:
|
|
|
63
101
|
energies = lh5.read_as(
|
|
64
102
|
f"{args.raw_table_name}/daqenergy", raw_files, library="np"
|
|
65
103
|
)
|
|
66
|
-
idxs = np.where(energies
|
|
104
|
+
idxs = np.where(energies <= 10)[0]
|
|
67
105
|
tb_data = lh5.read(
|
|
68
106
|
args.raw_table_name, raw_files, n_rows=opt_dict["n_events"], idx=idxs
|
|
69
107
|
)
|
|
@@ -73,7 +111,7 @@ def par_geds_dsp_nopt() -> None:
|
|
|
73
111
|
|
|
74
112
|
msg = f"Select baselines {len(tb_data)}"
|
|
75
113
|
log.info(msg)
|
|
76
|
-
dsp_data =
|
|
114
|
+
dsp_data = build_dsp(raw_in=tb_data, dsp_config=dsp_config)
|
|
77
115
|
cut_dict = generate_cuts(dsp_data, cut_dict=opt_dict.pop("cut_pars"))
|
|
78
116
|
cut_idxs = get_cut_indexes(dsp_data, cut_dict)
|
|
79
117
|
tb_data = lh5.read(
|
|
@@ -7,9 +7,9 @@ from pathlib import Path
|
|
|
7
7
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
from dbetto.catalog import Props
|
|
10
|
+
from dspeed import build_dsp
|
|
10
11
|
from lgdo import lh5
|
|
11
12
|
from pygama.pargen.data_cleaning import get_cut_indexes
|
|
12
|
-
from pygama.pargen.dsp_optimize import run_one_dsp
|
|
13
13
|
from pygama.pargen.pz_correct import PZCorrect
|
|
14
14
|
|
|
15
15
|
from ....utils import (
|
|
@@ -20,6 +20,48 @@ from ....utils import (
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def par_geds_dsp_pz() -> None:
|
|
23
|
+
"""Determine the pole-zero (PZ) decay constant(s) for HPGe waveforms.
|
|
24
|
+
|
|
25
|
+
CLI entry point registered as ``par-geds-dsp-pz``. Selects high-energy
|
|
26
|
+
events from raw LH5 files (above an ADC threshold, excluding pulser and
|
|
27
|
+
discharge-recovery events), processes them through a preliminary DSP chain,
|
|
28
|
+
and uses :class:`pygama.pargen.pz_correct.PZCorrect` to fit either a
|
|
29
|
+
single- or double-exponential decay model to the waveform tails.
|
|
30
|
+
|
|
31
|
+
The fitted time constant(s) are written to *output-file* in JSON/YAML
|
|
32
|
+
format. Optionally, diagnostic waveform and slope plots are serialised to
|
|
33
|
+
a pickle file at *plot-path*.
|
|
34
|
+
|
|
35
|
+
Notes
|
|
36
|
+
-----
|
|
37
|
+
**Command-line arguments**
|
|
38
|
+
|
|
39
|
+
``--processing-chain`` : list of str
|
|
40
|
+
Processing chain configuration file(s).
|
|
41
|
+
``--config-file`` : list of str
|
|
42
|
+
PZ calibration configuration file(s). Must contain ``run_tau``
|
|
43
|
+
(bool), ``threshold`` (ADC), ``n_events`` (int), ``wf_field`` (str),
|
|
44
|
+
and ``mode`` (``"single"`` or ``"double"``).
|
|
45
|
+
``--log-config`` : str, optional
|
|
46
|
+
Logging configuration file.
|
|
47
|
+
``--raw-table-name`` : str
|
|
48
|
+
LH5 table path within the raw file (e.g. ``ch1057600/raw``).
|
|
49
|
+
``--plot-path`` : str, optional
|
|
50
|
+
Output path for the diagnostic pickle file.
|
|
51
|
+
``--output-file`` : str
|
|
52
|
+
Output path for the fitted PZ parameters (JSON/YAML).
|
|
53
|
+
``--pulser-file`` : str, optional
|
|
54
|
+
Path to the pulser mask file.
|
|
55
|
+
``-p`` / ``--no-pulse``
|
|
56
|
+
Flag indicating that no pulser is present; skips pulser masking.
|
|
57
|
+
``--raw-files`` : list of str
|
|
58
|
+
Raw LH5 input file(s).
|
|
59
|
+
``--pz-files`` : list of str, optional
|
|
60
|
+
Alternative input file(s) used instead of ``--raw-files`` when
|
|
61
|
+
present (e.g. preselected PZ calibration events).
|
|
62
|
+
``--log`` : str, optional
|
|
63
|
+
Path to the log file.
|
|
64
|
+
"""
|
|
23
65
|
argparser = argparse.ArgumentParser()
|
|
24
66
|
argparser.add_argument("--log", help="log file", type=str)
|
|
25
67
|
argparser.add_argument(
|
|
@@ -134,7 +176,7 @@ def par_geds_dsp_pz() -> None:
|
|
|
134
176
|
if "frac" in dsp_config["outputs"]:
|
|
135
177
|
dsp_config_optimise_removed["outputs"].remove("frac")
|
|
136
178
|
|
|
137
|
-
tb_out =
|
|
179
|
+
tb_out = build_dsp(raw_in=tb_data, dsp_config=dsp_config_optimise_removed)
|
|
138
180
|
log.debug("Processed Data")
|
|
139
181
|
cut_parameters = kwarg_dict.get("cut_parameters", None)
|
|
140
182
|
if cut_parameters is not None:
|