pytesprocess 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytesprocess/__init__.py +9 -0
- pytesprocess/_version.py +2 -0
- pytesprocess/cli/__init__.py +1 -0
- pytesprocess/cli/commands/__init__.py +5 -0
- pytesprocess/cli/commands/event.py +66 -0
- pytesprocess/cli/commands/filter.py +17 -0
- pytesprocess/cli/commands/ivsweep.py +29 -0
- pytesprocess/cli/common.py +86 -0
- pytesprocess/cli/main.py +81 -0
- pytesprocess/config/__init__.py +4 -0
- pytesprocess/config/loader.py +94 -0
- pytesprocess/config/manager.py +297 -0
- pytesprocess/config/resolvers/__init__.py +5 -0
- pytesprocess/config/resolvers/common.py +56 -0
- pytesprocess/config/resolvers/feature.py +293 -0
- pytesprocess/config/resolvers/salting.py +86 -0
- pytesprocess/config/resolvers/trigger.py +84 -0
- pytesprocess/config/selectors.py +108 -0
- pytesprocess/config/validation.py +314 -0
- pytesprocess/config/warnings.py +2 -0
- pytesprocess/core/__init__.py +10 -0
- pytesprocess/core/algorithms.py +1455 -0
- pytesprocess/core/didv.py +1648 -0
- pytesprocess/core/eventbuilder.py +495 -0
- pytesprocess/core/filterbuilder.py +81 -0
- pytesprocess/core/filterdata.py +1849 -0
- pytesprocess/core/ivsweep.py +2072 -0
- pytesprocess/core/noise.py +923 -0
- pytesprocess/core/noisemodel.py +1408 -0
- pytesprocess/core/oftrigger.py +1035 -0
- pytesprocess/core/template.py +450 -0
- pytesprocess/process/__init__.py +6 -0
- pytesprocess/process/data_source.py +185 -0
- pytesprocess/process/event_context.py +35 -0
- pytesprocess/process/feature_plan.py +186 -0
- pytesprocess/process/feature_resources.py +267 -0
- pytesprocess/process/features.py +1024 -0
- pytesprocess/process/filterprocess.py +1176 -0
- pytesprocess/process/ivprocess.py +1380 -0
- pytesprocess/process/processing_data.py +967 -0
- pytesprocess/process/randoms.py +921 -0
- pytesprocess/process/triggers.py +1011 -0
- pytesprocess/salting/__init__.py +7 -0
- pytesprocess/salting/generator.py +364 -0
- pytesprocess/salting/injector.py +329 -0
- pytesprocess/salting/sampling.py +84 -0
- pytesprocess/utils/__init__.py +5 -0
- pytesprocess/utils/arg_utils.py +122 -0
- pytesprocess/utils/dataframe_output.py +120 -0
- pytesprocess/utils/filter_hdf5.py +594 -0
- pytesprocess/utils/utils.py +701 -0
- pytesprocess/workflows/__init__.py +3 -0
- pytesprocess/workflows/processing.py +317 -0
- pytesprocess/workflows/salting.py +133 -0
- pytesprocess-0.1.1.dist-info/METADATA +211 -0
- pytesprocess-0.1.1.dist-info/RECORD +60 -0
- pytesprocess-0.1.1.dist-info/WHEEL +5 -0
- pytesprocess-0.1.1.dist-info/entry_points.txt +2 -0
- pytesprocess-0.1.1.dist-info/licenses/LICENSE +21 -0
- pytesprocess-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
from .loader import load_processing_yaml
|
|
7
|
+
from .warnings import ConfigDeprecationWarning
|
|
8
|
+
from .resolvers import FeatureConfigResolver, TriggerConfigResolver, SaltingConfigResolver
|
|
9
|
+
from .resolvers.common import rename_obsolete_keys
|
|
10
|
+
from .validation import (
|
|
11
|
+
normalize_workflows, validate_resolved_workflow,
|
|
12
|
+
format_validation_summary,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_LEGACY_TRIGGER_OVERALL = {'coincident_window_msec', 'coincident_window_samples'}
|
|
17
|
+
_LEGACY_SALTING_OVERALL = {
|
|
18
|
+
'dm_pdf_file', 'coincident_salts', 'energies', 'nsalt', 'do_salt_deadtime',
|
|
19
|
+
'random_seed'
|
|
20
|
+
}
|
|
21
|
+
_RESERVED_TOP_LEVEL = {
|
|
22
|
+
'config_version', 'resources', 'trigger', 'feature', 'salting', 'filter',
|
|
23
|
+
'ivsweep', 'didv', 'noise', 'template', 'global', 'filter_file', 'didv_file'
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ProcessingConfig:
|
|
28
|
+
"""Load processing YAML once and resolve each workflow lazily.
|
|
29
|
+
|
|
30
|
+
The returned workflow dictionaries intentionally retain the runtime shape
|
|
31
|
+
expected by the existing TriggerProcessing/FeatureProcessing code. This
|
|
32
|
+
lets the public YAML evolve independently from processing internals.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, yaml_file, available_channels, sample_rate=None,
|
|
36
|
+
verbose=True):
|
|
37
|
+
self._yaml_file = yaml_file
|
|
38
|
+
self._available_channels = ([available_channels]
|
|
39
|
+
if isinstance(available_channels, str)
|
|
40
|
+
else list(available_channels or []))
|
|
41
|
+
self._sample_rate = sample_rate
|
|
42
|
+
self._verbose = bool(verbose)
|
|
43
|
+
raw, version, path = load_processing_yaml(yaml_file)
|
|
44
|
+
self._version = int(version)
|
|
45
|
+
self._path = path
|
|
46
|
+
self._raw = rename_obsolete_keys(raw, self._version)
|
|
47
|
+
self._canonical = self._normalize(self._raw)
|
|
48
|
+
self._cache = {}
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def config_version(self):
|
|
52
|
+
return self._version
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def path(self):
|
|
56
|
+
return str(self._path)
|
|
57
|
+
|
|
58
|
+
def validate(self, workflows=None, check_resources=False, display=False):
|
|
59
|
+
"""Resolve and validate requested workflow configurations up front.
|
|
60
|
+
|
|
61
|
+
Parameters
|
|
62
|
+
----------
|
|
63
|
+
workflows : str or iterable of str, optional
|
|
64
|
+
Workflows to validate (for example ``['trigger', 'feature']``).
|
|
65
|
+
If omitted, validate every workflow that is configured in the YAML.
|
|
66
|
+
check_resources : bool, optional
|
|
67
|
+
If True, also check that configured input files such as
|
|
68
|
+
``filter_file`` exist. FilterData contents and raw-data-dependent
|
|
69
|
+
checks remain runtime validation.
|
|
70
|
+
display : bool, optional
|
|
71
|
+
Print a compact validation summary.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
dict
|
|
76
|
+
Validation summary suitable for a future CLI startup report.
|
|
77
|
+
|
|
78
|
+
Notes
|
|
79
|
+
-----
|
|
80
|
+
This method performs no processing and intentionally is not called by
|
|
81
|
+
``scripts/process.py`` yet. It lets a future multi-step CLI validate
|
|
82
|
+
all requested workflows before starting the first processing stage.
|
|
83
|
+
"""
|
|
84
|
+
names = normalize_workflows(workflows)
|
|
85
|
+
if names is None:
|
|
86
|
+
names = self._configured_workflows()
|
|
87
|
+
|
|
88
|
+
summary = {
|
|
89
|
+
'valid': True,
|
|
90
|
+
'config_version': self._version,
|
|
91
|
+
'config_file': str(self._path),
|
|
92
|
+
'workflows': {},
|
|
93
|
+
}
|
|
94
|
+
for name in names:
|
|
95
|
+
resolved = self.get_config(name)
|
|
96
|
+
summary['workflows'][name] = validate_resolved_workflow(
|
|
97
|
+
name, resolved, check_resources=bool(check_resources))
|
|
98
|
+
|
|
99
|
+
if display:
|
|
100
|
+
print(format_validation_summary(summary))
|
|
101
|
+
return copy.deepcopy(summary)
|
|
102
|
+
|
|
103
|
+
def _configured_workflows(self):
|
|
104
|
+
names = []
|
|
105
|
+
for name in ('trigger', 'feature', 'salting', 'filter', 'ivsweep',
|
|
106
|
+
'didv', 'noise', 'template'):
|
|
107
|
+
section = self._canonical.get(name, {})
|
|
108
|
+
if not isinstance(section, dict):
|
|
109
|
+
continue
|
|
110
|
+
if name in {'trigger', 'salting'}:
|
|
111
|
+
configured = bool(section.get('global') or section.get('channels'))
|
|
112
|
+
elif name == 'feature':
|
|
113
|
+
configured = bool(
|
|
114
|
+
section.get('global') or section.get('presets') or
|
|
115
|
+
section.get('channels'))
|
|
116
|
+
else:
|
|
117
|
+
configured = bool(section)
|
|
118
|
+
if configured:
|
|
119
|
+
names.append(name)
|
|
120
|
+
return names
|
|
121
|
+
|
|
122
|
+
def get_config(self, processing_type=None):
|
|
123
|
+
if processing_type is None:
|
|
124
|
+
out = {
|
|
125
|
+
'global': copy.deepcopy(self._canonical['resources'])
|
|
126
|
+
}
|
|
127
|
+
for name in ('trigger', 'feature', 'salting', 'filter', 'ivsweep',
|
|
128
|
+
'didv', 'noise', 'template'):
|
|
129
|
+
out[name] = self.get_config(name)
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
name = str(processing_type).lower()
|
|
133
|
+
if name in self._cache:
|
|
134
|
+
return copy.deepcopy(self._cache[name])
|
|
135
|
+
|
|
136
|
+
if name == 'feature':
|
|
137
|
+
resolved = FeatureConfigResolver(
|
|
138
|
+
self._canonical['feature'],
|
|
139
|
+
self._canonical['resources'],
|
|
140
|
+
self._available_channels,
|
|
141
|
+
sample_rate_hz=self._sample_rate,
|
|
142
|
+
version=self._version,
|
|
143
|
+
).resolve()
|
|
144
|
+
elif name == 'trigger':
|
|
145
|
+
resolved = TriggerConfigResolver(
|
|
146
|
+
self._canonical['trigger'], self._canonical['resources'],
|
|
147
|
+
self._available_channels, version=self._version,
|
|
148
|
+
).resolve()
|
|
149
|
+
elif name == 'salting':
|
|
150
|
+
resolved = SaltingConfigResolver(
|
|
151
|
+
self._canonical['salting'], self._canonical['resources'],
|
|
152
|
+
self._available_channels, version=self._version,
|
|
153
|
+
).resolve()
|
|
154
|
+
elif name in {'filter', 'ivsweep'}:
|
|
155
|
+
resolved = copy.deepcopy(self._canonical.get(name, {}))
|
|
156
|
+
elif name in {'didv', 'noise', 'template'}:
|
|
157
|
+
resolved = self._resolve_generic_legacy_section(name)
|
|
158
|
+
else:
|
|
159
|
+
raise ValueError(f'ERROR: Configuration type "{processing_type}" not found!')
|
|
160
|
+
|
|
161
|
+
self._cache[name] = copy.deepcopy(resolved)
|
|
162
|
+
return copy.deepcopy(resolved)
|
|
163
|
+
|
|
164
|
+
def _normalize(self, raw):
|
|
165
|
+
if self._version >= 2:
|
|
166
|
+
return self._normalize_v2(raw)
|
|
167
|
+
return self._normalize_legacy(raw)
|
|
168
|
+
|
|
169
|
+
def _normalize_v2(self, raw):
|
|
170
|
+
allowed = {
|
|
171
|
+
'config_version', 'resources', 'trigger', 'feature', 'salting',
|
|
172
|
+
'filter', 'ivsweep', 'didv', 'noise', 'template'
|
|
173
|
+
}
|
|
174
|
+
unknown = set(raw) - allowed
|
|
175
|
+
if unknown:
|
|
176
|
+
raise ValueError(
|
|
177
|
+
f'ERROR: Unknown top-level config_version: 2 field(s): {sorted(unknown)}. '
|
|
178
|
+
f'Allowed fields are {sorted(allowed)}.'
|
|
179
|
+
)
|
|
180
|
+
resources = copy.deepcopy(raw.get('resources', {}))
|
|
181
|
+
if not isinstance(resources, dict):
|
|
182
|
+
raise ValueError('ERROR: "resources" must be a mapping.')
|
|
183
|
+
return {
|
|
184
|
+
'resources': resources,
|
|
185
|
+
'trigger': self._normalize_v2_workflow(raw.get('trigger', {}), 'trigger'),
|
|
186
|
+
'feature': copy.deepcopy(raw.get('feature', {})),
|
|
187
|
+
'salting': self._normalize_v2_workflow(raw.get('salting', {}), 'salting'),
|
|
188
|
+
'filter': copy.deepcopy(raw.get('filter', {})),
|
|
189
|
+
'ivsweep': copy.deepcopy(raw.get('ivsweep', {})),
|
|
190
|
+
'didv': copy.deepcopy(raw.get('didv', {})),
|
|
191
|
+
'noise': copy.deepcopy(raw.get('noise', {})),
|
|
192
|
+
'template': copy.deepcopy(raw.get('template', {})),
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
def _normalize_v2_workflow(self, section, name):
|
|
196
|
+
if section is None:
|
|
197
|
+
return {'global': {}, 'channels': {}}
|
|
198
|
+
if not isinstance(section, dict):
|
|
199
|
+
raise ValueError(f'ERROR: "{name}" must be a mapping.')
|
|
200
|
+
allowed = {'global', 'channels'}
|
|
201
|
+
unknown = set(section) - allowed
|
|
202
|
+
if unknown:
|
|
203
|
+
raise ValueError(
|
|
204
|
+
f'ERROR: config_version: 2 requires {name} settings under '
|
|
205
|
+
f'"{name}.global" or "{name}.channels". Unknown: {sorted(unknown)}'
|
|
206
|
+
)
|
|
207
|
+
return {
|
|
208
|
+
'global': copy.deepcopy(section.get('global', {})),
|
|
209
|
+
'channels': copy.deepcopy(section.get('channels', {})),
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
def _normalize_legacy(self, raw):
|
|
213
|
+
warnings.warn(
|
|
214
|
+
'Legacy processing YAML syntax is deprecated. Add "config_version: 2" '
|
|
215
|
+
'and move feature channel configuration under "feature.channels".',
|
|
216
|
+
ConfigDeprecationWarning,
|
|
217
|
+
stacklevel=3,
|
|
218
|
+
)
|
|
219
|
+
work = copy.deepcopy(raw)
|
|
220
|
+
resources = {}
|
|
221
|
+
for key in ('filter_file', 'didv_file'):
|
|
222
|
+
if key in work:
|
|
223
|
+
resources[key] = work.pop(key)
|
|
224
|
+
|
|
225
|
+
trigger_raw = work.pop('trigger', {}) or {}
|
|
226
|
+
salting_raw = work.pop('salting', {}) or {}
|
|
227
|
+
explicit_feature = work.pop('feature', {}) or {}
|
|
228
|
+
filter_cfg = work.pop('filter', {}) or {}
|
|
229
|
+
ivsweep_cfg = work.pop('ivsweep', {}) or {}
|
|
230
|
+
didv_cfg = work.pop('didv', {}) or {}
|
|
231
|
+
noise_cfg = work.pop('noise', {}) or {}
|
|
232
|
+
template_cfg = work.pop('template', {}) or {}
|
|
233
|
+
global_feature = work.pop('global', {}) or {}
|
|
234
|
+
work.pop('config_version', None)
|
|
235
|
+
work.pop('resources', None)
|
|
236
|
+
|
|
237
|
+
trigger = self._split_legacy_section(trigger_raw, _LEGACY_TRIGGER_OVERALL)
|
|
238
|
+
salting = self._split_legacy_section(salting_raw, _LEGACY_SALTING_OVERALL)
|
|
239
|
+
|
|
240
|
+
feature = {'global': {}, 'channels': {}}
|
|
241
|
+
if isinstance(explicit_feature, dict):
|
|
242
|
+
if 'global' in explicit_feature:
|
|
243
|
+
feature['global'].update(copy.deepcopy(explicit_feature['global']))
|
|
244
|
+
for key, value in explicit_feature.items():
|
|
245
|
+
if key != 'global':
|
|
246
|
+
feature['channels'][key] = copy.deepcopy(value)
|
|
247
|
+
feature['global'].update(copy.deepcopy(global_feature))
|
|
248
|
+
|
|
249
|
+
# Any remaining legacy root key is a feature channel/expression.
|
|
250
|
+
for key, value in work.items():
|
|
251
|
+
if key in _RESERVED_TOP_LEVEL:
|
|
252
|
+
continue
|
|
253
|
+
feature['channels'][key] = copy.deepcopy(value)
|
|
254
|
+
|
|
255
|
+
return {
|
|
256
|
+
'resources': resources,
|
|
257
|
+
'trigger': trigger,
|
|
258
|
+
'feature': feature,
|
|
259
|
+
'salting': salting,
|
|
260
|
+
'filter': filter_cfg,
|
|
261
|
+
'ivsweep': ivsweep_cfg,
|
|
262
|
+
'didv': didv_cfg,
|
|
263
|
+
'noise': noise_cfg,
|
|
264
|
+
'template': template_cfg,
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
@staticmethod
|
|
268
|
+
def _split_legacy_section(section, overall_keys):
|
|
269
|
+
if section is None:
|
|
270
|
+
return {'global': {}, 'channels': {}}
|
|
271
|
+
if not isinstance(section, dict):
|
|
272
|
+
raise ValueError('ERROR: Legacy workflow section must be a mapping.')
|
|
273
|
+
global_cfg = {}
|
|
274
|
+
channels = {}
|
|
275
|
+
for key, value in section.items():
|
|
276
|
+
if key in overall_keys:
|
|
277
|
+
global_cfg[key] = copy.deepcopy(value)
|
|
278
|
+
elif key == 'global' and isinstance(value, dict):
|
|
279
|
+
global_cfg.update(copy.deepcopy(value))
|
|
280
|
+
elif key == 'channels' and isinstance(value, dict):
|
|
281
|
+
channels.update(copy.deepcopy(value))
|
|
282
|
+
else:
|
|
283
|
+
channels[key] = copy.deepcopy(value)
|
|
284
|
+
return {'global': global_cfg, 'channels': channels}
|
|
285
|
+
|
|
286
|
+
def _resolve_generic_legacy_section(self, name):
|
|
287
|
+
section = copy.deepcopy(self._canonical.get(name, {}))
|
|
288
|
+
if not section:
|
|
289
|
+
return {'overall': {}, 'channels': {}}
|
|
290
|
+
if isinstance(section, dict) and ('global' in section or 'channels' in section):
|
|
291
|
+
return {
|
|
292
|
+
'overall': copy.deepcopy(section.get('global', {})),
|
|
293
|
+
'channels': copy.deepcopy(section.get('channels', {})),
|
|
294
|
+
}
|
|
295
|
+
# Preserve the old generic behavior: arbitrary section keys were treated
|
|
296
|
+
# as channel configurations.
|
|
297
|
+
return {'overall': {}, 'channels': section if isinstance(section, dict) else {}}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
from ..warnings import ConfigDeprecationWarning
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
LEGACY_OBSOLETE_KEYS = {
|
|
10
|
+
'trigger_name': 'trigger_channel',
|
|
11
|
+
'nb_samples': 'trace_length_samples',
|
|
12
|
+
'nb_pretrigger_samples': 'pretrigger_length_samples',
|
|
13
|
+
'template_time_tags': 'template_group_ids',
|
|
14
|
+
'psd_tag': 'csd_tag',
|
|
15
|
+
'noise_tag': 'csd_tag',
|
|
16
|
+
'deadtime_salt': 'do_salt_deadtime',
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
V2_OBSOLETE_KEYS = {
|
|
20
|
+
'trigger_name': 'trigger_channel',
|
|
21
|
+
'template_time_tags': 'template_group_ids',
|
|
22
|
+
'noise_tag': 'csd_tag',
|
|
23
|
+
'deadtime_salt': 'do_salt_deadtime',
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def deep_merge(base, override):
|
|
28
|
+
out = copy.deepcopy(base)
|
|
29
|
+
for key, value in (override or {}).items():
|
|
30
|
+
if key in out and isinstance(out[key], dict) and isinstance(value, dict):
|
|
31
|
+
out[key] = deep_merge(out[key], value)
|
|
32
|
+
else:
|
|
33
|
+
out[key] = copy.deepcopy(value)
|
|
34
|
+
return out
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def rename_obsolete_keys(data, version, _warned=None):
|
|
38
|
+
if _warned is None:
|
|
39
|
+
_warned = set()
|
|
40
|
+
if isinstance(data, dict):
|
|
41
|
+
mapping = LEGACY_OBSOLETE_KEYS if version < 2 else V2_OBSOLETE_KEYS
|
|
42
|
+
out = {}
|
|
43
|
+
for key, value in data.items():
|
|
44
|
+
new_key = mapping.get(key, key)
|
|
45
|
+
if new_key != key and key not in _warned:
|
|
46
|
+
warnings.warn(
|
|
47
|
+
f'Configuration key "{key}" is deprecated; use "{new_key}".',
|
|
48
|
+
ConfigDeprecationWarning,
|
|
49
|
+
stacklevel=4,
|
|
50
|
+
)
|
|
51
|
+
_warned.add(key)
|
|
52
|
+
out[new_key] = rename_obsolete_keys(value, version, _warned)
|
|
53
|
+
return out
|
|
54
|
+
if isinstance(data, list):
|
|
55
|
+
return [rename_obsolete_keys(item, version, _warned) for item in data]
|
|
56
|
+
return copy.deepcopy(data)
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
from ..selectors import expand_channel_selector
|
|
7
|
+
from ..warnings import ConfigDeprecationWarning
|
|
8
|
+
from .common import deep_merge
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_TRACE_KEYS_MSEC = ('trace_length_msec', 'pretrigger_length_msec')
|
|
12
|
+
_TRACE_KEYS_SAMPLES = ('trace_length_samples', 'pretrigger_length_samples')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class FeatureConfigResolver:
|
|
16
|
+
"""Resolve feature configuration into the legacy runtime dictionary shape."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, config, resources, available_channels, sample_rate_hz=None,
|
|
19
|
+
version=1):
|
|
20
|
+
self._config = copy.deepcopy(config or {})
|
|
21
|
+
self._resources = copy.deepcopy(resources or {})
|
|
22
|
+
self._available_channels = list(available_channels or [])
|
|
23
|
+
self._sample_rate_hz = sample_rate_hz
|
|
24
|
+
self._version = int(version)
|
|
25
|
+
self._legacy_sample_warning_emitted = False
|
|
26
|
+
|
|
27
|
+
def resolve(self):
|
|
28
|
+
if self._version >= 2:
|
|
29
|
+
return self._resolve_v2()
|
|
30
|
+
return self._resolve_legacy()
|
|
31
|
+
|
|
32
|
+
def _resolve_legacy(self):
|
|
33
|
+
cfg = copy.deepcopy(self._config)
|
|
34
|
+
overall = copy.deepcopy(cfg.get('global', {}))
|
|
35
|
+
channels = copy.deepcopy(cfg.get('channels', {}))
|
|
36
|
+
for key, value in self._resources.items():
|
|
37
|
+
overall.setdefault(key, value)
|
|
38
|
+
|
|
39
|
+
warnings.warn(
|
|
40
|
+
'Legacy feature configuration is deprecated. Move feature settings '
|
|
41
|
+
'under a top-level "feature:" section and use config_version: 2.',
|
|
42
|
+
ConfigDeprecationWarning,
|
|
43
|
+
stacklevel=3,
|
|
44
|
+
)
|
|
45
|
+
return self._resolve_channels(overall, channels, presets={}, legacy=True)
|
|
46
|
+
|
|
47
|
+
def _resolve_v2(self):
|
|
48
|
+
allowed = {'global', 'presets', 'channels'}
|
|
49
|
+
unknown = set(self._config) - allowed
|
|
50
|
+
if unknown:
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f'ERROR: Unknown feature configuration field(s): {sorted(unknown)}. '
|
|
53
|
+
f'Allowed fields are {sorted(allowed)}.'
|
|
54
|
+
)
|
|
55
|
+
overall = copy.deepcopy(self._config.get('global', {}))
|
|
56
|
+
channels = copy.deepcopy(self._config.get('channels', {}))
|
|
57
|
+
presets = copy.deepcopy(self._config.get('presets', {}))
|
|
58
|
+
for key, value in self._resources.items():
|
|
59
|
+
overall.setdefault(key, value)
|
|
60
|
+
|
|
61
|
+
self._reject_v2_sample_lengths(overall, 'feature.global')
|
|
62
|
+
for name, preset in presets.items():
|
|
63
|
+
self._reject_v2_sample_lengths(preset, f'feature.presets.{name}')
|
|
64
|
+
for selector, block in channels.items():
|
|
65
|
+
self._reject_v2_sample_lengths(block, f'feature.channels.{selector}')
|
|
66
|
+
return self._resolve_channels(overall, channels, presets=presets, legacy=False)
|
|
67
|
+
|
|
68
|
+
def _reject_v2_sample_lengths(self, obj, path):
|
|
69
|
+
if not isinstance(obj, dict):
|
|
70
|
+
return
|
|
71
|
+
for key, value in obj.items():
|
|
72
|
+
if key in _TRACE_KEYS_SAMPLES:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f'ERROR: "{path}.{key}" is not allowed in config_version: 2. '
|
|
75
|
+
'Specify trace and pretrigger lengths in milliseconds.'
|
|
76
|
+
)
|
|
77
|
+
if isinstance(value, dict):
|
|
78
|
+
self._reject_v2_sample_lengths(value, f'{path}.{key}')
|
|
79
|
+
|
|
80
|
+
def _normalize_block(self, block, legacy=False):
|
|
81
|
+
if block is None:
|
|
82
|
+
block = {}
|
|
83
|
+
if not isinstance(block, dict):
|
|
84
|
+
raise ValueError('ERROR: Feature channel/preset configuration must be a mapping.')
|
|
85
|
+
block = copy.deepcopy(block)
|
|
86
|
+
|
|
87
|
+
if legacy:
|
|
88
|
+
algorithms = {}
|
|
89
|
+
meta = {}
|
|
90
|
+
for key, value in block.items():
|
|
91
|
+
if isinstance(value, dict):
|
|
92
|
+
algorithms[key] = value
|
|
93
|
+
else:
|
|
94
|
+
meta[key] = value
|
|
95
|
+
meta['algorithms'] = algorithms
|
|
96
|
+
return meta
|
|
97
|
+
|
|
98
|
+
algorithms = block.get('algorithms', {})
|
|
99
|
+
if algorithms is None:
|
|
100
|
+
algorithms = {}
|
|
101
|
+
if not isinstance(algorithms, dict):
|
|
102
|
+
raise ValueError('ERROR: feature channel "algorithms" must be a mapping.')
|
|
103
|
+
out = {key: copy.deepcopy(value) for key, value in block.items()
|
|
104
|
+
if key != 'algorithms'}
|
|
105
|
+
out['algorithms'] = copy.deepcopy(algorithms)
|
|
106
|
+
return out
|
|
107
|
+
|
|
108
|
+
def _apply_presets(self, block, presets):
|
|
109
|
+
use = block.pop('use', None)
|
|
110
|
+
if use is None:
|
|
111
|
+
return block
|
|
112
|
+
names = [use] if isinstance(use, str) else list(use)
|
|
113
|
+
merged = {}
|
|
114
|
+
for name in names:
|
|
115
|
+
if name not in presets:
|
|
116
|
+
raise ValueError(f'ERROR: Unknown feature preset "{name}".')
|
|
117
|
+
preset = self._normalize_block(presets[name], legacy=False)
|
|
118
|
+
merged = deep_merge(merged, preset)
|
|
119
|
+
return deep_merge(merged, block)
|
|
120
|
+
|
|
121
|
+
def _resolve_channels(self, overall, channel_blocks, presets, legacy):
|
|
122
|
+
if not isinstance(channel_blocks, dict):
|
|
123
|
+
raise ValueError('ERROR: feature.channels must be a mapping.')
|
|
124
|
+
|
|
125
|
+
# Expand selectors, then apply from least to most specific. Equal-
|
|
126
|
+
# specificity overlap is rejected to avoid order-dependent YAML.
|
|
127
|
+
assignments = {}
|
|
128
|
+
for selector, raw_block in channel_blocks.items():
|
|
129
|
+
block = self._normalize_block(raw_block, legacy=legacy)
|
|
130
|
+
if not legacy:
|
|
131
|
+
block = self._apply_presets(block, presets)
|
|
132
|
+
for match in expand_channel_selector(selector, self._available_channels):
|
|
133
|
+
assignments.setdefault(match.target, []).append((match.specificity, selector, block,
|
|
134
|
+
match.physical_channels))
|
|
135
|
+
|
|
136
|
+
resolved_channels = {}
|
|
137
|
+
physical_channel_list = []
|
|
138
|
+
for target, items in assignments.items():
|
|
139
|
+
items = sorted(items, key=lambda item: item[0])
|
|
140
|
+
seen_specificity = {}
|
|
141
|
+
merged = {}
|
|
142
|
+
physical = None
|
|
143
|
+
for specificity, selector, block, physical_channels in items:
|
|
144
|
+
if specificity in seen_specificity:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
f'ERROR: Feature selectors "{seen_specificity[specificity]}" and '
|
|
147
|
+
f'"{selector}" both match "{target}" with the same precedence. '
|
|
148
|
+
'Use a more specific selector or combine the configuration.'
|
|
149
|
+
)
|
|
150
|
+
seen_specificity[specificity] = selector
|
|
151
|
+
merged = deep_merge(merged, block)
|
|
152
|
+
physical = physical_channels
|
|
153
|
+
|
|
154
|
+
if merged.get('disable', False) or merged.get('run') is False:
|
|
155
|
+
continue
|
|
156
|
+
merged.pop('disable', None)
|
|
157
|
+
merged.pop('run', None)
|
|
158
|
+
physical_channel_list.extend(physical or [])
|
|
159
|
+
|
|
160
|
+
resolved = self._resolve_one_channel(target, overall, merged, legacy)
|
|
161
|
+
if resolved is not None:
|
|
162
|
+
resolved_channels[target] = resolved
|
|
163
|
+
|
|
164
|
+
# Build runtime trace map + optional channel weights.
|
|
165
|
+
traces_config = {}
|
|
166
|
+
weights = {}
|
|
167
|
+
for channel, channel_cfg in resolved_channels.items():
|
|
168
|
+
physical = expand_channel_selector(channel, self._available_channels)[0].physical_channels
|
|
169
|
+
for key, value in channel_cfg.items():
|
|
170
|
+
if key.startswith('weight_'):
|
|
171
|
+
weights.setdefault(channel, {})[key] = value
|
|
172
|
+
for algo, algo_cfg in channel_cfg.items():
|
|
173
|
+
if not isinstance(algo_cfg, dict):
|
|
174
|
+
continue
|
|
175
|
+
trace_tuple = (algo_cfg['nb_samples'], algo_cfg['nb_pretrigger_samples'])
|
|
176
|
+
traces_config.setdefault(trace_tuple, []).extend(physical)
|
|
177
|
+
|
|
178
|
+
for trace_tuple in list(traces_config):
|
|
179
|
+
traces_config[trace_tuple] = list(dict.fromkeys(traces_config[trace_tuple]))
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
'overall': copy.deepcopy(overall),
|
|
183
|
+
'channels': resolved_channels,
|
|
184
|
+
'channel_list': list(dict.fromkeys(physical_channel_list)),
|
|
185
|
+
'traces_config': traces_config or None,
|
|
186
|
+
'weights': weights,
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
def _duration_pair(self, source, inherited=(None, None), legacy=False, path=''):
|
|
190
|
+
trace, pre = inherited
|
|
191
|
+
if not isinstance(source, dict):
|
|
192
|
+
return trace, pre
|
|
193
|
+
|
|
194
|
+
if legacy:
|
|
195
|
+
if 'trace_length_samples' in source:
|
|
196
|
+
if not self._legacy_sample_warning_emitted:
|
|
197
|
+
warnings.warn(
|
|
198
|
+
'Sample-count trace configuration is deprecated; use '
|
|
199
|
+
'trace_length_msec and pretrigger_length_msec.',
|
|
200
|
+
ConfigDeprecationWarning, stacklevel=5)
|
|
201
|
+
self._legacy_sample_warning_emitted = True
|
|
202
|
+
trace = ('samples', int(source['trace_length_samples']))
|
|
203
|
+
elif 'trace_length_msec' in source:
|
|
204
|
+
trace = ('msec', float(source['trace_length_msec']))
|
|
205
|
+
if 'pretrigger_length_samples' in source:
|
|
206
|
+
if not self._legacy_sample_warning_emitted:
|
|
207
|
+
warnings.warn(
|
|
208
|
+
'Sample-count trace configuration is deprecated; use '
|
|
209
|
+
'trace_length_msec and pretrigger_length_msec.',
|
|
210
|
+
ConfigDeprecationWarning, stacklevel=5)
|
|
211
|
+
self._legacy_sample_warning_emitted = True
|
|
212
|
+
pre = ('samples', int(source['pretrigger_length_samples']))
|
|
213
|
+
elif 'pretrigger_length_msec' in source:
|
|
214
|
+
pre = ('msec', float(source['pretrigger_length_msec']))
|
|
215
|
+
else:
|
|
216
|
+
if 'trace_length_msec' in source:
|
|
217
|
+
trace = ('msec', float(source['trace_length_msec']))
|
|
218
|
+
if 'pretrigger_length_msec' in source:
|
|
219
|
+
pre = ('msec', float(source['pretrigger_length_msec']))
|
|
220
|
+
return trace, pre
|
|
221
|
+
|
|
222
|
+
def _to_samples(self, value, label):
|
|
223
|
+
if value is None:
|
|
224
|
+
return None
|
|
225
|
+
units, number = value
|
|
226
|
+
if units == 'samples':
|
|
227
|
+
return int(number)
|
|
228
|
+
if self._sample_rate_hz is None:
|
|
229
|
+
raise ValueError(
|
|
230
|
+
f'ERROR: sample_rate_hz is required to resolve {label} specified in msec.'
|
|
231
|
+
)
|
|
232
|
+
try:
|
|
233
|
+
fs = float(self._sample_rate_hz)
|
|
234
|
+
except (TypeError, ValueError):
|
|
235
|
+
raise ValueError(
|
|
236
|
+
f'ERROR: Feature processing currently requires one sample rate, got '
|
|
237
|
+
f'{self._sample_rate_hz!r}. Millisecond configuration is retained; '
|
|
238
|
+
'per-stream sample-rate resolution will be handled in the later '
|
|
239
|
+
'ProcessingData refactor.'
|
|
240
|
+
)
|
|
241
|
+
return int(round(fs * float(number) / 1000.0))
|
|
242
|
+
|
|
243
|
+
def _resolve_one_channel(self, channel, overall, block, legacy):
|
|
244
|
+
global_pair = self._duration_pair(overall, legacy=legacy, path='feature.global')
|
|
245
|
+
channel_pair = self._duration_pair(
|
|
246
|
+
block, inherited=global_pair, legacy=legacy, path=f'feature.channels.{channel}')
|
|
247
|
+
|
|
248
|
+
algorithms = copy.deepcopy(block.get('algorithms', {}))
|
|
249
|
+
result = {key: copy.deepcopy(value) for key, value in block.items()
|
|
250
|
+
if key not in {'algorithms', *_TRACE_KEYS_MSEC, *_TRACE_KEYS_SAMPLES}}
|
|
251
|
+
|
|
252
|
+
enabled = 0
|
|
253
|
+
for algo, algo_cfg in algorithms.items():
|
|
254
|
+
if algo_cfg is None:
|
|
255
|
+
algo_cfg = {}
|
|
256
|
+
if not isinstance(algo_cfg, dict):
|
|
257
|
+
raise ValueError(
|
|
258
|
+
f'ERROR: Feature algorithm "{algo}" for channel "{channel}" '
|
|
259
|
+
'must be a mapping.'
|
|
260
|
+
)
|
|
261
|
+
algo_cfg = copy.deepcopy(algo_cfg)
|
|
262
|
+
# Missing ``run`` means enabled for both legacy and v2 YAML.
|
|
263
|
+
# Users only need to specify ``run: false`` to disable an
|
|
264
|
+
# algorithm explicitly.
|
|
265
|
+
run = algo_cfg.get('run', True)
|
|
266
|
+
if not run:
|
|
267
|
+
continue
|
|
268
|
+
algo_cfg['run'] = True
|
|
269
|
+
|
|
270
|
+
pair = self._duration_pair(
|
|
271
|
+
algo_cfg, inherited=channel_pair, legacy=legacy,
|
|
272
|
+
path=f'feature.channels.{channel}.algorithms.{algo}')
|
|
273
|
+
nb_samples = self._to_samples(pair[0], f'{channel}/{algo} trace length')
|
|
274
|
+
nb_pre = self._to_samples(pair[1], f'{channel}/{algo} pretrigger length')
|
|
275
|
+
if (nb_samples is None) != (nb_pre is None):
|
|
276
|
+
raise ValueError(
|
|
277
|
+
f'ERROR: Trace length and pretrigger length must both be defined '
|
|
278
|
+
f'for channel {channel}, algorithm {algo}.'
|
|
279
|
+
)
|
|
280
|
+
if nb_samples is not None:
|
|
281
|
+
if nb_samples <= 0 or nb_pre < 0 or nb_pre > nb_samples:
|
|
282
|
+
raise ValueError(
|
|
283
|
+
f'ERROR: Invalid trace/pretrigger lengths for {channel}/{algo}: '
|
|
284
|
+
f'{nb_samples}/{nb_pre} samples.'
|
|
285
|
+
)
|
|
286
|
+
algo_cfg['nb_samples'] = nb_samples
|
|
287
|
+
algo_cfg['nb_pretrigger_samples'] = nb_pre
|
|
288
|
+
for key in (*_TRACE_KEYS_MSEC, *_TRACE_KEYS_SAMPLES):
|
|
289
|
+
algo_cfg.pop(key, None)
|
|
290
|
+
result[algo] = algo_cfg
|
|
291
|
+
enabled += 1
|
|
292
|
+
|
|
293
|
+
return result if enabled else None
|