pytesprocess 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytesprocess/__init__.py +9 -0
- pytesprocess/_version.py +2 -0
- pytesprocess/cli/__init__.py +1 -0
- pytesprocess/cli/commands/__init__.py +5 -0
- pytesprocess/cli/commands/event.py +66 -0
- pytesprocess/cli/commands/filter.py +17 -0
- pytesprocess/cli/commands/ivsweep.py +29 -0
- pytesprocess/cli/common.py +86 -0
- pytesprocess/cli/main.py +81 -0
- pytesprocess/config/__init__.py +4 -0
- pytesprocess/config/loader.py +94 -0
- pytesprocess/config/manager.py +297 -0
- pytesprocess/config/resolvers/__init__.py +5 -0
- pytesprocess/config/resolvers/common.py +56 -0
- pytesprocess/config/resolvers/feature.py +293 -0
- pytesprocess/config/resolvers/salting.py +86 -0
- pytesprocess/config/resolvers/trigger.py +84 -0
- pytesprocess/config/selectors.py +108 -0
- pytesprocess/config/validation.py +314 -0
- pytesprocess/config/warnings.py +2 -0
- pytesprocess/core/__init__.py +10 -0
- pytesprocess/core/algorithms.py +1455 -0
- pytesprocess/core/didv.py +1648 -0
- pytesprocess/core/eventbuilder.py +495 -0
- pytesprocess/core/filterbuilder.py +81 -0
- pytesprocess/core/filterdata.py +1849 -0
- pytesprocess/core/ivsweep.py +2072 -0
- pytesprocess/core/noise.py +923 -0
- pytesprocess/core/noisemodel.py +1408 -0
- pytesprocess/core/oftrigger.py +1035 -0
- pytesprocess/core/template.py +450 -0
- pytesprocess/process/__init__.py +6 -0
- pytesprocess/process/data_source.py +185 -0
- pytesprocess/process/event_context.py +35 -0
- pytesprocess/process/feature_plan.py +186 -0
- pytesprocess/process/feature_resources.py +267 -0
- pytesprocess/process/features.py +1024 -0
- pytesprocess/process/filterprocess.py +1176 -0
- pytesprocess/process/ivprocess.py +1380 -0
- pytesprocess/process/processing_data.py +967 -0
- pytesprocess/process/randoms.py +921 -0
- pytesprocess/process/triggers.py +1011 -0
- pytesprocess/salting/__init__.py +7 -0
- pytesprocess/salting/generator.py +364 -0
- pytesprocess/salting/injector.py +329 -0
- pytesprocess/salting/sampling.py +84 -0
- pytesprocess/utils/__init__.py +5 -0
- pytesprocess/utils/arg_utils.py +122 -0
- pytesprocess/utils/dataframe_output.py +120 -0
- pytesprocess/utils/filter_hdf5.py +594 -0
- pytesprocess/utils/utils.py +701 -0
- pytesprocess/workflows/__init__.py +3 -0
- pytesprocess/workflows/processing.py +317 -0
- pytesprocess/workflows/salting.py +133 -0
- pytesprocess-0.1.1.dist-info/METADATA +211 -0
- pytesprocess-0.1.1.dist-info/RECORD +60 -0
- pytesprocess-0.1.1.dist-info/WHEEL +5 -0
- pytesprocess-0.1.1.dist-info/entry_points.txt +2 -0
- pytesprocess-0.1.1.dist-info/licenses/LICENSE +21 -0
- pytesprocess-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1849 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
|
|
3
|
+
import matplotlib.pyplot as plt
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from qetpy import calc_corrcoeff_from_csd
|
|
7
|
+
import qetpy.plotting as plotting
|
|
8
|
+
from qetpy.utils import (
|
|
9
|
+
convert_channel_list_to_name,
|
|
10
|
+
convert_channel_name_to_list,
|
|
11
|
+
fold_spectrum,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from pytesprocess.utils import FilterH5IO, estimate_sampling_rate
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class FilterData:
|
|
18
|
+
"""
|
|
19
|
+
Class to manage Template, noise psd, csd and IV/dIdV data
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, verbose=True, filter_data=None):
|
|
23
|
+
"""
|
|
24
|
+
Initialize class
|
|
25
|
+
|
|
26
|
+
Parameters:
|
|
27
|
+
----------
|
|
28
|
+
verbose : bool, optional
|
|
29
|
+
display information
|
|
30
|
+
|
|
31
|
+
filter_data : dict, optional
|
|
32
|
+
shared filter-data dictionary. If None, a new one is created.
|
|
33
|
+
"""
|
|
34
|
+
self._verbose = verbose
|
|
35
|
+
|
|
36
|
+
if filter_data is None:
|
|
37
|
+
self._filter_data = dict()
|
|
38
|
+
else:
|
|
39
|
+
if not isinstance(filter_data, dict):
|
|
40
|
+
raise ValueError(
|
|
41
|
+
'ERROR: "filter_data" should be a dictionary or None!'
|
|
42
|
+
)
|
|
43
|
+
self._filter_data = filter_data
|
|
44
|
+
|
|
45
|
+
@staticmethod
|
|
46
|
+
def _metadata_with_channel(metadata, channel, **updates):
|
|
47
|
+
"""Return an independent metadata dictionary for one stored object."""
|
|
48
|
+
|
|
49
|
+
output = copy.deepcopy(metadata) if metadata is not None else {}
|
|
50
|
+
output.update(updates)
|
|
51
|
+
output['channel'] = channel
|
|
52
|
+
return output
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def verbose(self):
|
|
56
|
+
return self._verbose
|
|
57
|
+
|
|
58
|
+
@verbose.setter
|
|
59
|
+
def verbose(self, value):
|
|
60
|
+
self._verbose=value
|
|
61
|
+
|
|
62
|
+
def describe(self, channels=None):
|
|
63
|
+
"""
|
|
64
|
+
Print filter data info
|
|
65
|
+
|
|
66
|
+
Parameters:
|
|
67
|
+
----------
|
|
68
|
+
channels : str or list
|
|
69
|
+
optional list of channels
|
|
70
|
+
|
|
71
|
+
Return
|
|
72
|
+
-------
|
|
73
|
+
None
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
if not self._filter_data:
|
|
77
|
+
print('No filter data available! '
|
|
78
|
+
'Perhaps you need to load data first '
|
|
79
|
+
'(function load_hdf5(file_name)')
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
# channels:
|
|
83
|
+
if (channels is not None
|
|
84
|
+
and isinstance(channels, str)):
|
|
85
|
+
channels = [channels]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Let's first loop channel and get tags/display msg
|
|
89
|
+
filter_display = dict()
|
|
90
|
+
|
|
91
|
+
# list of possible parameters
|
|
92
|
+
parameter_list = [
|
|
93
|
+
'psd', 'psd_power_2poles', 'psd_power_3poles', 'template',
|
|
94
|
+
'csd',
|
|
95
|
+
'dpdi_2poles', 'dpdi_3poles',
|
|
96
|
+
'dpdi_err_2poles', 'dpdi_err_3poles',
|
|
97
|
+
'ivsweep_data',
|
|
98
|
+
'ivsweep_results_noise',
|
|
99
|
+
'ivsweep_results_didv',
|
|
100
|
+
'didv_results_2poles_fit',
|
|
101
|
+
'didv_results_2poles_params',
|
|
102
|
+
'didv_results_2poles_errors',
|
|
103
|
+
'didv_results_2poles_biasparams',
|
|
104
|
+
'didv_results_2poles_biasparams_infinite_lgain',
|
|
105
|
+
'didv_results_2poles_smallsignalparams',
|
|
106
|
+
'didv_results_2poles_ssp_light',
|
|
107
|
+
'didv_results_3poles_fit',
|
|
108
|
+
'didv_results_3poles_params',
|
|
109
|
+
'didv_results_3poles_errors',
|
|
110
|
+
'didv_results_3poles_biasparams',
|
|
111
|
+
'didv_results_3poles_biasparams_infinite_lgain',
|
|
112
|
+
'didv_results_3poles_smallsignalparams',
|
|
113
|
+
'didv_results_3poles_ssp_light',
|
|
114
|
+
'didv_processing',
|
|
115
|
+
'noise_processing',
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
for chan, chan_dict in self._filter_data.items():
|
|
119
|
+
|
|
120
|
+
if channels is not None:
|
|
121
|
+
do_display = False
|
|
122
|
+
for user_chan in channels:
|
|
123
|
+
if user_chan in chan:
|
|
124
|
+
do_display = True
|
|
125
|
+
break
|
|
126
|
+
|
|
127
|
+
if not do_display:
|
|
128
|
+
continue
|
|
129
|
+
|
|
130
|
+
if chan not in filter_display.keys():
|
|
131
|
+
filter_display[chan] = dict()
|
|
132
|
+
|
|
133
|
+
for par_name, val in chan_dict.items():
|
|
134
|
+
|
|
135
|
+
# check if metadata
|
|
136
|
+
if '_metadata' in par_name:
|
|
137
|
+
continue
|
|
138
|
+
|
|
139
|
+
# check if metadata
|
|
140
|
+
if ('_inds' in par_name
|
|
141
|
+
or 'csd_freqs' in par_name):
|
|
142
|
+
continue
|
|
143
|
+
|
|
144
|
+
# find tag
|
|
145
|
+
base_par, tag = self._split_parameter_name(par_name, parameter_list)
|
|
146
|
+
if tag is None:
|
|
147
|
+
continue
|
|
148
|
+
|
|
149
|
+
if tag not in filter_display[chan]:
|
|
150
|
+
filter_display[chan][tag] = list()
|
|
151
|
+
|
|
152
|
+
msg = base_par
|
|
153
|
+
if isinstance(val, pd.Series):
|
|
154
|
+
msg += ': pandas.Series '
|
|
155
|
+
elif isinstance(val, pd.DataFrame):
|
|
156
|
+
msg += ': pandas.DataFrame '
|
|
157
|
+
elif isinstance(val, np.ndarray):
|
|
158
|
+
ndim = val.ndim
|
|
159
|
+
msg += f': {ndim}D numpy.array '
|
|
160
|
+
else:
|
|
161
|
+
msg += (str(type(val)) + ' ')
|
|
162
|
+
|
|
163
|
+
shape = getattr(val, 'shape', None)
|
|
164
|
+
if shape is not None:
|
|
165
|
+
msg += str(shape)
|
|
166
|
+
|
|
167
|
+
if (base_par == 'ivsweep_data'
|
|
168
|
+
and isinstance(val, pd.DataFrame)
|
|
169
|
+
and 'state' in val.columns):
|
|
170
|
+
nb_norm = len(np.where(val['state']=='normal')[0])
|
|
171
|
+
if nb_norm == 0:
|
|
172
|
+
nb_norm = 'Unknown'
|
|
173
|
+
nb_sc = len(np.where(val['state']=='sc')[0])
|
|
174
|
+
if nb_sc == 0:
|
|
175
|
+
nb_sc = 'Unknown'
|
|
176
|
+
msg += '\n Nb SC points: ' + str(nb_sc)
|
|
177
|
+
msg += '\n Nb Normal points: ' + str(nb_norm)
|
|
178
|
+
|
|
179
|
+
filter_display[chan][tag].append(msg)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# loop and display
|
|
183
|
+
channels = list(filter_display.keys())
|
|
184
|
+
print(f'List of channels: {channels}')
|
|
185
|
+
for chan, chan_vals in filter_display.items():
|
|
186
|
+
print('\nChannel ' + chan + ':')
|
|
187
|
+
for tag, tag_info in chan_vals.items():
|
|
188
|
+
print(' * Tag "' + tag + '":')
|
|
189
|
+
if tag_info:
|
|
190
|
+
for msg in tag_info:
|
|
191
|
+
print(' ' + msg)
|
|
192
|
+
|
|
193
|
+
def clear_data(self, channels=None, tag=None):
|
|
194
|
+
"""
|
|
195
|
+
clear filter data
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
if (channels is None and tag is not None):
|
|
199
|
+
raise ValueError(
|
|
200
|
+
'ERROR: "channels" argument needed when '
|
|
201
|
+
'"tag" is provided'
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
if (channels is None and tag is None):
|
|
205
|
+
self._filter_data.clear()
|
|
206
|
+
|
|
207
|
+
elif channels is not None:
|
|
208
|
+
|
|
209
|
+
if isinstance(channels, str):
|
|
210
|
+
channels = [channels]
|
|
211
|
+
|
|
212
|
+
for chan in channels:
|
|
213
|
+
|
|
214
|
+
# check if channel exist
|
|
215
|
+
if chan not in self._filter_data.keys():
|
|
216
|
+
continue
|
|
217
|
+
|
|
218
|
+
# remove specific item
|
|
219
|
+
if tag is None:
|
|
220
|
+
self._filter_data.pop(chan)
|
|
221
|
+
else:
|
|
222
|
+
key_list = list(
|
|
223
|
+
self._filter_data[chan].keys()
|
|
224
|
+
).copy()
|
|
225
|
+
|
|
226
|
+
for key in key_list:
|
|
227
|
+
if tag in key:
|
|
228
|
+
self._filter_data[chan].pop(key)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def load_hdf5(self, file_name, overwrite=True):
|
|
232
|
+
"""
|
|
233
|
+
Load filter data from file. Key may be overwritten if
|
|
234
|
+
already exist
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
Parameters:
|
|
238
|
+
----------
|
|
239
|
+
|
|
240
|
+
file_name : str
|
|
241
|
+
filter file name
|
|
242
|
+
overwrite : boolean
|
|
243
|
+
if True, overwrite exising data
|
|
244
|
+
|
|
245
|
+
Return
|
|
246
|
+
-------
|
|
247
|
+
None
|
|
248
|
+
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
filter_io = FilterH5IO(file_name)
|
|
252
|
+
data = filter_io.load()
|
|
253
|
+
|
|
254
|
+
if self._verbose:
|
|
255
|
+
print('INFO: Loading filter data from file '
|
|
256
|
+
+ file_name)
|
|
257
|
+
|
|
258
|
+
# update
|
|
259
|
+
self.set_data(data, overwrite=overwrite)
|
|
260
|
+
|
|
261
|
+
def set_data(self, data, overwrite=False):
|
|
262
|
+
"""
|
|
263
|
+
Set data directly
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
if not isinstance(data, dict):
|
|
267
|
+
raise ValueError('ERROR: filter data should be a '
|
|
268
|
+
'dictionary!')
|
|
269
|
+
|
|
270
|
+
# update
|
|
271
|
+
if not self._filter_data:
|
|
272
|
+
self._filter_data.update(data)
|
|
273
|
+
else:
|
|
274
|
+
for key, item in data.items():
|
|
275
|
+
if key not in self._filter_data:
|
|
276
|
+
self._filter_data[key] = item
|
|
277
|
+
continue
|
|
278
|
+
for par_name, value in item.items():
|
|
279
|
+
if (overwrite or par_name not in self._filter_data[key].keys()):
|
|
280
|
+
self._filter_data[key][par_name] = (
|
|
281
|
+
data[key][par_name]
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def save_hdf5(self, file_name, overwrite=False):
|
|
287
|
+
"""
|
|
288
|
+
Save filter data to HDF file. Key may be overwritten if
|
|
289
|
+
already exist
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
Parameters:
|
|
293
|
+
----------
|
|
294
|
+
|
|
295
|
+
file_name : str
|
|
296
|
+
filter file name
|
|
297
|
+
overwrite : boolean
|
|
298
|
+
if True, overwrite exising data
|
|
299
|
+
|
|
300
|
+
Return
|
|
301
|
+
-------
|
|
302
|
+
None
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
if self._verbose:
|
|
306
|
+
|
|
307
|
+
msg = ('INFO: Saving filter/TES data to file '
|
|
308
|
+
+ file_name + '!')
|
|
309
|
+
if overwrite:
|
|
310
|
+
msg += ' (overwrite=True)'
|
|
311
|
+
print(msg)
|
|
312
|
+
|
|
313
|
+
filter_io = FilterH5IO(file_name, verbose=False)
|
|
314
|
+
filter_io.save_fromdict(self._filter_data,
|
|
315
|
+
overwrite=overwrite)
|
|
316
|
+
|
|
317
|
+
def get_psd(self, channels, tag='default',
|
|
318
|
+
fold=False,
|
|
319
|
+
return_metadata=False):
|
|
320
|
+
"""
|
|
321
|
+
Get PSD for a specific channel in unit of Amps^2/Hz
|
|
322
|
+
|
|
323
|
+
Parameters:
|
|
324
|
+
----------
|
|
325
|
+
|
|
326
|
+
channels : str or list of string
|
|
327
|
+
channel name
|
|
328
|
+
|
|
329
|
+
tag : str, optional
|
|
330
|
+
psd tag, default: No tag
|
|
331
|
+
|
|
332
|
+
fold : boolean, option
|
|
333
|
+
if True, return folded psd
|
|
334
|
+
|
|
335
|
+
Return
|
|
336
|
+
------
|
|
337
|
+
|
|
338
|
+
psd : ndarray,
|
|
339
|
+
psd [Amps^2/Hz]
|
|
340
|
+
f : ndarray
|
|
341
|
+
psd frequencies
|
|
342
|
+
metadata : dict
|
|
343
|
+
"""
|
|
344
|
+
|
|
345
|
+
if isinstance(channels, str):
|
|
346
|
+
channels = [channels]
|
|
347
|
+
|
|
348
|
+
output_psd = None
|
|
349
|
+
output_psd_freqs = None
|
|
350
|
+
output_metadata = dict()
|
|
351
|
+
|
|
352
|
+
for chan in channels:
|
|
353
|
+
|
|
354
|
+
psd, psd_freqs, metadata = self._get_param_array(
|
|
355
|
+
'psd', chan, tag=tag, return_metadata=True)
|
|
356
|
+
output_metadata[chan] = copy.deepcopy(metadata)
|
|
357
|
+
|
|
358
|
+
if fold:
|
|
359
|
+
|
|
360
|
+
sample_rate = metadata.get('sample_rate') or metadata.get('sample_rate_hz')
|
|
361
|
+
if sample_rate is not None:
|
|
362
|
+
sample_rate = float(sample_rate)
|
|
363
|
+
else:
|
|
364
|
+
sample_rate = estimate_sampling_rate(psd_freqs)
|
|
365
|
+
|
|
366
|
+
psd_freqs, psd = fold_spectrum(psd, sample_rate)
|
|
367
|
+
|
|
368
|
+
if output_psd is None:
|
|
369
|
+
output_psd = psd.copy()
|
|
370
|
+
output_psd = output_psd[np.newaxis, :]
|
|
371
|
+
output_psd_freqs = psd_freqs.copy()
|
|
372
|
+
else:
|
|
373
|
+
|
|
374
|
+
if (psd.shape[0] != output_psd.shape[-1]
|
|
375
|
+
or np.any(psd_freqs != output_psd_freqs)):
|
|
376
|
+
raise ValueError(
|
|
377
|
+
'ERROR: unable to retrieve psd '
|
|
378
|
+
'for multiple channels. Inconsistent '
|
|
379
|
+
'number of samples. Get psd one by one...')
|
|
380
|
+
psd = psd[np.newaxis, :].copy()
|
|
381
|
+
output_psd = np.concatenate((output_psd, psd), axis=0)
|
|
382
|
+
|
|
383
|
+
if output_psd.shape[0] == 1:
|
|
384
|
+
output_psd = np.squeeze(output_psd[0,:])
|
|
385
|
+
|
|
386
|
+
if return_metadata:
|
|
387
|
+
if len(channels) == 1:
|
|
388
|
+
output_metadata = output_metadata[channels[0]]
|
|
389
|
+
return output_psd, output_psd_freqs, output_metadata
|
|
390
|
+
else:
|
|
391
|
+
return output_psd, output_psd_freqs
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def get_psd_power(self, channels, poles=3, tag='default',
|
|
395
|
+
fold=False, return_metadata=False):
|
|
396
|
+
"""Get a power-referred PSD in W^2/Hz.
|
|
397
|
+
|
|
398
|
+
Power PSDs are stored separately from the current PSD so both products
|
|
399
|
+
remain available for the same background-stream tag.
|
|
400
|
+
"""
|
|
401
|
+
if poles not in (2, 3):
|
|
402
|
+
raise ValueError('ERROR: "poles" should be 2 or 3!')
|
|
403
|
+
if isinstance(channels, str):
|
|
404
|
+
channels = [channels]
|
|
405
|
+
|
|
406
|
+
base_name = f'psd_power_{poles}poles'
|
|
407
|
+
output = None
|
|
408
|
+
output_freqs = None
|
|
409
|
+
output_metadata = {}
|
|
410
|
+
for channel in channels:
|
|
411
|
+
spectrum, freqs, metadata = self._get_param_array(
|
|
412
|
+
base_name, channel, tag=tag, return_metadata=True
|
|
413
|
+
)
|
|
414
|
+
output_metadata[channel] = copy.deepcopy(metadata)
|
|
415
|
+
if fold:
|
|
416
|
+
sample_rate = metadata.get('sample_rate') or metadata.get('sample_rate_hz')
|
|
417
|
+
if sample_rate is None:
|
|
418
|
+
sample_rate = estimate_sampling_rate(freqs)
|
|
419
|
+
freqs, spectrum = fold_spectrum(spectrum, float(sample_rate))
|
|
420
|
+
|
|
421
|
+
if output is None:
|
|
422
|
+
output = spectrum[np.newaxis, :].copy()
|
|
423
|
+
output_freqs = freqs.copy()
|
|
424
|
+
else:
|
|
425
|
+
if spectrum.shape[0] != output.shape[-1] or np.any(freqs != output_freqs):
|
|
426
|
+
raise ValueError(
|
|
427
|
+
'ERROR: unable to retrieve power PSD for multiple channels. '
|
|
428
|
+
'Frequency arrays are inconsistent.'
|
|
429
|
+
)
|
|
430
|
+
output = np.concatenate((output, spectrum[np.newaxis, :]), axis=0)
|
|
431
|
+
|
|
432
|
+
if output.shape[0] == 1:
|
|
433
|
+
output = np.squeeze(output[0, :])
|
|
434
|
+
if return_metadata:
|
|
435
|
+
if len(channels) == 1:
|
|
436
|
+
output_metadata = output_metadata[channels[0]]
|
|
437
|
+
return output, output_freqs, output_metadata
|
|
438
|
+
return output, output_freqs
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def get_csd(self, channels, tag='default',
|
|
442
|
+
fold=False,
|
|
443
|
+
return_metadata=False):
|
|
444
|
+
"""
|
|
445
|
+
Get CSD for a specified channel list or string with channels
|
|
446
|
+
separated with "|" in unit of Amps^2/Hz. Channel order is
|
|
447
|
+
important!
|
|
448
|
+
|
|
449
|
+
Parameters:
|
|
450
|
+
----------
|
|
451
|
+
|
|
452
|
+
channels : str or list
|
|
453
|
+
channel string (multi-channels use "|" separation)
|
|
454
|
+
or list
|
|
455
|
+
|
|
456
|
+
tag : str, optional
|
|
457
|
+
psd tag, default: No tag
|
|
458
|
+
|
|
459
|
+
fold : boolean, option
|
|
460
|
+
if True, return folded psd
|
|
461
|
+
|
|
462
|
+
Return
|
|
463
|
+
------
|
|
464
|
+
|
|
465
|
+
csd : ndarray,
|
|
466
|
+
csd [Amps^2/Hz]
|
|
467
|
+
f : ndarray
|
|
468
|
+
csd frequencies
|
|
469
|
+
metadata : dict
|
|
470
|
+
csd metadata if return_metadata=True
|
|
471
|
+
"""
|
|
472
|
+
|
|
473
|
+
# check channel
|
|
474
|
+
channel_list = convert_channel_name_to_list(channels)
|
|
475
|
+
channel_name = convert_channel_list_to_name(channels)
|
|
476
|
+
nb_channels = len(channel_list)
|
|
477
|
+
|
|
478
|
+
if nb_channels == 1:
|
|
479
|
+
return self.get_psd(channel_name, tag=tag,
|
|
480
|
+
fold=fold,
|
|
481
|
+
return_metadata=return_metadata)
|
|
482
|
+
|
|
483
|
+
# get values
|
|
484
|
+
output_metadata = dict()
|
|
485
|
+
csd, csd_freqs, metadata = (
|
|
486
|
+
self._get_param_array('csd',
|
|
487
|
+
channel_name,
|
|
488
|
+
tag=tag,
|
|
489
|
+
return_metadata=True)
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
output_metadata = copy.deepcopy(metadata)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
if fold:
|
|
497
|
+
|
|
498
|
+
sample_rate = (
|
|
499
|
+
metadata.get('sample_rate')
|
|
500
|
+
or metadata.get('sample_rate_hz')
|
|
501
|
+
or metadata.get('sample_rate-hz') # legacy typo
|
|
502
|
+
)
|
|
503
|
+
if sample_rate is not None:
|
|
504
|
+
sample_rate = float(sample_rate)
|
|
505
|
+
else:
|
|
506
|
+
sample_rate = estimate_sampling_rate(csd_freqs)
|
|
507
|
+
|
|
508
|
+
csd_freqs, csd = fold_spectrum(csd, sample_rate)
|
|
509
|
+
|
|
510
|
+
if return_metadata:
|
|
511
|
+
return csd, csd_freqs, output_metadata
|
|
512
|
+
else:
|
|
513
|
+
return csd, csd_freqs
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def get_corrcoeff(self, channels, tag='default', fold=False,
|
|
517
|
+
return_metadata=False):
|
|
518
|
+
"""Calculate the correlation coefficient from a stored CSD."""
|
|
519
|
+
csd, freqs, metadata = self.get_csd(
|
|
520
|
+
channels, tag=tag, fold=fold, return_metadata=True
|
|
521
|
+
)
|
|
522
|
+
corrcoeff = calc_corrcoeff_from_csd(csd.copy())
|
|
523
|
+
if return_metadata:
|
|
524
|
+
return corrcoeff, freqs, metadata
|
|
525
|
+
return corrcoeff, freqs
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def get_template(self, channel, tag='default',
|
|
529
|
+
return_metadata=False):
|
|
530
|
+
"""
|
|
531
|
+
Get template for a specific channel
|
|
532
|
+
|
|
533
|
+
Parameters:
|
|
534
|
+
----------
|
|
535
|
+
|
|
536
|
+
channel : str
|
|
537
|
+
channel name
|
|
538
|
+
|
|
539
|
+
tag : str, optional
|
|
540
|
+
template tag, default: No tag
|
|
541
|
+
|
|
542
|
+
Return
|
|
543
|
+
------
|
|
544
|
+
|
|
545
|
+
template : ndarray,
|
|
546
|
+
psd in units of amps
|
|
547
|
+
time : ndarray
|
|
548
|
+
time array
|
|
549
|
+
"""
|
|
550
|
+
|
|
551
|
+
# return values
|
|
552
|
+
return self._get_param_array(
|
|
553
|
+
'template',
|
|
554
|
+
channel,
|
|
555
|
+
tag=tag,
|
|
556
|
+
return_metadata=return_metadata)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def get_dpdi(self, channel, poles, return_dpdi_err=False,
|
|
561
|
+
tag='default'):
|
|
562
|
+
"""
|
|
563
|
+
Get dpdi for a specific channel in units of Volts
|
|
564
|
+
|
|
565
|
+
Parameters:
|
|
566
|
+
----------
|
|
567
|
+
|
|
568
|
+
channel : str
|
|
569
|
+
channel name
|
|
570
|
+
|
|
571
|
+
poles: int
|
|
572
|
+
2 or 3-poles fit
|
|
573
|
+
|
|
574
|
+
return_dpdi_err : bool
|
|
575
|
+
return dpdi error (optinal to keep back compatibility)
|
|
576
|
+
|
|
577
|
+
tag : str, optional
|
|
578
|
+
dpdi tag, default: No tag
|
|
579
|
+
|
|
580
|
+
Return
|
|
581
|
+
------
|
|
582
|
+
|
|
583
|
+
dpdi : ndarray,
|
|
584
|
+
dpdi [Volts]
|
|
585
|
+
|
|
586
|
+
dpdi_err : ndarray, optional return
|
|
587
|
+
|
|
588
|
+
f : ndarray
|
|
589
|
+
dpdi frequencies
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
"""
|
|
595
|
+
|
|
596
|
+
if poles not in [2,3]:
|
|
597
|
+
raise ValueError('ERROR: "poles" should be '
|
|
598
|
+
'2 or 3!')
|
|
599
|
+
# dpdi
|
|
600
|
+
par_name = f'dpdi_{poles}poles'
|
|
601
|
+
dpdi,f = self._get_param_array(par_name,channel,
|
|
602
|
+
tag=tag)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
# dpdi error
|
|
606
|
+
if return_dpdi_err:
|
|
607
|
+
par_name = f'dpdi_err_{poles}poles'
|
|
608
|
+
dpdi_err,_ = self._get_param_array(par_name,channel,
|
|
609
|
+
tag=tag)
|
|
610
|
+
|
|
611
|
+
if return_dpdi_err:
|
|
612
|
+
return dpdi, dpdi_err, f
|
|
613
|
+
else:
|
|
614
|
+
return dpdi, f
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def set_template(self, channels, template,
|
|
618
|
+
sample_rate=None,
|
|
619
|
+
pretrigger_length_msec=None,
|
|
620
|
+
pretrigger_length_samples=None,
|
|
621
|
+
template_time_tags=None,
|
|
622
|
+
metadata=None,
|
|
623
|
+
tag='default'):
|
|
624
|
+
"""
|
|
625
|
+
set template array
|
|
626
|
+
"""
|
|
627
|
+
|
|
628
|
+
# check array type/dim
|
|
629
|
+
if not isinstance(template, np.ndarray):
|
|
630
|
+
raise ValueError('ERROR: "template" argument '
|
|
631
|
+
'should be a numpy array!')
|
|
632
|
+
|
|
633
|
+
channel_list = convert_channel_name_to_list(channels)
|
|
634
|
+
channel_name = convert_channel_list_to_name(channels)
|
|
635
|
+
nb_channels = len(channel_list)
|
|
636
|
+
|
|
637
|
+
# check array shape
|
|
638
|
+
if (nb_channels == 1 and template.ndim != 1):
|
|
639
|
+
raise ValueError(
|
|
640
|
+
'ERROR: For single channel, expecting and 1D '
|
|
641
|
+
'array '
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
if (nb_channels > 1 and template.ndim != 3):
|
|
645
|
+
raise ValueError(
|
|
646
|
+
'ERROR: For multiple channels, expecting 3D '
|
|
647
|
+
'array [nchans, ntemps, nsamples] '
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
if template_time_tags is not None:
|
|
651
|
+
if not isinstance(template_time_tags, np.ndarray):
|
|
652
|
+
raise ValueError(
|
|
653
|
+
'ERROR: Expecting "template_time_tags" to '
|
|
654
|
+
'be an array or None'
|
|
655
|
+
)
|
|
656
|
+
if template_time_tags.ndim != 1 or \
|
|
657
|
+
len(template_time_tags) != template.shape[1]:
|
|
658
|
+
raise ValueError(
|
|
659
|
+
'ERROR: Expecting "template_time_tags" to '
|
|
660
|
+
'be a 1D array with length = # templates'
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
# sample rate / pretrigger length
|
|
664
|
+
if sample_rate is None:
|
|
665
|
+
raise ValueError('ERROR: "sample_rate" argument required!')
|
|
666
|
+
|
|
667
|
+
if (pretrigger_length_msec is None
|
|
668
|
+
and pretrigger_length_samples is None):
|
|
669
|
+
raise ValueError('ERROR: pretrigger length (samples or msec)'
|
|
670
|
+
' required!')
|
|
671
|
+
|
|
672
|
+
if pretrigger_length_msec is not None:
|
|
673
|
+
pretrigger_length_samples = int(
|
|
674
|
+
round(pretrigger_length_msec*sample_rate*1e-3)
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
# time array
|
|
678
|
+
dt = 1/sample_rate
|
|
679
|
+
t = np.asarray(list(range(template.shape[-1])))*dt
|
|
680
|
+
|
|
681
|
+
# parameter name
|
|
682
|
+
template_name = 'template' + '_' + tag
|
|
683
|
+
|
|
684
|
+
metadata = self._metadata_with_channel(
|
|
685
|
+
metadata,
|
|
686
|
+
channel_name,
|
|
687
|
+
sample_rate=sample_rate,
|
|
688
|
+
sample_rate_hz=sample_rate,
|
|
689
|
+
nb_samples=template.shape[-1],
|
|
690
|
+
nb_pretrigger_samples=pretrigger_length_samples,
|
|
691
|
+
template_time_tags=template_time_tags,
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
# store
|
|
695
|
+
if channel_name not in self._filter_data.keys():
|
|
696
|
+
self._filter_data[channel_name] = dict()
|
|
697
|
+
|
|
698
|
+
if template.ndim == 1:
|
|
699
|
+
self._filter_data[channel_name][template_name] = (
|
|
700
|
+
pd.Series(template, t)
|
|
701
|
+
)
|
|
702
|
+
else:
|
|
703
|
+
self._filter_data[channel_name][template_name] = template
|
|
704
|
+
|
|
705
|
+
# inds
|
|
706
|
+
inds_name = template_name + '_inds'
|
|
707
|
+
self._filter_data[channel_name][inds_name ] = t
|
|
708
|
+
|
|
709
|
+
self._filter_data[channel_name][template_name + '_metadata'] = metadata
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def set_psd(self, channels, psd, psd_freqs,
|
|
713
|
+
sample_rate=None,
|
|
714
|
+
pretrigger_length_msec=None,
|
|
715
|
+
pretrigger_length_samples=None,
|
|
716
|
+
metadata=None,
|
|
717
|
+
tag='default'):
|
|
718
|
+
"""
|
|
719
|
+
set psd array
|
|
720
|
+
"""
|
|
721
|
+
|
|
722
|
+
# check array type/dim
|
|
723
|
+
if isinstance(psd, list):
|
|
724
|
+
psd = np.array(psd)
|
|
725
|
+
elif not isinstance(psd, np.ndarray):
|
|
726
|
+
raise ValueError('ERROR: Expecting numpy array!')
|
|
727
|
+
if psd.ndim == 1:
|
|
728
|
+
psd = psd[np.newaxis, :]
|
|
729
|
+
|
|
730
|
+
metadata = copy.deepcopy(metadata) if metadata is not None else {}
|
|
731
|
+
|
|
732
|
+
# check is sample rate in metadata
|
|
733
|
+
if sample_rate is None:
|
|
734
|
+
|
|
735
|
+
sample_rate = metadata.get('sample_rate') or metadata.get('sample_rate_hz')
|
|
736
|
+
if sample_rate is not None:
|
|
737
|
+
sample_rate = float(sample_rate)
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
# check frequency array
|
|
741
|
+
if isinstance(psd_freqs, list):
|
|
742
|
+
psd_freqs = np.array(psd_freqs)
|
|
743
|
+
elif not isinstance(psd_freqs, np.ndarray):
|
|
744
|
+
raise ValueError('ERROR: Expecting numpy array '
|
|
745
|
+
'for "psd_freqs" argument')
|
|
746
|
+
|
|
747
|
+
# add dimension if needed
|
|
748
|
+
if psd_freqs.ndim == 1:
|
|
749
|
+
psd_freqs = psd_freqs[np.newaxis, :]
|
|
750
|
+
|
|
751
|
+
# check if folded -> NOT ALLOWED
|
|
752
|
+
is_folded = not np.any(psd_freqs<0)
|
|
753
|
+
if is_folded:
|
|
754
|
+
raise ValueError('ERROR: psd needs to be two-sided!')
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
sample_rate_array = estimate_sampling_rate(psd_freqs[0,:])
|
|
758
|
+
if sample_rate is None:
|
|
759
|
+
sample_rate = sample_rate_array
|
|
760
|
+
elif round(sample_rate_array) != round(sample_rate):
|
|
761
|
+
raise ValueError('ERROR: sample_rate is inconsistent with '
|
|
762
|
+
'frequency array!')
|
|
763
|
+
|
|
764
|
+
# number of channels
|
|
765
|
+
if isinstance(channels, str):
|
|
766
|
+
channels = [channels]
|
|
767
|
+
nb_channels = len(channels)
|
|
768
|
+
|
|
769
|
+
# check array shape
|
|
770
|
+
if psd.shape[0] != nb_channels:
|
|
771
|
+
raise ValueError(
|
|
772
|
+
'ERROR: psd shape is not '
|
|
773
|
+
' consistent with number of channels')
|
|
774
|
+
|
|
775
|
+
if psd_freqs.shape[0] != nb_channels:
|
|
776
|
+
if psd_freqs.shape[0] == 1:
|
|
777
|
+
psd_freqs = np.repeat(psd_freqs, nb_channels, axis=0)
|
|
778
|
+
else:
|
|
779
|
+
raise ValueError(
|
|
780
|
+
'ERROR: psd_freqs shape is not '
|
|
781
|
+
' consistent with number of channels')
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
# parameter name
|
|
785
|
+
psd_name = 'psd' + '_' + tag
|
|
786
|
+
|
|
787
|
+
# add pretrigger length (not required)
|
|
788
|
+
if pretrigger_length_msec is not None:
|
|
789
|
+
pretrigger_length_samples = int(
|
|
790
|
+
round(pretrigger_length_msec*sample_rate*1e-3)
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
metadata['sample_rate'] = sample_rate
|
|
794
|
+
metadata['sample_rate_hz'] = sample_rate
|
|
795
|
+
metadata['nb_samples'] = psd.shape[1]
|
|
796
|
+
if pretrigger_length_samples is not None:
|
|
797
|
+
metadata['nb_pretrigger_samples'] = pretrigger_length_samples
|
|
798
|
+
|
|
799
|
+
# loop channels and store
|
|
800
|
+
for ichan in range(nb_channels):
|
|
801
|
+
|
|
802
|
+
# channel name
|
|
803
|
+
chan = channels[ichan]
|
|
804
|
+
|
|
805
|
+
# psd
|
|
806
|
+
psd_chan = np.squeeze(psd[ichan, :])
|
|
807
|
+
freqs = np.squeeze(psd_freqs[ichan, :])
|
|
808
|
+
|
|
809
|
+
# add channel
|
|
810
|
+
if chan not in self._filter_data.keys():
|
|
811
|
+
self._filter_data[chan] = dict()
|
|
812
|
+
else:
|
|
813
|
+
# check template/psd have same length
|
|
814
|
+
template_name = 'template' + '_' + tag
|
|
815
|
+
if template_name in self._filter_data[chan]:
|
|
816
|
+
template = self._filter_data[chan][template_name].values
|
|
817
|
+
if len(psd_chan) != len(template):
|
|
818
|
+
raise ValueError(
|
|
819
|
+
f'ERROR: template and psd for channel {chan} '
|
|
820
|
+
f'are required to have same length for '
|
|
821
|
+
f'tag {tag}. Use a different tag, clear previous data '
|
|
822
|
+
f'first using "clear_data(...)" or set '
|
|
823
|
+
f'psd length to {len(template)}')
|
|
824
|
+
|
|
825
|
+
self._filter_data[chan][psd_name] = pd.Series(psd_chan, freqs)
|
|
826
|
+
|
|
827
|
+
channel_metadata = self._metadata_with_channel(metadata, chan)
|
|
828
|
+
self._filter_data[chan][psd_name + '_metadata'] = channel_metadata
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def set_psd_power(self, channels, psd, psd_freqs, poles=3,
|
|
832
|
+
sample_rate=None, metadata=None, tag='default'):
|
|
833
|
+
"""Store a two-sided power-referred PSD in W^2/Hz."""
|
|
834
|
+
if poles not in (2, 3):
|
|
835
|
+
raise ValueError('ERROR: "poles" should be 2 or 3!')
|
|
836
|
+
psd = np.asarray(psd)
|
|
837
|
+
psd_freqs = np.asarray(psd_freqs)
|
|
838
|
+
if psd.ndim == 1:
|
|
839
|
+
psd = psd[np.newaxis, :]
|
|
840
|
+
if psd_freqs.ndim == 1:
|
|
841
|
+
psd_freqs = psd_freqs[np.newaxis, :]
|
|
842
|
+
if not np.any(psd_freqs < 0):
|
|
843
|
+
raise ValueError('ERROR: power PSD needs to be two-sided!')
|
|
844
|
+
|
|
845
|
+
if isinstance(channels, str):
|
|
846
|
+
channels = [channels]
|
|
847
|
+
n_channels = len(channels)
|
|
848
|
+
if psd.shape[0] != n_channels:
|
|
849
|
+
raise ValueError('ERROR: power PSD shape is inconsistent with number of channels!')
|
|
850
|
+
if psd_freqs.shape[0] == 1 and n_channels > 1:
|
|
851
|
+
psd_freqs = np.repeat(psd_freqs, n_channels, axis=0)
|
|
852
|
+
if psd_freqs.shape[0] != n_channels:
|
|
853
|
+
raise ValueError('ERROR: power PSD frequency shape is inconsistent with channels!')
|
|
854
|
+
|
|
855
|
+
sample_rate_array = estimate_sampling_rate(psd_freqs[0, :])
|
|
856
|
+
if sample_rate is None:
|
|
857
|
+
sample_rate = sample_rate_array
|
|
858
|
+
elif round(float(sample_rate)) != round(float(sample_rate_array)):
|
|
859
|
+
raise ValueError('ERROR: sample_rate is inconsistent with power PSD frequencies!')
|
|
860
|
+
|
|
861
|
+
base_name = f'psd_power_{poles}poles_{tag}'
|
|
862
|
+
metadata = copy.deepcopy(metadata) if metadata is not None else {}
|
|
863
|
+
metadata.update({
|
|
864
|
+
'sample_rate': float(sample_rate),
|
|
865
|
+
'sample_rate_hz': float(sample_rate),
|
|
866
|
+
'nb_samples': int(psd.shape[-1]),
|
|
867
|
+
'poles': int(poles),
|
|
868
|
+
'units': 'W^2/Hz',
|
|
869
|
+
})
|
|
870
|
+
|
|
871
|
+
for index, channel in enumerate(channels):
|
|
872
|
+
if channel not in self._filter_data:
|
|
873
|
+
self._filter_data[channel] = {}
|
|
874
|
+
spectrum = np.squeeze(psd[index, :])
|
|
875
|
+
freqs = np.squeeze(psd_freqs[index, :])
|
|
876
|
+
self._filter_data[channel][base_name] = pd.Series(spectrum, freqs)
|
|
877
|
+
self._filter_data[channel][base_name + '_metadata'] = (
|
|
878
|
+
self._metadata_with_channel(metadata, channel)
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def set_csd(self, channels, csd, csd_freqs,
|
|
883
|
+
sample_rate=None,
|
|
884
|
+
pretrigger_length_msec=None,
|
|
885
|
+
pretrigger_length_samples=None,
|
|
886
|
+
metadata=None,
|
|
887
|
+
tag='default'):
|
|
888
|
+
"""
|
|
889
|
+
set csd array
|
|
890
|
+
"""
|
|
891
|
+
|
|
892
|
+
# channels
|
|
893
|
+
channel_list = convert_channel_name_to_list(channels)
|
|
894
|
+
channel_name = convert_channel_list_to_name(channels)
|
|
895
|
+
nb_channels = len(channel_list)
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
metadata = copy.deepcopy(metadata) if metadata is not None else {}
|
|
899
|
+
|
|
900
|
+
# check array type/dim
|
|
901
|
+
if (not isinstance(csd, np.ndarray)
|
|
902
|
+
or csd.ndim != 3):
|
|
903
|
+
raise ValueError('ERROR: Expecting a 3D numpy array!')
|
|
904
|
+
|
|
905
|
+
if (nb_channels != csd.shape[0]
|
|
906
|
+
or csd.shape[0] != csd.shape[1]):
|
|
907
|
+
raise ValueError('ERROR: Array shape not consistent with '
|
|
908
|
+
'number of channels!')
|
|
909
|
+
|
|
910
|
+
# check frequency array
|
|
911
|
+
if isinstance(csd_freqs, list):
|
|
912
|
+
csd_freqs = np.array(csd_freqs)
|
|
913
|
+
elif not isinstance(csd_freqs, np.ndarray):
|
|
914
|
+
raise ValueError('ERROR: Expecting a numpy array '
|
|
915
|
+
'for "csd_freqs" argument')
|
|
916
|
+
# squeeze
|
|
917
|
+
if csd_freqs.ndim == 2:
|
|
918
|
+
csd_freqs = np.squeeze(csd_freqs[0,:])
|
|
919
|
+
|
|
920
|
+
# check if folded -> NOT ALLOWED
|
|
921
|
+
is_folded = not np.any(csd_freqs<0)
|
|
922
|
+
if is_folded:
|
|
923
|
+
raise ValueError('ERROR: psd needs to be two-sided!')
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
sample_rate_array = estimate_sampling_rate(csd_freqs)
|
|
927
|
+
if sample_rate is None:
|
|
928
|
+
sample_rate = sample_rate_array
|
|
929
|
+
elif round(sample_rate_array) != round(sample_rate):
|
|
930
|
+
raise ValueError('ERROR: sample_rate is inconsistent with '
|
|
931
|
+
'frequency array!')
|
|
932
|
+
|
|
933
|
+
# parameter name
|
|
934
|
+
csd_name = 'csd_' + tag
|
|
935
|
+
csd_freqs_name = csd_name + '_inds'
|
|
936
|
+
|
|
937
|
+
# add pretrigger length (not required)
|
|
938
|
+
if pretrigger_length_msec is not None:
|
|
939
|
+
pretrigger_length_samples = int(
|
|
940
|
+
round(pretrigger_length_msec*sample_rate*1e-3)
|
|
941
|
+
)
|
|
942
|
+
metadata['sample_rate'] = sample_rate
|
|
943
|
+
metadata['sample_rate_hz'] = sample_rate
|
|
944
|
+
metadata['nb_samples'] = csd.shape[-1]
|
|
945
|
+
if pretrigger_length_samples is not None:
|
|
946
|
+
metadata['nb_pretrigger_samples'] = pretrigger_length_samples
|
|
947
|
+
metadata = self._metadata_with_channel(metadata, channel_name)
|
|
948
|
+
|
|
949
|
+
# add channel
|
|
950
|
+
if channel_name not in self._filter_data.keys():
|
|
951
|
+
self._filter_data[channel_name] = dict()
|
|
952
|
+
|
|
953
|
+
self._filter_data[channel_name][csd_name] = csd
|
|
954
|
+
self._filter_data[channel_name][csd_name + '_metadata'] = metadata
|
|
955
|
+
self._filter_data[channel_name][csd_freqs_name] = csd_freqs
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def set_dpdi(self, channels,
|
|
959
|
+
dpdi, dpdi_err,
|
|
960
|
+
dpdi_freqs,
|
|
961
|
+
poles,
|
|
962
|
+
sample_rate=None,
|
|
963
|
+
metadata=None,
|
|
964
|
+
tag='default'):
|
|
965
|
+
"""
|
|
966
|
+
set dpdi array
|
|
967
|
+
"""
|
|
968
|
+
|
|
969
|
+
# check poles
|
|
970
|
+
if poles not in [2,3]:
|
|
971
|
+
raise ValueError('ERROR: Poles should be '
|
|
972
|
+
'2 or 3!')
|
|
973
|
+
|
|
974
|
+
# check array type/dim
|
|
975
|
+
if isinstance(dpdi, list):
|
|
976
|
+
dpdi = np.array(dpdi)
|
|
977
|
+
elif not isinstance(dpdi, np.ndarray):
|
|
978
|
+
raise ValueError('ERROR: Expecting numpy array!')
|
|
979
|
+
|
|
980
|
+
if dpdi.ndim == 1:
|
|
981
|
+
dpdi = dpdi[np.newaxis, :]
|
|
982
|
+
|
|
983
|
+
# dP/dI uncertainty
|
|
984
|
+
if isinstance(dpdi_err, list):
|
|
985
|
+
dpdi_err = np.array(dpdi_err)
|
|
986
|
+
elif not isinstance(dpdi_err, np.ndarray):
|
|
987
|
+
raise ValueError('ERROR: Expecting numpy array for "dpdi_err"!')
|
|
988
|
+
|
|
989
|
+
# check array type/dim
|
|
990
|
+
if isinstance(dpdi_freqs, list):
|
|
991
|
+
dpdi_freqs = np.array(dpdi_freqs)
|
|
992
|
+
elif not isinstance(dpdi_freqs, np.ndarray):
|
|
993
|
+
raise ValueError('ERROR: Expecting numpy array!')
|
|
994
|
+
|
|
995
|
+
if dpdi_freqs.ndim == 1:
|
|
996
|
+
dpdi_freqs = dpdi_freqs[np.newaxis, :]
|
|
997
|
+
|
|
998
|
+
# check if folded -> NOT ALLOWED
|
|
999
|
+
is_folded = not np.any(dpdi_freqs<0)
|
|
1000
|
+
if is_folded:
|
|
1001
|
+
raise ValueError('ERROR: dpdi needs to be two-sided!')
|
|
1002
|
+
|
|
1003
|
+
sample_rate_array = estimate_sampling_rate(dpdi_freqs[0,:])
|
|
1004
|
+
if sample_rate is None:
|
|
1005
|
+
sample_rate = sample_rate_array
|
|
1006
|
+
elif round(sample_rate_array) != round(sample_rate):
|
|
1007
|
+
raise ValueError(f'ERROR: sample rate ({sample_rate}) '
|
|
1008
|
+
f'is inconsistent with sample rate from '
|
|
1009
|
+
f'frequency array ({sample_rate_array})!')
|
|
1010
|
+
|
|
1011
|
+
# number of channels
|
|
1012
|
+
if isinstance(channels, str):
|
|
1013
|
+
channels = [channels]
|
|
1014
|
+
nb_channels = len(channels)
|
|
1015
|
+
|
|
1016
|
+
# check array shape
|
|
1017
|
+
if dpdi.shape[0] != nb_channels:
|
|
1018
|
+
raise ValueError(
|
|
1019
|
+
'ERROR: Array shape is not consistent with '
|
|
1020
|
+
'number of channels')
|
|
1021
|
+
|
|
1022
|
+
if dpdi_freqs.shape[0] != nb_channels:
|
|
1023
|
+
if dpdi_freqs.shape[0] == 1:
|
|
1024
|
+
dpdi_freqs = np.repeat(dpdi_freqs, nb_channels, axis=0)
|
|
1025
|
+
else:
|
|
1026
|
+
raise ValueError(
|
|
1027
|
+
'ERROR:dpdi_freqs shape is not '
|
|
1028
|
+
' consistent with number of channels')
|
|
1029
|
+
|
|
1030
|
+
if dpdi_err.ndim == 1:
|
|
1031
|
+
if nb_channels == 1:
|
|
1032
|
+
dpdi_err = dpdi_err[np.newaxis, :]
|
|
1033
|
+
else:
|
|
1034
|
+
raise ValueError(
|
|
1035
|
+
'ERROR: 1D dpdi_err is only valid for a single channel')
|
|
1036
|
+
if dpdi_err.ndim != 2 or dpdi_err.shape != dpdi.shape:
|
|
1037
|
+
raise ValueError(
|
|
1038
|
+
'ERROR: dpdi_err shape is not consistent with dpdi shape')
|
|
1039
|
+
|
|
1040
|
+
# parameter name
|
|
1041
|
+
dpdi_name = f'dpdi_{poles}poles_{tag}'
|
|
1042
|
+
dpdi_err_name = f'dpdi_err_{poles}poles_{tag}'
|
|
1043
|
+
|
|
1044
|
+
metadata = copy.deepcopy(metadata) if metadata is not None else {}
|
|
1045
|
+
metadata['sample_rate'] = sample_rate
|
|
1046
|
+
metadata['sample_rate_hz'] = sample_rate
|
|
1047
|
+
metadata['nb_samples'] = dpdi.shape[-1]
|
|
1048
|
+
metadata['poles'] = poles
|
|
1049
|
+
|
|
1050
|
+
# loop channels and store
|
|
1051
|
+
for ichan in range(nb_channels):
|
|
1052
|
+
|
|
1053
|
+
# channel name
|
|
1054
|
+
chan = channels[ichan]
|
|
1055
|
+
|
|
1056
|
+
# dP/dI
|
|
1057
|
+
dpdi_chan = np.squeeze(dpdi[ichan,:])
|
|
1058
|
+
dpdi_err_chan = np.squeeze(dpdi_err[ichan, :])
|
|
1059
|
+
dpdi_freqs_chan = np.squeeze(dpdi_freqs[ichan,:])
|
|
1060
|
+
|
|
1061
|
+
# add channel if needed
|
|
1062
|
+
if chan not in self._filter_data.keys():
|
|
1063
|
+
self._filter_data[chan] = dict()
|
|
1064
|
+
|
|
1065
|
+
self._filter_data[chan][dpdi_name] = (
|
|
1066
|
+
pd.Series(dpdi_chan, dpdi_freqs_chan)
|
|
1067
|
+
)
|
|
1068
|
+
|
|
1069
|
+
self._filter_data[chan][dpdi_err_name] = (
|
|
1070
|
+
pd.Series(dpdi_err_chan, dpdi_freqs_chan)
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
channel_metadata = self._metadata_with_channel(metadata, chan)
|
|
1074
|
+
self._filter_data[chan][dpdi_name + '_metadata'] = channel_metadata
|
|
1075
|
+
self._filter_data[chan][dpdi_err_name + '_metadata'] = copy.deepcopy(
|
|
1076
|
+
channel_metadata
|
|
1077
|
+
)
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def set_ivsweep_data(self,
|
|
1081
|
+
channel,
|
|
1082
|
+
dataframe,
|
|
1083
|
+
metadata=None,
|
|
1084
|
+
tag='default'):
|
|
1085
|
+
"""
|
|
1086
|
+
Set IV-dIdV Sweep processed dataframe
|
|
1087
|
+
"""
|
|
1088
|
+
|
|
1089
|
+
|
|
1090
|
+
# check dataframe
|
|
1091
|
+
if not isinstance(dataframe, pd.DataFrame):
|
|
1092
|
+
raise ValueError(
|
|
1093
|
+
'ERROR: Input is not a pandas Datafame!')
|
|
1094
|
+
|
|
1095
|
+
# create channel dictionary
|
|
1096
|
+
if channel not in self._filter_data.keys():
|
|
1097
|
+
self._filter_data[channel] = dict()
|
|
1098
|
+
|
|
1099
|
+
# data
|
|
1100
|
+
data_tag = 'ivsweep_data_' + tag
|
|
1101
|
+
self._filter_data[channel][data_tag] = dataframe
|
|
1102
|
+
|
|
1103
|
+
self._filter_data[channel][data_tag + '_metadata'] = (
|
|
1104
|
+
self._metadata_with_channel(metadata, channel)
|
|
1105
|
+
)
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def set_ivsweep_data_from_dict(self, data_dict,
|
|
1109
|
+
tag='default'):
|
|
1110
|
+
"""
|
|
1111
|
+
Set IV-dIdV sweep data from dictionary
|
|
1112
|
+
(key=channel name, value=datframe)
|
|
1113
|
+
"""
|
|
1114
|
+
|
|
1115
|
+
for chan, df in data_dict.items():
|
|
1116
|
+
self.set_ivsweep_data(chan, df, tag=tag)
|
|
1117
|
+
|
|
1118
|
+
def get_ivsweep_data(self,
|
|
1119
|
+
channel,
|
|
1120
|
+
tag='default'):
|
|
1121
|
+
"""
|
|
1122
|
+
Get IV-dIdV Sweep processed dataframe
|
|
1123
|
+
"""
|
|
1124
|
+
|
|
1125
|
+
# check channels
|
|
1126
|
+
if channel not in self._filter_data.keys():
|
|
1127
|
+
raise ValueError(
|
|
1128
|
+
f'ERROR: no channel {channel} available! '
|
|
1129
|
+
'Did you load from file first?')
|
|
1130
|
+
|
|
1131
|
+
data_tag = 'ivsweep_data_' + tag
|
|
1132
|
+
if data_tag not in self._filter_data[channel].keys():
|
|
1133
|
+
raise ValueError(
|
|
1134
|
+
f'ERROR: no sweep data for channel {channel} available! '
|
|
1135
|
+
'Did you load from file first?')
|
|
1136
|
+
|
|
1137
|
+
return self._filter_data[channel][data_tag]
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def set_ivsweep_results(self,
|
|
1141
|
+
channel,
|
|
1142
|
+
results,
|
|
1143
|
+
iv_type,
|
|
1144
|
+
metadata=None,
|
|
1145
|
+
tag='default'):
|
|
1146
|
+
"""
|
|
1147
|
+
Set IV-dIdV Sweep analysis results
|
|
1148
|
+
(independent of bias point)
|
|
1149
|
+
"""
|
|
1150
|
+
|
|
1151
|
+
# check input
|
|
1152
|
+
if isinstance(results, dict):
|
|
1153
|
+
results = pd.Series(results)
|
|
1154
|
+
elif not isinstance(results, pd.Series):
|
|
1155
|
+
raise ValueError(
|
|
1156
|
+
'ERROR: Input is not a pandas Series!')
|
|
1157
|
+
|
|
1158
|
+
# create channel dictionary
|
|
1159
|
+
if channel not in self._filter_data.keys():
|
|
1160
|
+
self._filter_data[channel] = dict()
|
|
1161
|
+
|
|
1162
|
+
# data
|
|
1163
|
+
data_tag = 'ivsweep_results_' + iv_type + '_' + tag
|
|
1164
|
+
self._filter_data[channel][data_tag] = results
|
|
1165
|
+
|
|
1166
|
+
self._filter_data[channel][data_tag + '_metadata'] = (
|
|
1167
|
+
self._metadata_with_channel(metadata, channel)
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
def get_ivsweep_results(self,
|
|
1172
|
+
channel,
|
|
1173
|
+
iv_type='iv',
|
|
1174
|
+
include_bias_parameters=False,
|
|
1175
|
+
tes_bias=None,
|
|
1176
|
+
lgc_return_series=False,
|
|
1177
|
+
tag='default'):
|
|
1178
|
+
"""
|
|
1179
|
+
Get IV-dIdV Sweep result
|
|
1180
|
+
"""
|
|
1181
|
+
|
|
1182
|
+
# check channels
|
|
1183
|
+
if channel not in self._filter_data.keys():
|
|
1184
|
+
raise ValueError(
|
|
1185
|
+
f'ERROR: no channel {channel} available! '
|
|
1186
|
+
'Did you load from file first?')
|
|
1187
|
+
|
|
1188
|
+
# check argument
|
|
1189
|
+
if (include_bias_parameters
|
|
1190
|
+
and tes_bias is None):
|
|
1191
|
+
raise ValueError(
|
|
1192
|
+
f'ERROR: "tes_bias" needs to be provided '
|
|
1193
|
+
f'when "include_bias_parameters" = True!')
|
|
1194
|
+
|
|
1195
|
+
# result data tag
|
|
1196
|
+
data_tag = f'ivsweep_results_{iv_type}_{tag}'
|
|
1197
|
+
if iv_type == 'iv' and data_tag not in self._filter_data[channel].keys():
|
|
1198
|
+
data_tag = f'ivsweep_results_noise_{tag}' # legacy
|
|
1199
|
+
|
|
1200
|
+
data_series = self._filter_data[channel].get(data_tag)
|
|
1201
|
+
|
|
1202
|
+
if data_series is None:
|
|
1203
|
+
|
|
1204
|
+
# not available -> check for didv
|
|
1205
|
+
iv_type_new = 'didv'
|
|
1206
|
+
if iv_type == 'didv':
|
|
1207
|
+
iv_type_new = 'iv'
|
|
1208
|
+
data_tag = f'ivsweep_results_{iv_type}_{tag}'
|
|
1209
|
+
data_series = self._filter_data[channel].get(data_tag)
|
|
1210
|
+
|
|
1211
|
+
if data_series is not None:
|
|
1212
|
+
|
|
1213
|
+
raise ValueError(
|
|
1214
|
+
f'ERROR: No sweep results for channel {channel} available '
|
|
1215
|
+
f'using data type "{iv_type}"! Change "iv_type" '
|
|
1216
|
+
f'argument to {iv_type_new}')
|
|
1217
|
+
|
|
1218
|
+
else:
|
|
1219
|
+
|
|
1220
|
+
raise ValueError(
|
|
1221
|
+
f'ERROR: No sweep results for channel {channel} available. Did '
|
|
1222
|
+
'you run the sweep analysis?')
|
|
1223
|
+
|
|
1224
|
+
# convert to dictionary
|
|
1225
|
+
results = data_series.to_dict()
|
|
1226
|
+
|
|
1227
|
+
# include bias parameters
|
|
1228
|
+
if include_bias_parameters:
|
|
1229
|
+
|
|
1230
|
+
# get dataframe
|
|
1231
|
+
df = self.get_ivsweep_data(channel, tag=tag)
|
|
1232
|
+
absolute_difference = abs(df['tes_bias'] - tes_bias)
|
|
1233
|
+
closest_index = absolute_difference.idxmin()
|
|
1234
|
+
params = df.loc[closest_index].to_dict()
|
|
1235
|
+
|
|
1236
|
+
# add parameters
|
|
1237
|
+
results['tes_bias'] = params['tes_bias']
|
|
1238
|
+
results['ibias'] = params['ibias_true_' + iv_type]
|
|
1239
|
+
results['ibias_err'] = params['ibias_true_err_' + iv_type]
|
|
1240
|
+
results['i0'] = params['i0_' + iv_type]
|
|
1241
|
+
results['i0_err'] = params['i0_err_' + iv_type]
|
|
1242
|
+
results['r0'] = params['r0_' + iv_type]
|
|
1243
|
+
results['r0_err'] = params['r0_err_' + iv_type]
|
|
1244
|
+
results['p0'] = params['p0_' + iv_type]
|
|
1245
|
+
results['p0_err'] = params['p0_err_' + iv_type]
|
|
1246
|
+
|
|
1247
|
+
# add temperature
|
|
1248
|
+
temperature_list = ['mc','cp','still']
|
|
1249
|
+
for temp in temperature_list:
|
|
1250
|
+
temp_par = 'temperature_' + temp
|
|
1251
|
+
temp_val = np.nan
|
|
1252
|
+
if temp_par in params:
|
|
1253
|
+
results[temp_par] = params[temp_par]
|
|
1254
|
+
|
|
1255
|
+
# infinite loop gain
|
|
1256
|
+
if 'didv_3poles_r0_infinite_lgain' in params:
|
|
1257
|
+
|
|
1258
|
+
results['i0_infinite_lgain'] = (
|
|
1259
|
+
params['didv_3poles_i0_infinite_lgain']
|
|
1260
|
+
)
|
|
1261
|
+
results['i0_err_infinite_lgain'] = (
|
|
1262
|
+
params['didv_3poles_i0_err_infinite_lgain']
|
|
1263
|
+
)
|
|
1264
|
+
|
|
1265
|
+
results['r0_infinite_lgain'] = (
|
|
1266
|
+
params['didv_3poles_r0_infinite_lgain']
|
|
1267
|
+
)
|
|
1268
|
+
results['r0_err_infinite_lgain'] = (
|
|
1269
|
+
params['didv_3poles_r0_err_infinite_lgain']
|
|
1270
|
+
)
|
|
1271
|
+
|
|
1272
|
+
results['p0_infinite_lgain'] = (
|
|
1273
|
+
params['didv_3poles_p0_infinite_lgain']
|
|
1274
|
+
)
|
|
1275
|
+
results['p0_err_infinite_lgain'] = (
|
|
1276
|
+
params['didv_3poles_p0_err_infinite_lgain']
|
|
1277
|
+
)
|
|
1278
|
+
|
|
1279
|
+
# ssp
|
|
1280
|
+
didv_parameters = ['chi2',
|
|
1281
|
+
'tau+','tau-','tau3',
|
|
1282
|
+
'l', 'l_err',
|
|
1283
|
+
'beta', 'beta_err',
|
|
1284
|
+
'gratio', 'gratio_err',
|
|
1285
|
+
'tau0','tau0_err',
|
|
1286
|
+
'L','L_err']
|
|
1287
|
+
|
|
1288
|
+
for model_poles in [2,3]:
|
|
1289
|
+
|
|
1290
|
+
for didv_par in didv_parameters:
|
|
1291
|
+
par_name = f'didv_{model_poles}poles_{didv_par}'
|
|
1292
|
+
if par_name in params:
|
|
1293
|
+
results[par_name] = params[par_name]
|
|
1294
|
+
|
|
1295
|
+
if 'resolution_dirac' in params:
|
|
1296
|
+
results['resolution_dirac'] = params['resolution_dirac']
|
|
1297
|
+
results['resolution_collection_efficiency'] = (
|
|
1298
|
+
params['resolution_collection_efficiency']
|
|
1299
|
+
)
|
|
1300
|
+
if 'resolution_template' in params:
|
|
1301
|
+
results['resolution_template'] = params['resolution_template']
|
|
1302
|
+
|
|
1303
|
+
if lgc_return_series:
|
|
1304
|
+
results = pd.Series(results)
|
|
1305
|
+
|
|
1306
|
+
return results
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
def set_didv_results(self,
|
|
1311
|
+
channel,
|
|
1312
|
+
results,
|
|
1313
|
+
poles,
|
|
1314
|
+
metadata=None,
|
|
1315
|
+
tag='default'):
|
|
1316
|
+
"""
|
|
1317
|
+
Set results from dIdV fit
|
|
1318
|
+
"""
|
|
1319
|
+
|
|
1320
|
+
# check input
|
|
1321
|
+
if not isinstance(results, dict):
|
|
1322
|
+
raise ValueError(
|
|
1323
|
+
'ERROR: "results" argument should be a dictionary!')
|
|
1324
|
+
|
|
1325
|
+
if not isinstance(poles, int):
|
|
1326
|
+
raise ValueError(
|
|
1327
|
+
'ERROR: "poles" argument should be an interger!')
|
|
1328
|
+
|
|
1329
|
+
# create channel dictionary
|
|
1330
|
+
if channel not in self._filter_data.keys():
|
|
1331
|
+
self._filter_data[channel] = dict()
|
|
1332
|
+
|
|
1333
|
+
# base name, poles
|
|
1334
|
+
base_name = 'didv_results_' + str(poles) + 'poles'
|
|
1335
|
+
|
|
1336
|
+
metadata = self._metadata_with_channel(metadata, channel)
|
|
1337
|
+
|
|
1338
|
+
# sub-dictionaries
|
|
1339
|
+
subdict_list = ['biasparams', 'biasparams_infinite_lgain',
|
|
1340
|
+
'errors','params', 'smallsignalparams',
|
|
1341
|
+
'ssp_light']
|
|
1342
|
+
|
|
1343
|
+
# fit results
|
|
1344
|
+
fit_data = dict()
|
|
1345
|
+
for par_name, par_val in results.items():
|
|
1346
|
+
if not isinstance(par_val, dict):
|
|
1347
|
+
fit_data[par_name] = par_val
|
|
1348
|
+
|
|
1349
|
+
if fit_data:
|
|
1350
|
+
pd_series = pd.Series(fit_data)
|
|
1351
|
+
data_name = base_name + '_fit_' + tag
|
|
1352
|
+
self._filter_data[channel][data_name] = pd_series
|
|
1353
|
+
self._filter_data[channel][data_name + '_metadata'] = (
|
|
1354
|
+
metadata
|
|
1355
|
+
)
|
|
1356
|
+
|
|
1357
|
+
for keyname in subdict_list:
|
|
1358
|
+
|
|
1359
|
+
if (keyname not in results.keys() or
|
|
1360
|
+
not isinstance(results[keyname], dict)):
|
|
1361
|
+
continue
|
|
1362
|
+
|
|
1363
|
+
data_dict = results[keyname]
|
|
1364
|
+
if (keyname == 'ssp_light'
|
|
1365
|
+
and 'vals' in results['ssp_light'].keys()):
|
|
1366
|
+
data_dict = dict()
|
|
1367
|
+
data_dict['cov'] = results['ssp_light']['cov']
|
|
1368
|
+
data_dict.update(results['ssp_light']['vals'])
|
|
1369
|
+
data_dict.update(results['ssp_light']['sigmas'])
|
|
1370
|
+
|
|
1371
|
+
pd_series = pd.Series(data_dict)
|
|
1372
|
+
data_name = base_name + '_' + keyname + '_' + tag
|
|
1373
|
+
self._filter_data[channel][data_name] = pd_series
|
|
1374
|
+
self._filter_data[channel][data_name + '_metadata'] = (
|
|
1375
|
+
metadata
|
|
1376
|
+
)
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
def get_didv_results(self,
|
|
1380
|
+
channel,
|
|
1381
|
+
poles,
|
|
1382
|
+
tag='default'):
|
|
1383
|
+
"""
|
|
1384
|
+
Get dIdV fit results
|
|
1385
|
+
"""
|
|
1386
|
+
|
|
1387
|
+
# check channels
|
|
1388
|
+
if channel not in self._filter_data.keys():
|
|
1389
|
+
raise ValueError(
|
|
1390
|
+
f'ERROR: no channel {channel} available! '
|
|
1391
|
+
)
|
|
1392
|
+
|
|
1393
|
+
output_data = dict()
|
|
1394
|
+
|
|
1395
|
+
# base name
|
|
1396
|
+
base_name = 'didv_results_' + str(poles) + 'poles'
|
|
1397
|
+
|
|
1398
|
+
# fit
|
|
1399
|
+
par_name = base_name + '_fit_' + tag
|
|
1400
|
+
if par_name not in self._filter_data[channel].keys():
|
|
1401
|
+
raise ValueError(f'ERROR: No dIdV {poles}-poles fit results '
|
|
1402
|
+
f'for channel {channel}!')
|
|
1403
|
+
|
|
1404
|
+
output_data.update(self._filter_data[channel][par_name].to_dict())
|
|
1405
|
+
output_data['metadata'] = (
|
|
1406
|
+
self._filter_data[channel][par_name + '_metadata']
|
|
1407
|
+
)
|
|
1408
|
+
|
|
1409
|
+
# other par list
|
|
1410
|
+
par_list = ['biasparams', 'biasparams_infinite_lgain',
|
|
1411
|
+
'errors','params', 'smallsignalparams',
|
|
1412
|
+
'ssp_light']
|
|
1413
|
+
|
|
1414
|
+
for par in par_list:
|
|
1415
|
+
par_name = base_name + '_' + par + '_' + tag
|
|
1416
|
+
if par_name in self._filter_data[channel].keys():
|
|
1417
|
+
output_data[par] = self._filter_data[channel][par_name].to_dict()
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
if 'ssp_light' in output_data:
|
|
1421
|
+
ssp_light = {'vals':dict(), 'sigmas':dict()}
|
|
1422
|
+
for par, val in output_data['ssp_light'].items():
|
|
1423
|
+
if par == 'cov':
|
|
1424
|
+
continue
|
|
1425
|
+
if 'sigma' in par:
|
|
1426
|
+
ssp_light['sigmas'][par] = val
|
|
1427
|
+
else:
|
|
1428
|
+
ssp_light['vals'][par] = val
|
|
1429
|
+
|
|
1430
|
+
output_data['ssp_light'].update(ssp_light)
|
|
1431
|
+
|
|
1432
|
+
dpdipar = 'dpdi_' + str(poles) + 'poles_' + tag
|
|
1433
|
+
if dpdipar in self._filter_data[channel].keys():
|
|
1434
|
+
output_data[dpdipar] = self._filter_data[channel][dpdipar].to_dict()
|
|
1435
|
+
|
|
1436
|
+
return output_data
|
|
1437
|
+
|
|
1438
|
+
def set_didv_dataframe(self,
|
|
1439
|
+
channel,
|
|
1440
|
+
dataframe,
|
|
1441
|
+
metadata=None,
|
|
1442
|
+
tag='default'):
|
|
1443
|
+
"""
|
|
1444
|
+
Set dIdV processing
|
|
1445
|
+
"""
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
# check dataframe
|
|
1449
|
+
if not isinstance(dataframe, pd.DataFrame):
|
|
1450
|
+
raise ValueError(
|
|
1451
|
+
'ERROR: Input is not a pandas Datafame!')
|
|
1452
|
+
|
|
1453
|
+
# create channel dictionary
|
|
1454
|
+
if channel not in self._filter_data.keys():
|
|
1455
|
+
self._filter_data[channel] = dict()
|
|
1456
|
+
|
|
1457
|
+
# data
|
|
1458
|
+
data_tag = 'didv_processing_' + tag
|
|
1459
|
+
self._filter_data[channel][data_tag] = dataframe
|
|
1460
|
+
|
|
1461
|
+
self._filter_data[channel][data_tag + '_metadata'] = (
|
|
1462
|
+
self._metadata_with_channel(metadata, channel)
|
|
1463
|
+
)
|
|
1464
|
+
|
|
1465
|
+
def get_didv_dataframe(self,
|
|
1466
|
+
channel,
|
|
1467
|
+
tag='default'):
|
|
1468
|
+
"""
|
|
1469
|
+
Get dIdV processed dataframe
|
|
1470
|
+
"""
|
|
1471
|
+
|
|
1472
|
+
# check channels
|
|
1473
|
+
if channel not in self._filter_data.keys():
|
|
1474
|
+
raise ValueError(
|
|
1475
|
+
f'ERROR: no channel {channel} available! '
|
|
1476
|
+
'Did you load from file first?')
|
|
1477
|
+
|
|
1478
|
+
data_tag = 'didv_processing_' + tag
|
|
1479
|
+
if data_tag not in self._filter_data[channel].keys():
|
|
1480
|
+
raise ValueError(
|
|
1481
|
+
f'ERROR: no dIdV data for channel {channel} available! '
|
|
1482
|
+
'Did you load from file first?')
|
|
1483
|
+
|
|
1484
|
+
return self._filter_data[channel][data_tag]
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def set_noise_dataframe(self,
|
|
1488
|
+
channel,
|
|
1489
|
+
dataframe,
|
|
1490
|
+
metadata=None,
|
|
1491
|
+
tag='default'):
|
|
1492
|
+
"""
|
|
1493
|
+
Set noise processing
|
|
1494
|
+
"""
|
|
1495
|
+
|
|
1496
|
+
|
|
1497
|
+
# check dataframe
|
|
1498
|
+
if not isinstance(dataframe, pd.DataFrame):
|
|
1499
|
+
raise ValueError(
|
|
1500
|
+
'ERROR: Input is not a pandas Datafame!')
|
|
1501
|
+
|
|
1502
|
+
# create channel dictionary
|
|
1503
|
+
if channel not in self._filter_data.keys():
|
|
1504
|
+
self._filter_data[channel] = dict()
|
|
1505
|
+
|
|
1506
|
+
# data
|
|
1507
|
+
data_tag = 'noise_processing_' + tag
|
|
1508
|
+
self._filter_data[channel][data_tag] = dataframe
|
|
1509
|
+
|
|
1510
|
+
self._filter_data[channel][data_tag + '_metadata'] = (
|
|
1511
|
+
self._metadata_with_channel(metadata, channel)
|
|
1512
|
+
)
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def get_noise_dataframe(self,
|
|
1516
|
+
channel,
|
|
1517
|
+
tag='default'):
|
|
1518
|
+
"""
|
|
1519
|
+
Get noise processed dataframe
|
|
1520
|
+
"""
|
|
1521
|
+
|
|
1522
|
+
# check channels
|
|
1523
|
+
if channel not in self._filter_data.keys():
|
|
1524
|
+
raise ValueError(
|
|
1525
|
+
f'ERROR: no channel {channel} available! '
|
|
1526
|
+
'Did you load from file first?')
|
|
1527
|
+
|
|
1528
|
+
data_tag = 'noise_processing_' + tag
|
|
1529
|
+
if data_tag not in self._filter_data[channel].keys():
|
|
1530
|
+
raise ValueError(
|
|
1531
|
+
f'ERROR: no noise data for channel {channel} available! '
|
|
1532
|
+
'Did you load from file first?')
|
|
1533
|
+
|
|
1534
|
+
return self._filter_data[channel][data_tag]
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def plot_template(self, channels,
|
|
1538
|
+
xmin=None, xmax=None,
|
|
1539
|
+
tag='default',
|
|
1540
|
+
figsize=(8,5)):
|
|
1541
|
+
"""
|
|
1542
|
+
Plot template for specified channel(s)
|
|
1543
|
+
|
|
1544
|
+
Parameters:
|
|
1545
|
+
----------
|
|
1546
|
+
|
|
1547
|
+
channels : str or list of str (required)
|
|
1548
|
+
channel name or list of channels
|
|
1549
|
+
|
|
1550
|
+
tag : str (optional)
|
|
1551
|
+
psd name suffix: "psd_[tag]" or "psd_fold_[tag]"
|
|
1552
|
+
if tag is None, then "psd" or "psd_fold" is used
|
|
1553
|
+
|
|
1554
|
+
Return:
|
|
1555
|
+
-------
|
|
1556
|
+
None
|
|
1557
|
+
|
|
1558
|
+
"""
|
|
1559
|
+
|
|
1560
|
+
if isinstance(channels, str):
|
|
1561
|
+
channels = [channels]
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
# define fig size
|
|
1566
|
+
fig, ax = plt.subplots(figsize=figsize)
|
|
1567
|
+
|
|
1568
|
+
|
|
1569
|
+
for chan in channels:
|
|
1570
|
+
template, t = self.get_template(chan, tag=tag)
|
|
1571
|
+
if template is None:
|
|
1572
|
+
continue
|
|
1573
|
+
ax.plot(t*1e3, template, label=chan)
|
|
1574
|
+
|
|
1575
|
+
# add axis
|
|
1576
|
+
ax.legend()
|
|
1577
|
+
ax.tick_params(which='both', direction='in', right=True, top=True)
|
|
1578
|
+
ax.grid(which='minor', linestyle='dotted')
|
|
1579
|
+
ax.grid(which='major')
|
|
1580
|
+
ax.set_title('Template', fontweight='bold')
|
|
1581
|
+
ax.set_xlabel('Time [msec]', fontweight='bold')
|
|
1582
|
+
|
|
1583
|
+
if (xmin is not None or xmax is not None):
|
|
1584
|
+
ax.set_xlim(xmin=xmin, xmax=xmax)
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
def plot_psd(self, channels, tag='default',
|
|
1589
|
+
unit='pA', figsize=(8,5)):
|
|
1590
|
+
"""
|
|
1591
|
+
Plot PSD for specified channel(s)
|
|
1592
|
+
|
|
1593
|
+
Parameters:
|
|
1594
|
+
----------
|
|
1595
|
+
|
|
1596
|
+
channels : str or list of str (required)
|
|
1597
|
+
channel name or list of channels
|
|
1598
|
+
|
|
1599
|
+
tag : str (optional)
|
|
1600
|
+
psd name suffix: "psd_[tag]" or "psd_fold_[tag]"
|
|
1601
|
+
if tag is None, then "psd" or "psd_fold" is used
|
|
1602
|
+
|
|
1603
|
+
unit : str (optional, default='pA')
|
|
1604
|
+
plot in Amps ('A') or pico Amps 'pA')
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
Return:
|
|
1608
|
+
-------
|
|
1609
|
+
None
|
|
1610
|
+
|
|
1611
|
+
"""
|
|
1612
|
+
|
|
1613
|
+
if isinstance(channels, str):
|
|
1614
|
+
channels = [channels]
|
|
1615
|
+
|
|
1616
|
+
# define fig size
|
|
1617
|
+
fig, ax = plt.subplots(figsize=figsize)
|
|
1618
|
+
|
|
1619
|
+
for chan in channels:
|
|
1620
|
+
|
|
1621
|
+
psd, freq = self.get_psd(chan, tag=tag, fold=True)
|
|
1622
|
+
|
|
1623
|
+
if psd is None:
|
|
1624
|
+
continue
|
|
1625
|
+
|
|
1626
|
+
# convert to A/rtHz
|
|
1627
|
+
psd = psd**0.5
|
|
1628
|
+
|
|
1629
|
+
if unit=='pA':
|
|
1630
|
+
psd *= 1e12
|
|
1631
|
+
|
|
1632
|
+
ax.loglog(freq, psd, label=chan)
|
|
1633
|
+
|
|
1634
|
+
# add axis
|
|
1635
|
+
ax.legend()
|
|
1636
|
+
ax.tick_params(which='both', direction='in', right=True, top=True)
|
|
1637
|
+
ax.grid(which='minor', linestyle='dotted')
|
|
1638
|
+
ax.grid(which='major')
|
|
1639
|
+
if tag != 'default':
|
|
1640
|
+
ax.set_title(f'Noise PSD ("{tag}")',
|
|
1641
|
+
fontweight='bold')
|
|
1642
|
+
else:
|
|
1643
|
+
ax.set_title(f'Noise PSD',fontweight='bold')
|
|
1644
|
+
|
|
1645
|
+
ax.set_xlabel('Frequency [Hz]',fontweight='bold')
|
|
1646
|
+
if unit=='pA':
|
|
1647
|
+
ax.set_ylabel('PSD [pA/rtHz]',fontweight='bold')
|
|
1648
|
+
else:
|
|
1649
|
+
ax.set_ylabel('PSD [A/rtHz]',fontweight='bold')
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def plot_csd(self, channels, whichcsd=['01'], lgcreal=True,
|
|
1653
|
+
lgcsave=False, savepath=None, figsize=(8,5),
|
|
1654
|
+
tag='default'):
|
|
1655
|
+
"""
|
|
1656
|
+
Plot CSD elements
|
|
1657
|
+
"""
|
|
1658
|
+
|
|
1659
|
+
csd, csd_freqs = self.get_csd(channels, tag=tag, fold=True)
|
|
1660
|
+
|
|
1661
|
+
if isinstance(channels, str):
|
|
1662
|
+
channels = channels.split('|')
|
|
1663
|
+
|
|
1664
|
+
plotting.plot_csd(csd=csd.copy(),
|
|
1665
|
+
csd_freqs=csd_freqs.copy(),
|
|
1666
|
+
channels=channels,
|
|
1667
|
+
whichcsd=whichcsd, lgcreal=lgcreal,
|
|
1668
|
+
lgcsave=lgcsave,
|
|
1669
|
+
savepath=savepath,
|
|
1670
|
+
figsize=figsize)
|
|
1671
|
+
|
|
1672
|
+
|
|
1673
|
+
def plot_corrcoeff(self, channels, lgcsmooth=True, nwindow=7,
|
|
1674
|
+
lgcsave=False, savepath=None, figsize=(8,5),
|
|
1675
|
+
tag='default'):
|
|
1676
|
+
"""
|
|
1677
|
+
Plot correlation coefficient
|
|
1678
|
+
"""
|
|
1679
|
+
|
|
1680
|
+
csd, csd_freqs = self.get_csd(channels, tag=tag, fold=True)
|
|
1681
|
+
corrcoeff = calc_corrcoeff_from_csd(csd.copy())
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
if isinstance(channels, str):
|
|
1685
|
+
channels = channels.split('|')
|
|
1686
|
+
|
|
1687
|
+
plotting.plot_corrcoeff(
|
|
1688
|
+
corrcoeff=corrcoeff.copy(),
|
|
1689
|
+
corrcoeff_freqs=csd_freqs.copy(),
|
|
1690
|
+
channels=channels,
|
|
1691
|
+
lgcsmooth=lgcsmooth,
|
|
1692
|
+
nwindow=nwindow, lgcsave=lgcsave,
|
|
1693
|
+
savepath=savepath, figsize=figsize)
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def plot_ivsweep_offset(self, channel, tag='default'):
|
|
1697
|
+
"""
|
|
1698
|
+
Plot offset vs tes_bias with errors from IV and if available
|
|
1699
|
+
dIdV offset
|
|
1700
|
+
"""
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
# get data frame
|
|
1704
|
+
df = self.get_ivsweep_data(channel=channel,
|
|
1705
|
+
tag=tag)
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
if 'tes_bias_ua' in df.columns:
|
|
1709
|
+
bias = df['tes_bias_ua'].values
|
|
1710
|
+
elif 'tes_bias_uA' in df.columns: # legacy
|
|
1711
|
+
bias = df['tes_bias_uA'].values
|
|
1712
|
+
else:
|
|
1713
|
+
raise ValueError(
|
|
1714
|
+
'ERROR: IV sweep dataframe is missing tes_bias_ua/tes_bias_uA')
|
|
1715
|
+
offset_noise = None
|
|
1716
|
+
offset_noise_err = None
|
|
1717
|
+
offset_didv = None
|
|
1718
|
+
offset_didv_err = None
|
|
1719
|
+
iv_offset_name = None
|
|
1720
|
+
iv_offset_err_name = None
|
|
1721
|
+
iv_offset_label = 'Offset from IV'
|
|
1722
|
+
if 'offset_iv' in df.columns:
|
|
1723
|
+
iv_offset_name = 'offset_iv'
|
|
1724
|
+
iv_offset_err_name = 'offset_err_iv'
|
|
1725
|
+
elif 'offset_noise' in df.columns: # legacy analysis naming
|
|
1726
|
+
iv_offset_name = 'offset_noise'
|
|
1727
|
+
iv_offset_err_name = 'offset_err_noise'
|
|
1728
|
+
iv_offset_label = 'Offset from noise'
|
|
1729
|
+
|
|
1730
|
+
if iv_offset_name is not None:
|
|
1731
|
+
offset_noise = df[iv_offset_name].values*1e6
|
|
1732
|
+
offset_noise_err = df[iv_offset_err_name].values*1e6
|
|
1733
|
+
if 'offset_didv' in df.columns:
|
|
1734
|
+
offset_didv = df['offset_didv'].values*1e6
|
|
1735
|
+
offset_didv_err = df['offset_err_didv'].values*1e6
|
|
1736
|
+
|
|
1737
|
+
# Plotting the data with error bars
|
|
1738
|
+
if offset_noise is not None:
|
|
1739
|
+
plt.errorbar(bias, offset_noise,
|
|
1740
|
+
yerr=offset_noise_err,
|
|
1741
|
+
fmt='o', color='b',capsize=5,
|
|
1742
|
+
label=iv_offset_label)
|
|
1743
|
+
if offset_didv is not None:
|
|
1744
|
+
plt.errorbar(bias, offset_didv,
|
|
1745
|
+
yerr=offset_didv_err,
|
|
1746
|
+
fmt='+', color='r', capsize=5,
|
|
1747
|
+
label='Offset from didv')
|
|
1748
|
+
|
|
1749
|
+
# Some basic plot settings
|
|
1750
|
+
plt.title(f'TES bias sweep {channel}',fontweight='bold')
|
|
1751
|
+
plt.xlabel('TES bias [uA]',fontweight='bold')
|
|
1752
|
+
plt.ylabel('Offset [uA]', fontweight='bold')
|
|
1753
|
+
plt.legend(loc='best')
|
|
1754
|
+
plt.grid(True)
|
|
1755
|
+
plt.show()
|
|
1756
|
+
|
|
1757
|
+
|
|
1758
|
+
def _get_param_array(self, param_name, channels, tag='default',
|
|
1759
|
+
return_metadata=False):
|
|
1760
|
+
"""
|
|
1761
|
+
Get parameter stored in dictionary
|
|
1762
|
+
|
|
1763
|
+
Parameters:
|
|
1764
|
+
----------
|
|
1765
|
+
|
|
1766
|
+
param_name : str
|
|
1767
|
+
name of parameter
|
|
1768
|
+
|
|
1769
|
+
channel : str or list of str
|
|
1770
|
+
channel name or list of channels
|
|
1771
|
+
|
|
1772
|
+
tag : str, optional
|
|
1773
|
+
psd tag, default: No tag
|
|
1774
|
+
|
|
1775
|
+
Return
|
|
1776
|
+
------
|
|
1777
|
+
|
|
1778
|
+
array : ndarray,
|
|
1779
|
+
nD numpy array corresponding to parameters
|
|
1780
|
+
f : ndarray
|
|
1781
|
+
array frequencies
|
|
1782
|
+
|
|
1783
|
+
"""
|
|
1784
|
+
# check channel
|
|
1785
|
+
channel_list = convert_channel_name_to_list(channels)
|
|
1786
|
+
channel_name = convert_channel_list_to_name(channels)
|
|
1787
|
+
if (channel_name not in self._filter_data.keys()):
|
|
1788
|
+
msg = f'ERROR: Channel "{channel_name}" not available!'
|
|
1789
|
+
|
|
1790
|
+
if self._filter_data.keys():
|
|
1791
|
+
msg += ' List of channels in filter file: '
|
|
1792
|
+
msg += str(list(self._filter_data.keys()))
|
|
1793
|
+
|
|
1794
|
+
raise ValueError(msg)
|
|
1795
|
+
|
|
1796
|
+
|
|
1797
|
+
# parameter name
|
|
1798
|
+
data_name = param_name + '_' + tag
|
|
1799
|
+
metadata_name = data_name + '_metadata'
|
|
1800
|
+
inds_name = data_name + '_inds'
|
|
1801
|
+
|
|
1802
|
+
# check available tag
|
|
1803
|
+
if data_name not in self._filter_data[channel_name].keys():
|
|
1804
|
+
raise ValueError(f'ERROR: Parameter {data_name} not found '
|
|
1805
|
+
f'for channel {channel_name}!')
|
|
1806
|
+
|
|
1807
|
+
data = copy.deepcopy(self._filter_data[channel_name][data_name])
|
|
1808
|
+
vals_inds = None
|
|
1809
|
+
vals = None
|
|
1810
|
+
if isinstance(data, pd.Series):
|
|
1811
|
+
vals = data.values
|
|
1812
|
+
vals_inds = data.index
|
|
1813
|
+
vals_inds = vals_inds.to_numpy()
|
|
1814
|
+
|
|
1815
|
+
elif isinstance(data, np.ndarray):
|
|
1816
|
+
vals = data.copy()
|
|
1817
|
+
if inds_name in self._filter_data[channel_name]:
|
|
1818
|
+
vals_inds = self._filter_data[channel_name][inds_name]
|
|
1819
|
+
if isinstance(vals_inds, pd.Series):
|
|
1820
|
+
vals_inds = vals_inds.values
|
|
1821
|
+
elif param_name == 'csd':
|
|
1822
|
+
# Legacy Noise.calc_csd stored frequencies as
|
|
1823
|
+
# ``csd_freqs_<tag>`` instead of the canonical
|
|
1824
|
+
# ``csd_<tag>_inds`` key.
|
|
1825
|
+
legacy_inds_name = 'csd_freqs_' + tag
|
|
1826
|
+
if legacy_inds_name in self._filter_data[channel_name]:
|
|
1827
|
+
vals_inds = self._filter_data[channel_name][legacy_inds_name]
|
|
1828
|
+
if isinstance(vals_inds, pd.Series):
|
|
1829
|
+
vals_inds = vals_inds.values
|
|
1830
|
+
|
|
1831
|
+
metadata = dict()
|
|
1832
|
+
if metadata_name in self._filter_data[channel_name].keys():
|
|
1833
|
+
metadata = copy.deepcopy(
|
|
1834
|
+
self._filter_data[channel_name][metadata_name]
|
|
1835
|
+
)
|
|
1836
|
+
|
|
1837
|
+
if return_metadata:
|
|
1838
|
+
return vals, vals_inds, metadata
|
|
1839
|
+
else:
|
|
1840
|
+
return vals, vals_inds
|
|
1841
|
+
|
|
1842
|
+
def _split_parameter_name(self, parameter_name, base_list):
|
|
1843
|
+
# Match longest base first to avoid partial matches
|
|
1844
|
+
for base in sorted(base_list, key=len, reverse=True):
|
|
1845
|
+
prefix = base + "_"
|
|
1846
|
+
if parameter_name.startswith(prefix):
|
|
1847
|
+
tag = parameter_name[len(prefix):]
|
|
1848
|
+
return base, tag
|
|
1849
|
+
return None, None # or raise ValueError
|