DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,771 @@
|
|
|
1
|
+
"""Compute template waveforms, including accounting for drift and denoising
|
|
2
|
+
|
|
3
|
+
The class TemplateData in templates.py provides a friendlier interface,
|
|
4
|
+
where you can get templates using the TemplateConfig in config.py.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import threading
|
|
8
|
+
from typing import cast
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import torch
|
|
12
|
+
from scipy.spatial import KDTree
|
|
13
|
+
from scipy.spatial.distance import pdist
|
|
14
|
+
from sklearn.decomposition import TruncatedSVD, PCA
|
|
15
|
+
from tqdm.auto import tqdm
|
|
16
|
+
|
|
17
|
+
from ..util import spikeio
|
|
18
|
+
from ..util.data_util import DARTsortSorting, load_stored_tsvd
|
|
19
|
+
from ..util.drift_util import (
|
|
20
|
+
registered_template,
|
|
21
|
+
registered_geometry,
|
|
22
|
+
get_spike_pitch_shifts,
|
|
23
|
+
)
|
|
24
|
+
from ..util.multiprocessing_util import get_pool
|
|
25
|
+
from ..util.spiketorch import fast_nanmedian, ptp
|
|
26
|
+
from ..util.waveform_util import make_channel_index
|
|
27
|
+
from ..util.internal_config import RealignStrategy
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_templates(
|
|
31
|
+
recording,
|
|
32
|
+
sorting,
|
|
33
|
+
trough_offset_samples=42,
|
|
34
|
+
spike_length_samples=121,
|
|
35
|
+
spikes_per_unit=500,
|
|
36
|
+
motion_est=None,
|
|
37
|
+
geom=None,
|
|
38
|
+
pitch_shifts=None,
|
|
39
|
+
registered_geom=None,
|
|
40
|
+
low_rank_denoising=True,
|
|
41
|
+
denoising_tsvd=None,
|
|
42
|
+
denoising_rank=5,
|
|
43
|
+
denoising_fit_radius=75.0,
|
|
44
|
+
denoising_spikes_fit=25_000,
|
|
45
|
+
recompute_tsvd=False,
|
|
46
|
+
denoising_snr_threshold=50.0,
|
|
47
|
+
min_fraction_at_shift=0.25,
|
|
48
|
+
min_count_at_shift=25,
|
|
49
|
+
with_raw_std_dev=False,
|
|
50
|
+
reducer=fast_nanmedian,
|
|
51
|
+
random_seed=0,
|
|
52
|
+
units_per_job=8,
|
|
53
|
+
n_jobs=0,
|
|
54
|
+
dtype=np.float32,
|
|
55
|
+
show_progress=True,
|
|
56
|
+
device=None,
|
|
57
|
+
localizations_dataset_name="point_source_localizations",
|
|
58
|
+
times_s_dataset_name="times_seconds",
|
|
59
|
+
):
|
|
60
|
+
"""Raw, denoised, and shifted templates
|
|
61
|
+
|
|
62
|
+
Low-level helper function which does the work of template computation for
|
|
63
|
+
the template classes elsewhere in this folder
|
|
64
|
+
|
|
65
|
+
Arguments
|
|
66
|
+
---------
|
|
67
|
+
times, channels, labels : arrays of shape (n_spikes,)
|
|
68
|
+
The trough (or peak) times, main channels, and unit labels
|
|
69
|
+
geom : array of shape (n_channels, 2)
|
|
70
|
+
Probe channel geometry, needed to subsample channels when fitting
|
|
71
|
+
the low-rank denoising model, and also needed if the shifting
|
|
72
|
+
arguments are specified
|
|
73
|
+
pitch_shifts : int array of shape (n_spikes,)
|
|
74
|
+
When computing extended templates, these shifts are applied
|
|
75
|
+
before averaging
|
|
76
|
+
registered_geom : array of shape (n_channels_extended, 2)
|
|
77
|
+
Required if pitch_shifts is supplied. See drift_util.registered_geometry.
|
|
78
|
+
realign_peaks : bool
|
|
79
|
+
If True, a first round of raw templates are computed and used to shift
|
|
80
|
+
the spike times such that their peaks/troughs land on trough_offset_samples
|
|
81
|
+
trough_offset_samples, spike_length_samples : int
|
|
82
|
+
Waveform snippets will be loaded from times[i] - trough_offset_samples
|
|
83
|
+
to times[i] - trough_offset_samples + spike_length_samples
|
|
84
|
+
spikes_per_unit : int
|
|
85
|
+
Load at most this many randomly selected spikes per unit
|
|
86
|
+
low_rank_denoising : bool
|
|
87
|
+
Should we compute denoised templates? If not, raw averages.
|
|
88
|
+
denoising_model : sklearn Transformer
|
|
89
|
+
Pre-fit denoising model, in which case the next args are ignored
|
|
90
|
+
denoising_rank, denoising_fit_radius, denoising_spikes_fit
|
|
91
|
+
Parameters for the low rank model fit for denoising
|
|
92
|
+
denoising_snr_threshold : int
|
|
93
|
+
The SNR (=amplitude*sqrt(n_spikes)) threshold at which the
|
|
94
|
+
denoising is ignored and we just use the usual template
|
|
95
|
+
output_hdf5_filename : str or Path
|
|
96
|
+
Denoised and/or raw templates will be saved here under the dataset
|
|
97
|
+
names "raw_templates" and "denoised_templates"
|
|
98
|
+
keep_waveforms_in_hdf5 : bool
|
|
99
|
+
If True and output_hdf5_filename is supplied, waveforms extracted
|
|
100
|
+
for template computation are retained in the output hdf5. Else,
|
|
101
|
+
deleted to save disk space.
|
|
102
|
+
scratch_dir : str or Path
|
|
103
|
+
This is where a temporary directory will be made for intermediate
|
|
104
|
+
computations, if output_hdf5_filename is None. If it's left blank,
|
|
105
|
+
the tempfile default directory is used. If output_hdf5_file is not
|
|
106
|
+
None, that hdf5 file is used and this argument is ignored.
|
|
107
|
+
|
|
108
|
+
Returns
|
|
109
|
+
-------
|
|
110
|
+
dict whose keys vary based on the above arguments
|
|
111
|
+
|
|
112
|
+
"""
|
|
113
|
+
# validate arguments
|
|
114
|
+
raw_only = not low_rank_denoising
|
|
115
|
+
|
|
116
|
+
# use geometry and motion estimate to get pitch shifts and reg geom
|
|
117
|
+
if pitch_shifts is None and motion_est is not None:
|
|
118
|
+
assert geom is not None
|
|
119
|
+
registered_geom = registered_geometry(geom, motion_est=motion_est)
|
|
120
|
+
try:
|
|
121
|
+
spike_depths_um = getattr(sorting, localizations_dataset_name)[:, 2]
|
|
122
|
+
except AttributeError:
|
|
123
|
+
raise ValueError(
|
|
124
|
+
"Sorting must contain localizations in the attribute "
|
|
125
|
+
f"{localizations_dataset_name=} when computing registered templates."
|
|
126
|
+
)
|
|
127
|
+
spike_times_s = getattr(sorting, times_s_dataset_name)
|
|
128
|
+
pitch_shifts = get_spike_pitch_shifts(
|
|
129
|
+
spike_depths_um, geom, times_s=spike_times_s, motion_est=motion_est
|
|
130
|
+
)
|
|
131
|
+
if pitch_shifts is not None:
|
|
132
|
+
assert registered_geom is not None
|
|
133
|
+
geom_kw = dict(registered_geom=registered_geom)
|
|
134
|
+
else:
|
|
135
|
+
geom_kw = dict(registered_geom=geom)
|
|
136
|
+
|
|
137
|
+
# fit tsvd
|
|
138
|
+
need_denoiser = denoising_tsvd is None and (low_rank_denoising)
|
|
139
|
+
if need_denoiser:
|
|
140
|
+
denoising_tsvd = fit_tsvd(
|
|
141
|
+
recording,
|
|
142
|
+
sorting,
|
|
143
|
+
dtype=dtype,
|
|
144
|
+
denoising_rank=denoising_rank,
|
|
145
|
+
denoising_fit_radius=denoising_fit_radius,
|
|
146
|
+
denoising_spikes_fit=denoising_spikes_fit,
|
|
147
|
+
trough_offset_samples=trough_offset_samples,
|
|
148
|
+
spike_length_samples=spike_length_samples,
|
|
149
|
+
recompute_tsvd=recompute_tsvd,
|
|
150
|
+
random_seed=random_seed,
|
|
151
|
+
)
|
|
152
|
+
elif not low_rank_denoising:
|
|
153
|
+
denoising_tsvd = None
|
|
154
|
+
|
|
155
|
+
# template logic
|
|
156
|
+
# for each unit, get shifted raw and denoised averages and channel SNRs
|
|
157
|
+
res = get_all_shifted_raw_and_low_rank_templates(
|
|
158
|
+
recording,
|
|
159
|
+
sorting,
|
|
160
|
+
registered_geom=registered_geom,
|
|
161
|
+
denoising_tsvd=denoising_tsvd,
|
|
162
|
+
pitch_shifts=pitch_shifts,
|
|
163
|
+
spikes_per_unit=spikes_per_unit,
|
|
164
|
+
reducer=reducer,
|
|
165
|
+
n_jobs=n_jobs,
|
|
166
|
+
units_per_job=units_per_job,
|
|
167
|
+
random_seed=random_seed,
|
|
168
|
+
show_progress=show_progress,
|
|
169
|
+
dtype=dtype,
|
|
170
|
+
trough_offset_samples=trough_offset_samples,
|
|
171
|
+
spike_length_samples=spike_length_samples,
|
|
172
|
+
min_fraction_at_shift=min_fraction_at_shift,
|
|
173
|
+
min_count_at_shift=min_count_at_shift,
|
|
174
|
+
with_raw_std_dev=with_raw_std_dev,
|
|
175
|
+
device=device,
|
|
176
|
+
)
|
|
177
|
+
(
|
|
178
|
+
unit_ids,
|
|
179
|
+
spike_counts,
|
|
180
|
+
raw_templates,
|
|
181
|
+
raw_std_devs,
|
|
182
|
+
low_rank_templates,
|
|
183
|
+
snrs_by_channel,
|
|
184
|
+
spike_counts_by_channel,
|
|
185
|
+
) = res
|
|
186
|
+
|
|
187
|
+
if raw_only:
|
|
188
|
+
return dict(
|
|
189
|
+
sorting=sorting,
|
|
190
|
+
unit_ids=unit_ids,
|
|
191
|
+
spike_counts=spike_counts,
|
|
192
|
+
templates=raw_templates,
|
|
193
|
+
raw_templates=raw_templates,
|
|
194
|
+
raw_std_devs=raw_std_devs,
|
|
195
|
+
snrs_by_channel=snrs_by_channel,
|
|
196
|
+
spike_counts_by_channel=spike_counts_by_channel,
|
|
197
|
+
denoising_tsvd=denoising_tsvd,
|
|
198
|
+
**geom_kw,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
weights = denoising_weights(
|
|
202
|
+
snrs_by_channel,
|
|
203
|
+
spike_length_samples=spike_length_samples,
|
|
204
|
+
trough_offset=trough_offset_samples,
|
|
205
|
+
snr_threshold=denoising_snr_threshold,
|
|
206
|
+
)
|
|
207
|
+
templates = weights * raw_templates + (1 - weights) * low_rank_templates
|
|
208
|
+
templates = templates.astype(dtype)
|
|
209
|
+
|
|
210
|
+
return dict(
|
|
211
|
+
sorting=sorting,
|
|
212
|
+
unit_ids=unit_ids,
|
|
213
|
+
spike_counts=spike_counts,
|
|
214
|
+
templates=templates,
|
|
215
|
+
raw_std_devs=raw_std_devs,
|
|
216
|
+
raw_templates=raw_templates,
|
|
217
|
+
low_rank_templates=low_rank_templates,
|
|
218
|
+
snrs_by_channel=snrs_by_channel,
|
|
219
|
+
spike_counts_by_channel=spike_counts_by_channel,
|
|
220
|
+
denoising_tsvd=denoising_tsvd,
|
|
221
|
+
weights=weights,
|
|
222
|
+
**geom_kw,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# -- helpers
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def fit_tsvd(
|
|
230
|
+
recording,
|
|
231
|
+
sorting,
|
|
232
|
+
denoising_rank=5,
|
|
233
|
+
denoising_fit_radius=75.0,
|
|
234
|
+
denoising_spikes_fit=25_000,
|
|
235
|
+
trough_offset_samples=42,
|
|
236
|
+
spike_length_samples=121,
|
|
237
|
+
recompute_tsvd=False,
|
|
238
|
+
dtype=np.float32,
|
|
239
|
+
random_seed=0,
|
|
240
|
+
n_iter=15,
|
|
241
|
+
) -> PCA | TruncatedSVD:
|
|
242
|
+
tsvd = None
|
|
243
|
+
if not recompute_tsvd:
|
|
244
|
+
tsvd = load_stored_tsvd(sorting)
|
|
245
|
+
assert isinstance(tsvd, (TruncatedSVD, PCA))
|
|
246
|
+
if tsvd is not None:
|
|
247
|
+
return tsvd
|
|
248
|
+
|
|
249
|
+
# read spikes on channel neighborhood
|
|
250
|
+
geom = recording.get_channel_locations()
|
|
251
|
+
tsvd_channel_index = make_channel_index(geom, denoising_fit_radius)
|
|
252
|
+
|
|
253
|
+
# subset spikes used to fit tsvd
|
|
254
|
+
rg = np.random.default_rng(random_seed)
|
|
255
|
+
max_time = recording.get_num_samples() - (
|
|
256
|
+
spike_length_samples - trough_offset_samples
|
|
257
|
+
)
|
|
258
|
+
t_clip = sorting.times_samples.clip(trough_offset_samples, max_time)
|
|
259
|
+
valid = np.logical_and(
|
|
260
|
+
sorting.labels >= 0,
|
|
261
|
+
sorting.times_samples == t_clip,
|
|
262
|
+
)
|
|
263
|
+
choices = np.flatnonzero(valid)
|
|
264
|
+
if choices.size > denoising_spikes_fit:
|
|
265
|
+
choices = rg.choice(choices, denoising_spikes_fit, replace=False)
|
|
266
|
+
choices.sort()
|
|
267
|
+
times = sorting.times_samples[choices]
|
|
268
|
+
channels = sorting.channels[choices]
|
|
269
|
+
|
|
270
|
+
# grab waveforms
|
|
271
|
+
waveforms = spikeio.read_waveforms_channel_index(
|
|
272
|
+
recording,
|
|
273
|
+
times,
|
|
274
|
+
tsvd_channel_index,
|
|
275
|
+
channels,
|
|
276
|
+
trough_offset_samples=trough_offset_samples,
|
|
277
|
+
spike_length_samples=spike_length_samples,
|
|
278
|
+
fill_value=0.0, # all-0 rows don't change SVD basis
|
|
279
|
+
)
|
|
280
|
+
waveforms = waveforms.astype(dtype)
|
|
281
|
+
waveforms = waveforms.transpose(0, 2, 1)
|
|
282
|
+
waveforms = waveforms.reshape(len(times) * tsvd_channel_index.shape[1], -1)
|
|
283
|
+
|
|
284
|
+
# reshape, fit tsvd, and done
|
|
285
|
+
tsvd = TruncatedSVD(
|
|
286
|
+
n_components=denoising_rank, random_state=random_seed, n_iter=n_iter
|
|
287
|
+
)
|
|
288
|
+
tsvd.fit(waveforms)
|
|
289
|
+
|
|
290
|
+
return tsvd
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def denoising_weights(
|
|
294
|
+
snrs,
|
|
295
|
+
spike_length_samples,
|
|
296
|
+
trough_offset,
|
|
297
|
+
snr_threshold,
|
|
298
|
+
a=12.0,
|
|
299
|
+
b=12.0,
|
|
300
|
+
d=6.0,
|
|
301
|
+
edge_behavior="saturate",
|
|
302
|
+
):
|
|
303
|
+
"""Weights are applied to raw template, 1-weights to low rank"""
|
|
304
|
+
# v shaped function for time weighting
|
|
305
|
+
vt = np.abs(np.arange(spike_length_samples) - trough_offset, dtype=float)
|
|
306
|
+
if trough_offset < spike_length_samples:
|
|
307
|
+
vt[trough_offset:] /= vt[trough_offset:].max()
|
|
308
|
+
if trough_offset > 0:
|
|
309
|
+
vt[:trough_offset] /= vt[:trough_offset].max()
|
|
310
|
+
|
|
311
|
+
# snr weighting per channel
|
|
312
|
+
if edge_behavior == "saturate":
|
|
313
|
+
snc = np.minimum(snrs, snr_threshold) / snr_threshold
|
|
314
|
+
elif edge_behavior == "inf":
|
|
315
|
+
snc = np.minimum(snrs, snr_threshold) / snr_threshold
|
|
316
|
+
snc[snc >= 1.0] = np.inf
|
|
317
|
+
elif edge_behavior == "raw":
|
|
318
|
+
snc = snrs
|
|
319
|
+
else:
|
|
320
|
+
assert False
|
|
321
|
+
|
|
322
|
+
# pass it through a hand picked squashing function
|
|
323
|
+
wntc = 1.0 / (1.0 + np.exp(d + a * vt[None, :, None] - b * snc[:, None, :]))
|
|
324
|
+
|
|
325
|
+
return wntc
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
# -- main routine which does all the spike loading and computation
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def get_all_shifted_raw_and_low_rank_templates(
|
|
332
|
+
recording,
|
|
333
|
+
sorting,
|
|
334
|
+
registered_geom=None,
|
|
335
|
+
denoising_tsvd=None,
|
|
336
|
+
pitch_shifts=None,
|
|
337
|
+
with_raw_std_dev=False,
|
|
338
|
+
spikes_per_unit=500,
|
|
339
|
+
reducer=fast_nanmedian,
|
|
340
|
+
n_jobs=0,
|
|
341
|
+
units_per_job=8,
|
|
342
|
+
random_seed=0,
|
|
343
|
+
show_progress=True,
|
|
344
|
+
trough_offset_samples=42,
|
|
345
|
+
spike_length_samples=121,
|
|
346
|
+
min_fraction_at_shift=0.1,
|
|
347
|
+
min_count_at_shift=5,
|
|
348
|
+
dtype=np.float32,
|
|
349
|
+
device=None,
|
|
350
|
+
):
|
|
351
|
+
n_jobs, Executor, context, rank_queue = get_pool(n_jobs, with_rank_queue=True) # type: ignore
|
|
352
|
+
unit_ids, spike_counts = np.unique(sorting.labels, return_counts=True)
|
|
353
|
+
spike_counts = spike_counts[unit_ids >= 0]
|
|
354
|
+
unit_ids = unit_ids[unit_ids >= 0]
|
|
355
|
+
raw = denoising_tsvd is None
|
|
356
|
+
prefix = "Raw" if raw else "Denoised"
|
|
357
|
+
|
|
358
|
+
n_template_channels = recording.get_num_channels()
|
|
359
|
+
registered_kdtree = None
|
|
360
|
+
if registered_geom is not None:
|
|
361
|
+
n_template_channels = len(registered_geom)
|
|
362
|
+
registered_kdtree = KDTree(registered_geom)
|
|
363
|
+
|
|
364
|
+
n_units = unit_ids.size
|
|
365
|
+
raw_templates = np.zeros(
|
|
366
|
+
(n_units, spike_length_samples, n_template_channels),
|
|
367
|
+
dtype=dtype,
|
|
368
|
+
)
|
|
369
|
+
if with_raw_std_dev:
|
|
370
|
+
raw_square_templates = np.zeros(
|
|
371
|
+
(n_units, spike_length_samples, n_template_channels),
|
|
372
|
+
dtype=dtype,
|
|
373
|
+
)
|
|
374
|
+
low_rank_templates = None
|
|
375
|
+
if not raw:
|
|
376
|
+
low_rank_templates = np.zeros(
|
|
377
|
+
(n_units, spike_length_samples, n_template_channels),
|
|
378
|
+
dtype=dtype,
|
|
379
|
+
)
|
|
380
|
+
snrs_by_channel = np.zeros((n_units, n_template_channels), dtype=dtype)
|
|
381
|
+
spike_counts_by_channel = np.zeros((n_units, n_template_channels), dtype=dtype)
|
|
382
|
+
|
|
383
|
+
unit_id_chunks = [
|
|
384
|
+
unit_ids[i : i + units_per_job] for i in range(0, n_units, units_per_job)
|
|
385
|
+
]
|
|
386
|
+
|
|
387
|
+
with Executor(
|
|
388
|
+
max_workers=n_jobs,
|
|
389
|
+
mp_context=context,
|
|
390
|
+
initializer=_template_process_init,
|
|
391
|
+
initargs=(
|
|
392
|
+
rank_queue,
|
|
393
|
+
random_seed,
|
|
394
|
+
recording,
|
|
395
|
+
sorting,
|
|
396
|
+
registered_kdtree,
|
|
397
|
+
denoising_tsvd,
|
|
398
|
+
pitch_shifts,
|
|
399
|
+
spikes_per_unit,
|
|
400
|
+
min_fraction_at_shift,
|
|
401
|
+
min_count_at_shift,
|
|
402
|
+
with_raw_std_dev,
|
|
403
|
+
reducer,
|
|
404
|
+
trough_offset_samples,
|
|
405
|
+
spike_length_samples,
|
|
406
|
+
device,
|
|
407
|
+
units_per_job,
|
|
408
|
+
dtype,
|
|
409
|
+
),
|
|
410
|
+
) as pool:
|
|
411
|
+
# launch the jobs and wrap in a progress bar
|
|
412
|
+
results = pool.map(_template_job, unit_id_chunks)
|
|
413
|
+
if show_progress:
|
|
414
|
+
pbar = tqdm(
|
|
415
|
+
smoothing=0.01,
|
|
416
|
+
desc=f"{prefix} templates",
|
|
417
|
+
total=unit_ids.size,
|
|
418
|
+
unit="template",
|
|
419
|
+
)
|
|
420
|
+
for res in results:
|
|
421
|
+
if res is None:
|
|
422
|
+
continue
|
|
423
|
+
(
|
|
424
|
+
units_chunk,
|
|
425
|
+
raw_temps_chunk,
|
|
426
|
+
raw_square_temps_chunk,
|
|
427
|
+
low_rank_temps_chunk,
|
|
428
|
+
snrs_chunk,
|
|
429
|
+
chancounts_chunk,
|
|
430
|
+
) = res
|
|
431
|
+
ix_chunk = np.isin(unit_ids, units_chunk)
|
|
432
|
+
raw_templates[ix_chunk] = raw_temps_chunk
|
|
433
|
+
if with_raw_std_dev:
|
|
434
|
+
raw_square_templates[ix_chunk] = raw_square_temps_chunk # type: ignore
|
|
435
|
+
if not raw:
|
|
436
|
+
low_rank_templates[ix_chunk] = low_rank_temps_chunk # type: ignore
|
|
437
|
+
snrs_by_channel[ix_chunk] = snrs_chunk
|
|
438
|
+
spike_counts_by_channel[ix_chunk] = chancounts_chunk
|
|
439
|
+
if show_progress:
|
|
440
|
+
pbar.update(len(units_chunk)) # type: ignore
|
|
441
|
+
if show_progress:
|
|
442
|
+
pbar.close() # type: ignore
|
|
443
|
+
|
|
444
|
+
raw_std_dev = None
|
|
445
|
+
if with_raw_std_dev:
|
|
446
|
+
raw_std_dev = raw_square_templates # type: ignore
|
|
447
|
+
raw_std_dev -= raw_templates**2
|
|
448
|
+
np.abs(raw_std_dev, out=raw_std_dev)
|
|
449
|
+
raw_std_dev **= 0.5
|
|
450
|
+
|
|
451
|
+
return (
|
|
452
|
+
unit_ids,
|
|
453
|
+
spike_counts,
|
|
454
|
+
raw_templates,
|
|
455
|
+
raw_std_dev,
|
|
456
|
+
low_rank_templates,
|
|
457
|
+
snrs_by_channel,
|
|
458
|
+
spike_counts_by_channel,
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
class TemplateProcessContext:
|
|
463
|
+
def __init__(
|
|
464
|
+
self,
|
|
465
|
+
rg,
|
|
466
|
+
recording,
|
|
467
|
+
sorting,
|
|
468
|
+
registered_kdtree,
|
|
469
|
+
denoising_tsvd,
|
|
470
|
+
pitch_shifts,
|
|
471
|
+
spikes_per_unit,
|
|
472
|
+
min_fraction_at_shift,
|
|
473
|
+
min_count_at_shift,
|
|
474
|
+
with_raw_std_dev,
|
|
475
|
+
reducer,
|
|
476
|
+
trough_offset_samples,
|
|
477
|
+
spike_length_samples,
|
|
478
|
+
device,
|
|
479
|
+
units_per_job,
|
|
480
|
+
dtype,
|
|
481
|
+
):
|
|
482
|
+
self.n_channels = recording.get_num_channels()
|
|
483
|
+
self.registered = registered_kdtree is not None
|
|
484
|
+
|
|
485
|
+
self.rg = rg
|
|
486
|
+
self.device = device
|
|
487
|
+
self.dtype = dtype
|
|
488
|
+
self.recording = recording
|
|
489
|
+
self.sorting = sorting
|
|
490
|
+
self.denoising_tsvd = denoising_tsvd
|
|
491
|
+
if denoising_tsvd is not None:
|
|
492
|
+
self.denoising_tsvd = TorchSVDProjector(
|
|
493
|
+
torch.from_numpy(denoising_tsvd.components_.astype(dtype))
|
|
494
|
+
)
|
|
495
|
+
self.denoising_tsvd.to(self.device)
|
|
496
|
+
self.spikes_per_unit = spikes_per_unit
|
|
497
|
+
self.reducer = reducer
|
|
498
|
+
self.trough_offset_samples = trough_offset_samples
|
|
499
|
+
self.spike_length_samples = spike_length_samples
|
|
500
|
+
self.max_spike_time = recording.get_num_samples() - (
|
|
501
|
+
spike_length_samples - trough_offset_samples
|
|
502
|
+
)
|
|
503
|
+
self.min_fraction_at_shift = min_fraction_at_shift
|
|
504
|
+
self.min_count_at_shift = min_count_at_shift
|
|
505
|
+
self.with_raw_std_dev = with_raw_std_dev
|
|
506
|
+
|
|
507
|
+
self.spike_buffer = torch.zeros(
|
|
508
|
+
(
|
|
509
|
+
spikes_per_unit * units_per_job,
|
|
510
|
+
spike_length_samples,
|
|
511
|
+
self.n_channels,
|
|
512
|
+
),
|
|
513
|
+
device=device,
|
|
514
|
+
dtype=torch.from_numpy(np.zeros(1, dtype=dtype)).dtype,
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
self.n_template_channels = self.n_channels
|
|
518
|
+
if self.registered:
|
|
519
|
+
self.geom = recording.get_channel_locations()
|
|
520
|
+
self.match_distance = pdist(self.geom).min() / 2
|
|
521
|
+
self.registered_geom = registered_kdtree.data
|
|
522
|
+
self.registered_kdtree = registered_kdtree
|
|
523
|
+
self.pitch_shifts = pitch_shifts
|
|
524
|
+
self.n_template_channels = len(self.registered_geom)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
_template_process_context = threading.local()
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _template_process_init(
|
|
531
|
+
rank_queue,
|
|
532
|
+
random_seed,
|
|
533
|
+
recording,
|
|
534
|
+
sorting,
|
|
535
|
+
registered_kdtree,
|
|
536
|
+
denoising_tsvd,
|
|
537
|
+
pitch_shifts,
|
|
538
|
+
spikes_per_unit,
|
|
539
|
+
min_fraction_at_shift,
|
|
540
|
+
min_count_at_shift,
|
|
541
|
+
with_raw_std_dev,
|
|
542
|
+
reducer,
|
|
543
|
+
trough_offset_samples,
|
|
544
|
+
spike_length_samples,
|
|
545
|
+
device,
|
|
546
|
+
units_per_job,
|
|
547
|
+
dtype,
|
|
548
|
+
):
|
|
549
|
+
global _template_process_context
|
|
550
|
+
|
|
551
|
+
rank = rank_queue.get()
|
|
552
|
+
if device is None:
|
|
553
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
554
|
+
device = torch.device(device)
|
|
555
|
+
if device.type == "cuda" and device.index is None:
|
|
556
|
+
if torch.cuda.device_count() > 1:
|
|
557
|
+
device = torch.device("cuda", index=rank % torch.cuda.device_count())
|
|
558
|
+
torch.set_grad_enabled(False)
|
|
559
|
+
|
|
560
|
+
rg = np.random.default_rng(random_seed + rank)
|
|
561
|
+
_template_process_context.ctx = TemplateProcessContext(
|
|
562
|
+
rg,
|
|
563
|
+
recording,
|
|
564
|
+
sorting,
|
|
565
|
+
registered_kdtree,
|
|
566
|
+
denoising_tsvd,
|
|
567
|
+
pitch_shifts,
|
|
568
|
+
spikes_per_unit,
|
|
569
|
+
min_fraction_at_shift,
|
|
570
|
+
min_count_at_shift,
|
|
571
|
+
with_raw_std_dev,
|
|
572
|
+
reducer,
|
|
573
|
+
trough_offset_samples,
|
|
574
|
+
spike_length_samples,
|
|
575
|
+
device,
|
|
576
|
+
units_per_job,
|
|
577
|
+
dtype,
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _template_job(unit_ids):
|
|
582
|
+
p = _template_process_context.ctx
|
|
583
|
+
|
|
584
|
+
in_units_full = np.flatnonzero(np.isin(p.sorting.labels, unit_ids))
|
|
585
|
+
if not in_units_full.size:
|
|
586
|
+
return
|
|
587
|
+
labels_full = p.sorting.labels[in_units_full]
|
|
588
|
+
|
|
589
|
+
# only so many spikes per unit
|
|
590
|
+
uids, counts = np.unique(labels_full, return_counts=True)
|
|
591
|
+
n_spikes_grab = np.minimum(counts, p.spikes_per_unit).sum()
|
|
592
|
+
in_units = np.empty(n_spikes_grab, dtype=in_units_full.dtype)
|
|
593
|
+
labels = np.empty(n_spikes_grab, dtype=labels_full.dtype)
|
|
594
|
+
offset = 0
|
|
595
|
+
for u, c in zip(uids, counts):
|
|
596
|
+
if c > p.spikes_per_unit:
|
|
597
|
+
in_unit = p.rg.choice(
|
|
598
|
+
in_units_full[labels_full == u],
|
|
599
|
+
p.spikes_per_unit,
|
|
600
|
+
replace=False,
|
|
601
|
+
)
|
|
602
|
+
in_units[offset : offset + min(c, p.spikes_per_unit)] = in_unit
|
|
603
|
+
labels[offset : offset + min(c, p.spikes_per_unit)] = u
|
|
604
|
+
else:
|
|
605
|
+
in_units[offset : offset + c] = in_units_full[labels_full == u]
|
|
606
|
+
labels[offset : offset + c] = u
|
|
607
|
+
offset += min(c, p.spikes_per_unit)
|
|
608
|
+
order = np.argsort(in_units)
|
|
609
|
+
in_units = in_units[order]
|
|
610
|
+
labels = labels[order]
|
|
611
|
+
|
|
612
|
+
# read waveforms for all units
|
|
613
|
+
times = p.sorting.times_samples[in_units]
|
|
614
|
+
valid = np.flatnonzero(
|
|
615
|
+
(times >= p.trough_offset_samples) & (times <= p.max_spike_time)
|
|
616
|
+
)
|
|
617
|
+
if not valid.size:
|
|
618
|
+
return
|
|
619
|
+
in_units = in_units[valid]
|
|
620
|
+
labels = labels[valid]
|
|
621
|
+
times = times[valid]
|
|
622
|
+
waveforms = spikeio.read_full_waveforms(
|
|
623
|
+
p.recording,
|
|
624
|
+
times,
|
|
625
|
+
trough_offset_samples=p.trough_offset_samples,
|
|
626
|
+
spike_length_samples=p.spike_length_samples,
|
|
627
|
+
)
|
|
628
|
+
p.spike_buffer[: times.size] = torch.from_numpy(waveforms.astype(p.dtype))
|
|
629
|
+
waveforms = p.spike_buffer[: times.size]
|
|
630
|
+
n, t, c = waveforms.shape
|
|
631
|
+
|
|
632
|
+
# compute raw templates and spike counts per channel
|
|
633
|
+
raw_templates = []
|
|
634
|
+
raw_square_templates = []
|
|
635
|
+
counts = []
|
|
636
|
+
units_chunk = []
|
|
637
|
+
for u in uids:
|
|
638
|
+
in_unit = np.flatnonzero(labels == u)
|
|
639
|
+
if not in_unit.size:
|
|
640
|
+
continue
|
|
641
|
+
units_chunk.append(u)
|
|
642
|
+
in_unit_orig = in_units[labels == u]
|
|
643
|
+
unit_waveforms = waveforms[in_unit]
|
|
644
|
+
if p.registered:
|
|
645
|
+
raw_templates.append(
|
|
646
|
+
registered_template(
|
|
647
|
+
unit_waveforms,
|
|
648
|
+
p.pitch_shifts[in_unit_orig],
|
|
649
|
+
p.geom,
|
|
650
|
+
p.registered_geom,
|
|
651
|
+
min_fraction_at_shift=p.min_fraction_at_shift,
|
|
652
|
+
min_count_at_shift=p.min_count_at_shift,
|
|
653
|
+
registered_kdtree=p.registered_kdtree,
|
|
654
|
+
match_distance=p.match_distance,
|
|
655
|
+
reducer=p.reducer,
|
|
656
|
+
)
|
|
657
|
+
)
|
|
658
|
+
if p.with_raw_std_dev:
|
|
659
|
+
raw_square_templates.append(
|
|
660
|
+
registered_template(
|
|
661
|
+
unit_waveforms.square_(),
|
|
662
|
+
p.pitch_shifts[in_unit_orig],
|
|
663
|
+
p.geom,
|
|
664
|
+
p.registered_geom,
|
|
665
|
+
min_fraction_at_shift=p.min_fraction_at_shift,
|
|
666
|
+
min_count_at_shift=p.min_count_at_shift,
|
|
667
|
+
registered_kdtree=p.registered_kdtree,
|
|
668
|
+
match_distance=p.match_distance,
|
|
669
|
+
reducer=p.reducer,
|
|
670
|
+
)
|
|
671
|
+
)
|
|
672
|
+
counts.append(
|
|
673
|
+
registered_template(
|
|
674
|
+
np.ones((in_unit.size, p.n_channels)),
|
|
675
|
+
p.pitch_shifts[in_unit_orig],
|
|
676
|
+
p.geom,
|
|
677
|
+
p.registered_geom,
|
|
678
|
+
min_fraction_at_shift=p.min_fraction_at_shift,
|
|
679
|
+
min_count_at_shift=p.min_count_at_shift,
|
|
680
|
+
registered_kdtree=p.registered_kdtree,
|
|
681
|
+
match_distance=p.match_distance,
|
|
682
|
+
reducer=np.nansum,
|
|
683
|
+
)
|
|
684
|
+
)
|
|
685
|
+
else:
|
|
686
|
+
raw_templates.append(
|
|
687
|
+
p.reducer(waveforms[in_unit], axis=0).numpy(force=True)
|
|
688
|
+
)
|
|
689
|
+
if p.with_raw_std_dev:
|
|
690
|
+
raw_square_templates.append(
|
|
691
|
+
p.reducer(unit_waveforms.square_(), axis=0).numpy(force=True)
|
|
692
|
+
)
|
|
693
|
+
counts.append(in_unit.size)
|
|
694
|
+
snrs_by_chan = np.array(
|
|
695
|
+
[ptp(rt, 0) * np.sqrt(c) for rt, c in zip(raw_templates, counts)]
|
|
696
|
+
)
|
|
697
|
+
counts_by_chan = np.array(counts)
|
|
698
|
+
if counts_by_chan.ndim == 1:
|
|
699
|
+
counts_by_chan = np.broadcast_to(counts_by_chan[:, None], snrs_by_chan.shape)
|
|
700
|
+
raw_templates = np.array(raw_templates)
|
|
701
|
+
if p.with_raw_std_dev:
|
|
702
|
+
raw_square_templates = np.array(raw_square_templates)
|
|
703
|
+
else:
|
|
704
|
+
raw_square_templates = None
|
|
705
|
+
|
|
706
|
+
if p.denoising_tsvd is None:
|
|
707
|
+
return (
|
|
708
|
+
units_chunk,
|
|
709
|
+
raw_templates,
|
|
710
|
+
raw_square_templates,
|
|
711
|
+
None,
|
|
712
|
+
snrs_by_chan,
|
|
713
|
+
counts_by_chan,
|
|
714
|
+
)
|
|
715
|
+
|
|
716
|
+
# nt, t, ct = raw_templates.shape
|
|
717
|
+
# low_rank_templates = torch.tensor(raw_templates.transpose(0, 2, 1), device=p.device)
|
|
718
|
+
# low_rank_templates = low_rank_templates.reshape(nt * ct, t)
|
|
719
|
+
# low_rank_templates = p.denoising_tsvd(low_rank_templates, in_place=True)
|
|
720
|
+
# low_rank_templates = low_rank_templates.view(nt, ct, t).permute(0, 2, 1)
|
|
721
|
+
# low_rank_templates = low_rank_templates.numpy(force=True)
|
|
722
|
+
|
|
723
|
+
# apply denoising
|
|
724
|
+
waveforms = waveforms.permute(0, 2, 1).reshape(n * c, t)
|
|
725
|
+
waveforms = p.denoising_tsvd(waveforms, in_place=True)
|
|
726
|
+
waveforms = waveforms.reshape(n, c, t).permute(0, 2, 1)
|
|
727
|
+
|
|
728
|
+
# get low rank templates
|
|
729
|
+
low_rank_templates = []
|
|
730
|
+
for u in units_chunk:
|
|
731
|
+
in_unit = np.flatnonzero(labels == u)
|
|
732
|
+
in_unit_orig = in_units[labels == u]
|
|
733
|
+
if p.registered:
|
|
734
|
+
low_rank_templates.append(
|
|
735
|
+
registered_template(
|
|
736
|
+
waveforms[in_unit],
|
|
737
|
+
p.pitch_shifts[in_unit_orig],
|
|
738
|
+
p.geom,
|
|
739
|
+
p.registered_geom,
|
|
740
|
+
min_fraction_at_shift=p.min_fraction_at_shift,
|
|
741
|
+
min_count_at_shift=p.min_count_at_shift,
|
|
742
|
+
registered_kdtree=p.registered_kdtree,
|
|
743
|
+
match_distance=p.match_distance,
|
|
744
|
+
reducer=p.reducer,
|
|
745
|
+
)
|
|
746
|
+
)
|
|
747
|
+
else:
|
|
748
|
+
low_rank_templates.append(
|
|
749
|
+
p.reducer(waveforms[in_unit], axis=0).numpy(force=True)
|
|
750
|
+
)
|
|
751
|
+
low_rank_templates = np.array(low_rank_templates)
|
|
752
|
+
|
|
753
|
+
return (
|
|
754
|
+
units_chunk,
|
|
755
|
+
raw_templates,
|
|
756
|
+
raw_square_templates,
|
|
757
|
+
low_rank_templates,
|
|
758
|
+
snrs_by_chan,
|
|
759
|
+
counts_by_chan,
|
|
760
|
+
)
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
class TorchSVDProjector(torch.nn.Module):
|
|
764
|
+
def __init__(self, components):
|
|
765
|
+
super().__init__()
|
|
766
|
+
self.register_buffer("components", components)
|
|
767
|
+
|
|
768
|
+
def forward(self, x, in_place=False):
|
|
769
|
+
embed = x @ self.components.T
|
|
770
|
+
out = x if in_place else None
|
|
771
|
+
return torch.matmul(embed, cast(torch.Tensor, self.components), out=out)
|