DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1054 @@
|
|
|
1
|
+
"""Utility functions for dealing with drifting channels
|
|
2
|
+
|
|
3
|
+
A brief note on the sign of the drift.
|
|
4
|
+
|
|
5
|
+
- Words like "displacement" or "drift" always refer to the
|
|
6
|
+
displacement of the tissue relative to the probe.
|
|
7
|
+
|
|
8
|
+
Thus if a neuron's position at time 0 is z0, it's position
|
|
9
|
+
at time t is z0 + drift(t, z0).
|
|
10
|
+
|
|
11
|
+
The observed position z(t) is the reg pos zr(t) + drift(t, z(t)).
|
|
12
|
+
|
|
13
|
+
Alternatively, estimate reg poses by zr(t) = z(t) - drift(t, z(t)).
|
|
14
|
+
|
|
15
|
+
- Alternatively, one could imagine a reference frame where the brain
|
|
16
|
+
is fixed and the probe moves. In that case, the positions of the probe
|
|
17
|
+
contacts at time t are "registered geom" + [0, drift(t, geom[:, 1])].
|
|
18
|
+
|
|
19
|
+
The main concept here is the "registered geometry" made by the
|
|
20
|
+
function `registered_geometry`. The idea is to extend the
|
|
21
|
+
probe geometry to cover the range of drift experienced in the
|
|
22
|
+
recording. The probe's pitch (unit at which its geometry repeats
|
|
23
|
+
vertically) is the integer unit at which we shift channels when
|
|
24
|
+
extending the geometry, so that the registered probe contains the
|
|
25
|
+
original probe as a subset, as well as copies of the probe shifted
|
|
26
|
+
by integer numbers of pitches. As many shifted copies are created
|
|
27
|
+
as needed to capture all the drift.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
import torch
|
|
34
|
+
from scipy.spatial import KDTree
|
|
35
|
+
from scipy.spatial.distance import pdist
|
|
36
|
+
|
|
37
|
+
from .spiketorch import fast_nanmedian
|
|
38
|
+
from .waveform_util import get_orders, get_pitch, make_channel_index
|
|
39
|
+
|
|
40
|
+
# -- registered geometry and templates helpers
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def registered_geometry(
|
|
44
|
+
geom,
|
|
45
|
+
motion_est=None,
|
|
46
|
+
upward_drift=None,
|
|
47
|
+
downward_drift=None,
|
|
48
|
+
):
|
|
49
|
+
"""Pad probe to motion extent
|
|
50
|
+
|
|
51
|
+
This adds extra channels to a probe layout, matching the probe's geometry.
|
|
52
|
+
For instance, say we had a tiny NP1-like probe that looks like this:
|
|
53
|
+
|
|
54
|
+
o o -
|
|
55
|
+
o o | 20um row spacing
|
|
56
|
+
o o | 60um total height
|
|
57
|
+
o o -
|
|
58
|
+
|
|
59
|
+
Now, say that the probe moves from depth 0 at time 0 up to depth 20um
|
|
60
|
+
relative to the brain tissue at time 10:
|
|
61
|
+
|
|
62
|
+
time 0 time 10
|
|
63
|
+
|
|
64
|
+
o o
|
|
65
|
+
depth 40 o o
|
|
66
|
+
o o o o
|
|
67
|
+
depth 20 o o o o
|
|
68
|
+
o o
|
|
69
|
+
depth 0 o o
|
|
70
|
+
|
|
71
|
+
In this case, the motion estimate at time 0 would be 0, and the motion estimate
|
|
72
|
+
at time 10 would be 20.
|
|
73
|
+
|
|
74
|
+
This function would add new fake "target channels" (marked with xs):
|
|
75
|
+
|
|
76
|
+
o o
|
|
77
|
+
depth 20 o o
|
|
78
|
+
o o
|
|
79
|
+
depth 0 o o
|
|
80
|
+
x x
|
|
81
|
+
depth -20 x x
|
|
82
|
+
|
|
83
|
+
Later, in interpolation, the motion will be added to these target channels. The
|
|
84
|
+
"o" channels will then always line up with their true positions, and the x channels
|
|
85
|
+
can be handled by extrapolation or nan padding, etc.
|
|
86
|
+
|
|
87
|
+
This function works by guessing the vertical gap between channels (called "pitch" here).
|
|
88
|
+
In the NP1-like example above, that's 40um, not 20um, due to the staggered layout. Then,
|
|
89
|
+
the extent of the motion is determined and rounded away from 0 to the nearest pitch.
|
|
90
|
+
Copies of the probe shifted by each pitch in the range of this extent are then created,
|
|
91
|
+
and the unique set of site positions is grabbed to finish.
|
|
92
|
+
|
|
93
|
+
Another way of looking at it:
|
|
94
|
+
The probe will be extended upwards according to the amount of upward
|
|
95
|
+
drift and downward according to the amount of downward drift, where
|
|
96
|
+
here we mean drift of the probe relative to the spikes.
|
|
97
|
+
|
|
98
|
+
But, the convention for motion estimation is the opposite: motion of
|
|
99
|
+
the spikes relative to the probe. In other words, the displacement
|
|
100
|
+
in motion_est is such that
|
|
101
|
+
registered z = original z - displacement
|
|
102
|
+
So, if the probe moved up from t1->t2 (so that z(t2) < z(t1), since z
|
|
103
|
+
is the spike's absolute position on the probe), this would register
|
|
104
|
+
as a negative displacement. So, upward motion of the probe means a
|
|
105
|
+
negative displacement value estimated from spikes!
|
|
106
|
+
"""
|
|
107
|
+
assert geom.ndim == 2
|
|
108
|
+
pitch = get_pitch(geom)
|
|
109
|
+
|
|
110
|
+
# figure out how much upward and downward motion there is
|
|
111
|
+
if motion_est is not None:
|
|
112
|
+
# these look flipped! why? see __doc__
|
|
113
|
+
downward_drift = max(0, motion_est.displacement.max())
|
|
114
|
+
upward_drift = max(0, -motion_est.displacement.min())
|
|
115
|
+
else:
|
|
116
|
+
assert upward_drift is not None
|
|
117
|
+
assert downward_drift is not None
|
|
118
|
+
assert upward_drift >= 0
|
|
119
|
+
assert downward_drift >= 0
|
|
120
|
+
|
|
121
|
+
# pad with an integral number of pitches for simplicity
|
|
122
|
+
pitches_pad_up = int(np.round(upward_drift / pitch))
|
|
123
|
+
pitches_pad_down = int(np.round(downward_drift / pitch))
|
|
124
|
+
|
|
125
|
+
# we have to be careful about floating point error here
|
|
126
|
+
# two sites may be different due to floating point error
|
|
127
|
+
# we know they are the same if their distance is smaller than:
|
|
128
|
+
min_distance = np.sqrt(pdist(geom, metric="sqeuclidean").min()) / 2
|
|
129
|
+
|
|
130
|
+
# find all registered site positions
|
|
131
|
+
# the following naive algorithm is both quadratic now banished to a test
|
|
132
|
+
# for each shift:
|
|
133
|
+
# - shift all site positions
|
|
134
|
+
# - if any site is not eps-close to set of unique shifted pos,
|
|
135
|
+
# add it
|
|
136
|
+
# to un-quadratic it, make a kdtree of the geom, and shift the
|
|
137
|
+
# unique positions inversely
|
|
138
|
+
|
|
139
|
+
# better loop
|
|
140
|
+
chan_ix = np.arange(len(geom))
|
|
141
|
+
kdt = KDTree(geom)
|
|
142
|
+
n_shifts = pitches_pad_up + pitches_pad_down + 1
|
|
143
|
+
unique_shifted_positions_toobig = np.zeros(
|
|
144
|
+
(n_shifts * len(geom), geom.shape[1]), dtype=geom.dtype
|
|
145
|
+
)
|
|
146
|
+
unique_shifted_positions_toobig[: len(geom)] = geom
|
|
147
|
+
cur_ix = len(geom)
|
|
148
|
+
for shift in range(-pitches_pad_down, pitches_pad_up + 1):
|
|
149
|
+
cur_pos = unique_shifted_positions_toobig[:cur_ix]
|
|
150
|
+
|
|
151
|
+
# sign is flipped, since we're shifting the bad guys
|
|
152
|
+
cur_pos_shifted = cur_pos - [0, pitch * shift]
|
|
153
|
+
d, i = kdt.query(cur_pos_shifted, distance_upper_bound=min_distance)
|
|
154
|
+
|
|
155
|
+
# figure out which were unmatched
|
|
156
|
+
i_unobs = np.setdiff1d(chan_ix, i)
|
|
157
|
+
|
|
158
|
+
# add them to the set
|
|
159
|
+
n_new = len(i_unobs)
|
|
160
|
+
if not n_new:
|
|
161
|
+
continue
|
|
162
|
+
sh = [0, pitch * shift]
|
|
163
|
+
unique_shifted_positions_toobig[cur_ix : cur_ix + n_new] = geom[i_unobs] + sh
|
|
164
|
+
cur_ix += n_new
|
|
165
|
+
assert cur_ix <= len(unique_shifted_positions_toobig)
|
|
166
|
+
|
|
167
|
+
unique_shifted_positions = unique_shifted_positions_toobig[:cur_ix]
|
|
168
|
+
|
|
169
|
+
# if axes are decreasing, maintain that -- it leads to less mashing.
|
|
170
|
+
sorter_positions = unique_shifted_positions
|
|
171
|
+
flips = get_orders(geom)
|
|
172
|
+
if (flips < 0).any():
|
|
173
|
+
sorter_positions = sorter_positions * flips
|
|
174
|
+
|
|
175
|
+
# order by depth first, then horizontal position (unique goes the other way)
|
|
176
|
+
registered_geom = unique_shifted_positions[np.lexsort(sorter_positions.T)]
|
|
177
|
+
assert np.isclose(get_pitch(registered_geom), pitch)
|
|
178
|
+
|
|
179
|
+
return registered_geom
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def registered_channels(
|
|
183
|
+
channels, geom, n_pitches_shift, registered_geom, distance_upper_bound=None
|
|
184
|
+
):
|
|
185
|
+
"""What registered channels do `channels` land on after shifting by `n_pitches_shift`?"""
|
|
186
|
+
pitch = get_pitch(geom)
|
|
187
|
+
shifted_positions = geom.copy()[channels]
|
|
188
|
+
shifted_positions[:, 1] += n_pitches_shift * pitch
|
|
189
|
+
|
|
190
|
+
registered_kdtree = KDTree(registered_geom)
|
|
191
|
+
if distance_upper_bound is None:
|
|
192
|
+
distance_upper_bound = pdist(registered_geom).min() / 2
|
|
193
|
+
distances, registered_channels = registered_kdtree.query(
|
|
194
|
+
shifted_positions, distance_upper_bound=distance_upper_bound
|
|
195
|
+
)
|
|
196
|
+
# make sure there were no unmatched points
|
|
197
|
+
assert np.all(registered_channels < len(registered_geom))
|
|
198
|
+
|
|
199
|
+
return registered_channels
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def registered_average(
|
|
203
|
+
waveforms,
|
|
204
|
+
n_pitches_shift,
|
|
205
|
+
geom,
|
|
206
|
+
registered_geom,
|
|
207
|
+
registered_kdtree=None,
|
|
208
|
+
match_distance=None,
|
|
209
|
+
main_channels=None,
|
|
210
|
+
channel_index=None,
|
|
211
|
+
reducer=fast_nanmedian,
|
|
212
|
+
work_buffer=None,
|
|
213
|
+
pad_value=0.0,
|
|
214
|
+
return_n_samples=False,
|
|
215
|
+
):
|
|
216
|
+
static_waveforms = get_waveforms_on_static_channels(
|
|
217
|
+
waveforms,
|
|
218
|
+
geom,
|
|
219
|
+
main_channels=main_channels,
|
|
220
|
+
channel_index=channel_index,
|
|
221
|
+
n_pitches_shift=n_pitches_shift,
|
|
222
|
+
registered_geom=registered_geom,
|
|
223
|
+
target_kdtree=registered_kdtree,
|
|
224
|
+
match_distance=match_distance,
|
|
225
|
+
out=work_buffer,
|
|
226
|
+
fill_value=np.nan,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# take the mean
|
|
230
|
+
average = reducer(static_waveforms, axis=0)
|
|
231
|
+
if not np.isnan(pad_value):
|
|
232
|
+
average = np.nan_to_num(average, copy=False, nan=pad_value)
|
|
233
|
+
|
|
234
|
+
if return_n_samples:
|
|
235
|
+
# remove time dim if any
|
|
236
|
+
c = static_waveforms
|
|
237
|
+
if static_waveforms.ndim == 3:
|
|
238
|
+
c = static_waveforms[:, 0, :]
|
|
239
|
+
n_samples = np.isfinite(c).sum(axis=0)
|
|
240
|
+
return average, n_samples
|
|
241
|
+
|
|
242
|
+
return average
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def registered_template(
|
|
246
|
+
waveforms,
|
|
247
|
+
n_pitches_shift,
|
|
248
|
+
geom,
|
|
249
|
+
registered_geom,
|
|
250
|
+
registered_kdtree=None,
|
|
251
|
+
match_distance=None,
|
|
252
|
+
min_fraction_at_shift=0.5,
|
|
253
|
+
min_count_at_shift=25,
|
|
254
|
+
pad_value=0.0,
|
|
255
|
+
reducer=fast_nanmedian,
|
|
256
|
+
):
|
|
257
|
+
"""registered_average of waveforms would be more accurate if reducer=median, but slow."""
|
|
258
|
+
two_d = waveforms.ndim == 2
|
|
259
|
+
if two_d:
|
|
260
|
+
waveforms = waveforms[:, None, :]
|
|
261
|
+
uniq_shifts, counts = np.unique(n_pitches_shift, return_counts=True)
|
|
262
|
+
is_tensor = torch.is_tensor(waveforms)
|
|
263
|
+
if is_tensor:
|
|
264
|
+
drifty_templates = torch.zeros(
|
|
265
|
+
(len(uniq_shifts), *waveforms.shape[1:]),
|
|
266
|
+
dtype=waveforms.dtype,
|
|
267
|
+
device=waveforms.device,
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
drifty_templates = np.zeros(
|
|
271
|
+
(len(uniq_shifts), *waveforms.shape[1:]),
|
|
272
|
+
dtype=waveforms.dtype,
|
|
273
|
+
)
|
|
274
|
+
if is_tensor:
|
|
275
|
+
n_pitches_shift = torch.as_tensor(n_pitches_shift, device=waveforms.device)
|
|
276
|
+
for i, u in enumerate(uniq_shifts):
|
|
277
|
+
drifty_templates[i] = reducer(waveforms[n_pitches_shift == u], axis=0)
|
|
278
|
+
if is_tensor:
|
|
279
|
+
assert torch.is_tensor(drifty_templates)
|
|
280
|
+
drifty_templates = drifty_templates.numpy(force=True)
|
|
281
|
+
|
|
282
|
+
static_templates = get_waveforms_on_static_channels(
|
|
283
|
+
drifty_templates,
|
|
284
|
+
geom,
|
|
285
|
+
n_pitches_shift=uniq_shifts,
|
|
286
|
+
registered_geom=registered_geom,
|
|
287
|
+
target_kdtree=registered_kdtree,
|
|
288
|
+
match_distance=match_distance,
|
|
289
|
+
fill_value=np.nan,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# weighted mean is easier than weighted median, and we want this to be weighted
|
|
293
|
+
valid = ~np.isnan(static_templates[:, 0, :])
|
|
294
|
+
counts_ = counts[:, None]
|
|
295
|
+
fractions = counts_ / counts.sum()
|
|
296
|
+
valid &= (fractions >= min_fraction_at_shift) | (counts_ >= min_count_at_shift)
|
|
297
|
+
weights = valid[:, None, :] * counts_[:, None, :]
|
|
298
|
+
weights = weights / np.maximum(weights.sum(0), 1)
|
|
299
|
+
template = (np.nan_to_num(static_templates) * weights).sum(0)
|
|
300
|
+
dtype = str(waveforms.dtype).split(".")[1] if is_tensor else waveforms.dtype
|
|
301
|
+
template = template.astype(dtype)
|
|
302
|
+
template[:, ~valid.any(0)] = np.nan
|
|
303
|
+
if not np.isnan(pad_value):
|
|
304
|
+
template = np.nan_to_num(template, copy=False, nan=pad_value)
|
|
305
|
+
|
|
306
|
+
assert template.ndim == 2
|
|
307
|
+
if two_d:
|
|
308
|
+
template = template[0, :]
|
|
309
|
+
|
|
310
|
+
return template
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def get_spike_pitch_shifts(
|
|
314
|
+
depths_um=None,
|
|
315
|
+
geom=None,
|
|
316
|
+
registered_depths_um=None,
|
|
317
|
+
times_s=None,
|
|
318
|
+
sorting=None,
|
|
319
|
+
motion_est=None,
|
|
320
|
+
pitch=None,
|
|
321
|
+
mode="round",
|
|
322
|
+
localizations_dataset_name="point_source_localizations",
|
|
323
|
+
):
|
|
324
|
+
"""Figure out coarse pitch shifts based on spike positions
|
|
325
|
+
|
|
326
|
+
Determine the number of pitches the probe would need to shift in
|
|
327
|
+
order to coarsely align a waveform to its registered position.
|
|
328
|
+
"""
|
|
329
|
+
if pitch is None:
|
|
330
|
+
pitch = get_pitch(geom)
|
|
331
|
+
|
|
332
|
+
if depths_um is None:
|
|
333
|
+
assert sorting is not None
|
|
334
|
+
depths_um = getattr(sorting, localizations_dataset_name)[:, 2]
|
|
335
|
+
|
|
336
|
+
if registered_depths_um is None and motion_est is None:
|
|
337
|
+
return np.zeros(depths_um.shape, dtype=int)
|
|
338
|
+
|
|
339
|
+
if registered_depths_um is None:
|
|
340
|
+
assert motion_est is not None
|
|
341
|
+
if times_s is None:
|
|
342
|
+
assert sorting is not None
|
|
343
|
+
times_s = sorting.times_seconds
|
|
344
|
+
probe_displacement = -motion_est.disp_at_s(times_s, depths_um)
|
|
345
|
+
else:
|
|
346
|
+
probe_displacement = registered_depths_um - depths_um
|
|
347
|
+
assert np.isfinite(probe_displacement).all()
|
|
348
|
+
|
|
349
|
+
if mode == "floor":
|
|
350
|
+
n_pitches_shift = (probe_displacement / pitch).astype(int)
|
|
351
|
+
elif mode == "round":
|
|
352
|
+
n_pitches_shift = np.round(probe_displacement / pitch).astype(int)
|
|
353
|
+
else:
|
|
354
|
+
assert False
|
|
355
|
+
|
|
356
|
+
return n_pitches_shift
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def invert_motion_estimate(motion_est, t_s, registered_depths_um):
|
|
360
|
+
""" """
|
|
361
|
+
assert np.isscalar(t_s)
|
|
362
|
+
|
|
363
|
+
if (
|
|
364
|
+
hasattr(motion_est, "spatial_bin_centers_um")
|
|
365
|
+
and motion_est.spatial_bin_centers_um is not None
|
|
366
|
+
):
|
|
367
|
+
# nonrigid motion
|
|
368
|
+
bin_centers = motion_est.spatial_bin_centers_um
|
|
369
|
+
t_s = np.full(bin_centers.shape, t_s)
|
|
370
|
+
bin_center_disps = motion_est.disp_at_s(t_s, depth_um=bin_centers)
|
|
371
|
+
# registered_bin_centers = motion_est.correct_s(t_s, depths_um=bin_centers)
|
|
372
|
+
registered_bin_centers = bin_centers - bin_center_disps
|
|
373
|
+
assert np.all(np.diff(registered_bin_centers) > 0), "Invertibility issue."
|
|
374
|
+
disps = np.interp(
|
|
375
|
+
registered_depths_um.clip(
|
|
376
|
+
registered_bin_centers.min(),
|
|
377
|
+
registered_bin_centers.max(),
|
|
378
|
+
),
|
|
379
|
+
registered_bin_centers,
|
|
380
|
+
bin_center_disps,
|
|
381
|
+
)
|
|
382
|
+
else:
|
|
383
|
+
# rigid motion
|
|
384
|
+
disps = motion_est.disp_at_s(t_s)
|
|
385
|
+
|
|
386
|
+
return registered_depths_um + disps
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# -- waveform channel neighborhood shifting
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def get_waveforms_on_static_channels(
|
|
393
|
+
waveforms,
|
|
394
|
+
geom,
|
|
395
|
+
main_channels=None,
|
|
396
|
+
channel_index=None,
|
|
397
|
+
target_channels=None,
|
|
398
|
+
n_pitches_shift=None,
|
|
399
|
+
registered_geom=None,
|
|
400
|
+
target_kdtree=None,
|
|
401
|
+
match_distance=None,
|
|
402
|
+
out=None,
|
|
403
|
+
fill_value=np.nan,
|
|
404
|
+
workers=4,
|
|
405
|
+
return_padded=False,
|
|
406
|
+
):
|
|
407
|
+
"""Load a set of drifting waveforms on a static set of channels
|
|
408
|
+
|
|
409
|
+
Waveforms are by default detected waveforms on the full probe with channel
|
|
410
|
+
locations stored in geom. If main_channels and channel_index are supplied,
|
|
411
|
+
then waveforms[i] appears on the channels indexed by channel_index[main_channels[i]].
|
|
412
|
+
|
|
413
|
+
Now, the user wants to extract a subset of channels from each waveform such that
|
|
414
|
+
the same channel in each subsetted waveform is always at the same physical position,
|
|
415
|
+
accounting for drift. Here, the drift is inputted by n_pitches_shift.
|
|
416
|
+
|
|
417
|
+
Arguments
|
|
418
|
+
---------
|
|
419
|
+
waveforms : (n_spikes, t (optional), c) array
|
|
420
|
+
geom : (n_channels_tot, probe_dim) array
|
|
421
|
+
Original channel positions
|
|
422
|
+
main_channels : int (n_spikes,) array
|
|
423
|
+
`waveforms[i]` lives on channels `channel_index[main_channels[i]]`
|
|
424
|
+
in the original geometry array `geom`
|
|
425
|
+
channel_index : int (n_channels_tot, c) array
|
|
426
|
+
target_channels : int (n_channels_target,) array
|
|
427
|
+
Optional subset of channels to restrict to
|
|
428
|
+
`waveforms[i]` will be restricted to the intersection of its channels
|
|
429
|
+
and this channel subset, after correcting from drift
|
|
430
|
+
n_pitches_shift : int (n_spikes,) array
|
|
431
|
+
The number of probe pitches (see `get_pitch`) by which each spike's
|
|
432
|
+
channel neighborhood will be shifted before matching with target channels
|
|
433
|
+
registered_geom : (n_registered_channels, probe_dim) array
|
|
434
|
+
If supplied, the target channel positions are loaded from here
|
|
435
|
+
fill_value : float
|
|
436
|
+
The value to impute when a target channel does not land on a channel
|
|
437
|
+
when shifted according to n_pitches_shift
|
|
438
|
+
|
|
439
|
+
Returns
|
|
440
|
+
-------
|
|
441
|
+
static_waveforms : (n_spikes, t (optional), n_channels_target) array
|
|
442
|
+
"""
|
|
443
|
+
# this supports amplitude vectors (i.e., 2d arrays)
|
|
444
|
+
two_d = waveforms.ndim == 2
|
|
445
|
+
|
|
446
|
+
# validate inputs to avoid confusing errors
|
|
447
|
+
assert waveforms.ndim in (2, 3)
|
|
448
|
+
if two_d:
|
|
449
|
+
waveforms = waveforms[:, None, :]
|
|
450
|
+
n_spikes, t, c = waveforms.shape
|
|
451
|
+
assert geom.ndim == 2
|
|
452
|
+
n_channels_tot = geom.shape[0]
|
|
453
|
+
if main_channels is not None:
|
|
454
|
+
assert channel_index is not None
|
|
455
|
+
n_channels_tot_, c_ = channel_index.shape
|
|
456
|
+
assert c_ == c and n_channels_tot_ == n_channels_tot
|
|
457
|
+
assert main_channels.shape == (n_spikes,)
|
|
458
|
+
else:
|
|
459
|
+
# if per-wf channels are not supplied, then we assume
|
|
460
|
+
# that waveforms live on all channels
|
|
461
|
+
assert c == n_channels_tot
|
|
462
|
+
if n_pitches_shift is not None:
|
|
463
|
+
assert n_pitches_shift.shape == (n_spikes,)
|
|
464
|
+
if match_distance is None:
|
|
465
|
+
match_distance = pdist(geom).min() / 2
|
|
466
|
+
|
|
467
|
+
# grab the positions of the channels that we are targeting
|
|
468
|
+
target_geom = geom
|
|
469
|
+
if registered_geom is not None:
|
|
470
|
+
target_geom = registered_geom
|
|
471
|
+
if target_channels is not None:
|
|
472
|
+
pad_channel = target_geom.max(0) + 10 * match_distance
|
|
473
|
+
target_geom = np.concatenate((target_geom, pad_channel[None]))
|
|
474
|
+
target_geom = target_geom[target_channels]
|
|
475
|
+
|
|
476
|
+
# make kdtree
|
|
477
|
+
if target_kdtree is None:
|
|
478
|
+
target_kdtree = KDTree(target_geom)
|
|
479
|
+
else:
|
|
480
|
+
assert target_kdtree.n == len(target_geom)
|
|
481
|
+
n_static_channels = len(target_geom)
|
|
482
|
+
|
|
483
|
+
# figure out the positions of the channels that the waveforms live on
|
|
484
|
+
pitch = get_pitch(geom)
|
|
485
|
+
if n_pitches_shift is None:
|
|
486
|
+
n_pitches_shift = np.zeros(n_spikes, dtype=np.int64)
|
|
487
|
+
if main_channels is None:
|
|
488
|
+
# the case where all waveforms live on all channels, but
|
|
489
|
+
# these channels may be shifting
|
|
490
|
+
# shape is n_spikes, len(geom), spatial dim
|
|
491
|
+
static_waveforms = _full_probe_shifting_fast(
|
|
492
|
+
waveforms,
|
|
493
|
+
geom,
|
|
494
|
+
pitch,
|
|
495
|
+
target_kdtree,
|
|
496
|
+
n_pitches_shift,
|
|
497
|
+
match_distance,
|
|
498
|
+
fill_value,
|
|
499
|
+
out=out,
|
|
500
|
+
return_padded=return_padded,
|
|
501
|
+
)
|
|
502
|
+
if two_d:
|
|
503
|
+
static_waveforms = static_waveforms[:, 0, :]
|
|
504
|
+
return static_waveforms
|
|
505
|
+
# moving_positions = geom[None, :, :] + shifts[:, None, :]
|
|
506
|
+
# valid_chan = slice(None)
|
|
507
|
+
# else:
|
|
508
|
+
|
|
509
|
+
# the case where each waveform lives on its own channels
|
|
510
|
+
# nans will never be matched in k-d query below
|
|
511
|
+
padded_geom = np.pad(
|
|
512
|
+
geom.astype(np.float64), [(0, 1), (0, 0)], constant_values=np.nan
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
# ok, the kdtree query can get expensive when we have lots of these shifting
|
|
516
|
+
# positions. it turns out to be worth it to go through the effort of figuring
|
|
517
|
+
# out what the unique valid moving positions are and just targeting those
|
|
518
|
+
channels_and_shifts = np.c_[main_channels, n_pitches_shift]
|
|
519
|
+
uniq_channels_and_shifts, uniq_inv = np.unique(
|
|
520
|
+
channels_and_shifts, axis=0, return_inverse=True
|
|
521
|
+
)
|
|
522
|
+
uniq_channels, uniq_n_pitches_shift = uniq_channels_and_shifts.T
|
|
523
|
+
uniq_shifts = np.c_[np.zeros(uniq_channels.shape[0]), uniq_n_pitches_shift * pitch]
|
|
524
|
+
assert channel_index is not None
|
|
525
|
+
uniq_moving_pos = (
|
|
526
|
+
padded_geom[channel_index[uniq_channels]] + uniq_shifts[:, None, :]
|
|
527
|
+
)
|
|
528
|
+
uniq_valid = channel_index[uniq_channels] < n_channels_tot
|
|
529
|
+
|
|
530
|
+
_, uniq_shifted_channels = target_kdtree.query(
|
|
531
|
+
uniq_moving_pos[uniq_valid],
|
|
532
|
+
distance_upper_bound=match_distance,
|
|
533
|
+
workers=workers,
|
|
534
|
+
)
|
|
535
|
+
if uniq_shifted_channels.size < uniq_channels.shape[0] * c:
|
|
536
|
+
uniq_shifted_channels_ = uniq_shifted_channels
|
|
537
|
+
uniq_shifted_channels = np.full((uniq_channels.shape[0], c), target_kdtree.n)
|
|
538
|
+
uniq_shifted_channels[uniq_valid] = uniq_shifted_channels_
|
|
539
|
+
uniq_shifted_channels = uniq_shifted_channels.reshape(uniq_channels.shape[0], c)
|
|
540
|
+
|
|
541
|
+
# ok, now we can return to non-unique world
|
|
542
|
+
shifted_channels = uniq_shifted_channels[uniq_inv]
|
|
543
|
+
|
|
544
|
+
# # shape is n_spikes, c, spatial dim
|
|
545
|
+
# moving_positions = padded_geom[channel_index[main_channels]] + shifts[:, None, :]
|
|
546
|
+
# # valid_chan = ~np.isnan(moving_positions).any(axis=1)
|
|
547
|
+
# valid_chan = channel_index[main_channels] < n_channels_tot
|
|
548
|
+
|
|
549
|
+
# _, uniq_shifted_channels = target_kdtree.query(
|
|
550
|
+
# moving_positions[valid_chan[:, :]],
|
|
551
|
+
# distance_upper_bound=match_distance,
|
|
552
|
+
# workers=workers,
|
|
553
|
+
# )
|
|
554
|
+
# shifted_channels = uniq_shifted_channels[uniq_inv]
|
|
555
|
+
|
|
556
|
+
# now, if not all chans were valid, fix that up...
|
|
557
|
+
# if shifted_channels.size < n_spikes * c:
|
|
558
|
+
# shifted_channels_ = shifted_channels
|
|
559
|
+
# shifted_channels = np.full((n_spikes, c), target_kdtree.n)
|
|
560
|
+
# shifted_channels[valid_chan] = shifted_channels_
|
|
561
|
+
# shifted_channels = shifted_channels.reshape(n_spikes, c)
|
|
562
|
+
|
|
563
|
+
# scatter the waveforms into their static channel neighborhoods
|
|
564
|
+
if out is None and torch.is_tensor(waveforms):
|
|
565
|
+
static_waveforms = waveforms.new_full(
|
|
566
|
+
(n_spikes, t, n_static_channels + 1),
|
|
567
|
+
fill_value=fill_value,
|
|
568
|
+
)
|
|
569
|
+
elif torch.is_tensor(waveforms):
|
|
570
|
+
assert out is not None
|
|
571
|
+
assert out.shape == (n_spikes, t, n_static_channels + 1)
|
|
572
|
+
assert torch.is_tensor(out)
|
|
573
|
+
out.fill_(fill_value)
|
|
574
|
+
static_waveforms = out
|
|
575
|
+
else:
|
|
576
|
+
static_waveforms = None
|
|
577
|
+
|
|
578
|
+
if torch.is_tensor(waveforms):
|
|
579
|
+
ix = torch.asarray(shifted_channels[:, None, :])
|
|
580
|
+
ix = ix.broadcast_to(waveforms.shape)
|
|
581
|
+
assert static_waveforms is not None
|
|
582
|
+
static_waveforms.scatter_(dim=3, index=ix, src=waveforms)
|
|
583
|
+
if not return_padded:
|
|
584
|
+
static_waveforms = static_waveforms[:, :, :n_static_channels]
|
|
585
|
+
if two_d:
|
|
586
|
+
static_waveforms = static_waveforms[:, 0]
|
|
587
|
+
return static_waveforms
|
|
588
|
+
|
|
589
|
+
# below, numpy case.
|
|
590
|
+
if out is None:
|
|
591
|
+
static_waveforms = np.full(
|
|
592
|
+
(n_spikes, t, n_static_channels + 1),
|
|
593
|
+
fill_value=fill_value,
|
|
594
|
+
dtype=waveforms.dtype,
|
|
595
|
+
)
|
|
596
|
+
else:
|
|
597
|
+
assert isinstance(out, np.ndarray)
|
|
598
|
+
assert out.shape == (n_spikes, t, n_static_channels + 1)
|
|
599
|
+
out.fill(fill_value)
|
|
600
|
+
static_waveforms = out
|
|
601
|
+
spike_ix = np.arange(n_spikes)[:, None, None]
|
|
602
|
+
time_ix = np.arange(t)[None, :, None]
|
|
603
|
+
chan_ix = shifted_channels[:, None, :]
|
|
604
|
+
static_waveforms[spike_ix, time_ix, chan_ix] = waveforms
|
|
605
|
+
if not return_padded:
|
|
606
|
+
static_waveforms = static_waveforms[:, :, :n_static_channels]
|
|
607
|
+
if two_d:
|
|
608
|
+
static_waveforms = static_waveforms[:, 0, :]
|
|
609
|
+
return static_waveforms
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _full_probe_shifting_fast(
|
|
613
|
+
waveforms,
|
|
614
|
+
geom,
|
|
615
|
+
pitch,
|
|
616
|
+
target_kdtree,
|
|
617
|
+
n_pitches_shift,
|
|
618
|
+
match_distance,
|
|
619
|
+
fill_value,
|
|
620
|
+
out=None,
|
|
621
|
+
return_padded=False,
|
|
622
|
+
):
|
|
623
|
+
is_tensor = torch.is_tensor(waveforms)
|
|
624
|
+
|
|
625
|
+
if out is None:
|
|
626
|
+
if is_tensor:
|
|
627
|
+
static_waveforms = torch.full(
|
|
628
|
+
(*waveforms.shape[:2], target_kdtree.n + 1),
|
|
629
|
+
fill_value=fill_value,
|
|
630
|
+
dtype=waveforms.dtype,
|
|
631
|
+
device=waveforms.device,
|
|
632
|
+
)
|
|
633
|
+
else:
|
|
634
|
+
static_waveforms = np.full(
|
|
635
|
+
(*waveforms.shape[:2], target_kdtree.n + 1),
|
|
636
|
+
fill_value=fill_value,
|
|
637
|
+
dtype=waveforms.dtype,
|
|
638
|
+
)
|
|
639
|
+
else:
|
|
640
|
+
assert out.shape == (*waveforms.shape[:2], target_kdtree.n + 1)
|
|
641
|
+
out.fill(fill_value)
|
|
642
|
+
static_waveforms = out
|
|
643
|
+
|
|
644
|
+
no_shift = n_pitches_shift is None
|
|
645
|
+
if no_shift:
|
|
646
|
+
n_pitches_shift = [0]
|
|
647
|
+
unps, shift_inverse = np.unique(n_pitches_shift, return_inverse=True)
|
|
648
|
+
shifted_channels = np.full((len(unps), waveforms.shape[2]), target_kdtree.n)
|
|
649
|
+
for i, nps in enumerate(unps):
|
|
650
|
+
moving_geom = geom + [0, pitch * nps]
|
|
651
|
+
_, shifted_channels[i] = target_kdtree.query(
|
|
652
|
+
moving_geom, distance_upper_bound=match_distance
|
|
653
|
+
)
|
|
654
|
+
# static_waveforms[which[:, None, None], tix[None, :, None], shifted_channels[None, None, :]] = waveforms[which]
|
|
655
|
+
nix = np.arange(waveforms.shape[0])
|
|
656
|
+
tix = np.arange(waveforms.shape[1])
|
|
657
|
+
static_waveforms[
|
|
658
|
+
nix[:, None, None],
|
|
659
|
+
tix[None, :, None],
|
|
660
|
+
shifted_channels[shift_inverse][:, None, :],
|
|
661
|
+
] = waveforms
|
|
662
|
+
if return_padded:
|
|
663
|
+
return static_waveforms
|
|
664
|
+
else:
|
|
665
|
+
return static_waveforms[:, :, : target_kdtree.n]
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def static_channel_neighborhoods(
|
|
669
|
+
geom,
|
|
670
|
+
main_channels,
|
|
671
|
+
channel_index,
|
|
672
|
+
target_channels=None,
|
|
673
|
+
pitch=None,
|
|
674
|
+
n_pitches_shift=None,
|
|
675
|
+
registered_geom=None,
|
|
676
|
+
target_kdtree=None,
|
|
677
|
+
match_distance=None,
|
|
678
|
+
uniq_channels_and_shifts=None,
|
|
679
|
+
uniq_inv=None,
|
|
680
|
+
workers=4,
|
|
681
|
+
):
|
|
682
|
+
# validate inputs to avoid confusing errors
|
|
683
|
+
n_spikes = main_channels.shape[0]
|
|
684
|
+
assert geom.ndim == 2
|
|
685
|
+
n_channels_tot = geom.shape[0]
|
|
686
|
+
assert channel_index is not None
|
|
687
|
+
n_channels_tot_, c = channel_index.shape
|
|
688
|
+
assert n_channels_tot_ == n_channels_tot
|
|
689
|
+
assert main_channels.shape == (n_spikes,)
|
|
690
|
+
if n_pitches_shift is not None:
|
|
691
|
+
assert n_pitches_shift.shape == (n_spikes,)
|
|
692
|
+
if match_distance is None:
|
|
693
|
+
match_distance = pdist(geom).min() / 1.5
|
|
694
|
+
|
|
695
|
+
# grab the positions of the channels that we are targeting
|
|
696
|
+
target_geom = geom
|
|
697
|
+
if registered_geom is not None:
|
|
698
|
+
target_geom = registered_geom
|
|
699
|
+
if target_channels is not None:
|
|
700
|
+
pad_channel = target_geom.max(0) + 10 * match_distance
|
|
701
|
+
target_geom = np.concatenate((target_geom, pad_channel[None]))
|
|
702
|
+
target_geom = target_geom[target_channels]
|
|
703
|
+
|
|
704
|
+
# make kdtree
|
|
705
|
+
if target_kdtree is None:
|
|
706
|
+
target_kdtree = KDTree(target_geom)
|
|
707
|
+
else:
|
|
708
|
+
assert target_kdtree.n == len(target_geom)
|
|
709
|
+
|
|
710
|
+
if pitch is None:
|
|
711
|
+
pitch = get_pitch(geom)
|
|
712
|
+
if n_pitches_shift is None:
|
|
713
|
+
n_pitches_shift = np.zeros(n_spikes, dtype=np.int64)
|
|
714
|
+
|
|
715
|
+
# the case where each waveform lives on its own channels
|
|
716
|
+
# nans will never be matched in k-d query below
|
|
717
|
+
padded_geom = np.pad(geom.astype(float), [(0, 1), (0, 0)], constant_values=np.nan)
|
|
718
|
+
|
|
719
|
+
# ok, the kdtree query can get expensive when we have lots of these shifting
|
|
720
|
+
# positions. it turns out to be worth it to go through the effort of figuring
|
|
721
|
+
# out what the unique valid moving positions are and just targeting those
|
|
722
|
+
if uniq_channels_and_shifts is None:
|
|
723
|
+
channels_and_shifts = np.c_[main_channels, n_pitches_shift]
|
|
724
|
+
uniq_channels_and_shifts, uniq_inv = np.unique(
|
|
725
|
+
channels_and_shifts, axis=0, return_inverse=True
|
|
726
|
+
)
|
|
727
|
+
uniq_channels, uniq_n_pitches_shift = uniq_channels_and_shifts.T
|
|
728
|
+
uniq_shifts = np.c_[np.zeros(uniq_channels.shape[0]), uniq_n_pitches_shift * pitch]
|
|
729
|
+
uniq_moving_neighbs = (
|
|
730
|
+
padded_geom[channel_index[uniq_channels]] + uniq_shifts[:, None, :]
|
|
731
|
+
)
|
|
732
|
+
ii, jj = np.nonzero(channel_index[uniq_channels] < n_channels_tot)
|
|
733
|
+
|
|
734
|
+
_, uniq_shifted_channels = target_kdtree.query(
|
|
735
|
+
uniq_moving_neighbs[ii, jj],
|
|
736
|
+
distance_upper_bound=match_distance,
|
|
737
|
+
workers=workers,
|
|
738
|
+
)
|
|
739
|
+
if uniq_shifted_channels.size < uniq_channels.shape[0] * c:
|
|
740
|
+
uniq_shifted_channels_ = uniq_shifted_channels
|
|
741
|
+
uniq_shifted_channels = np.full((uniq_channels.shape[0], c), target_kdtree.n)
|
|
742
|
+
uniq_shifted_channels[ii, jj] = uniq_shifted_channels_
|
|
743
|
+
uniq_shifted_channels = uniq_shifted_channels.reshape(uniq_channels.shape[0], c)
|
|
744
|
+
|
|
745
|
+
# now, typically the unique channel-shift pairs are very redundant. many of
|
|
746
|
+
# those unique values correspond to the same final neighborhood. and it's well
|
|
747
|
+
# worth it for other parts of the code if we can cut down the number of
|
|
748
|
+
# neighborhoods. so, we do a last round of deduplication.
|
|
749
|
+
neighborhoods, scid_to_neighborhood_id = np.unique(
|
|
750
|
+
uniq_shifted_channels, axis=0, return_inverse=True
|
|
751
|
+
)
|
|
752
|
+
neighborhood_ids = scid_to_neighborhood_id[uniq_inv]
|
|
753
|
+
|
|
754
|
+
# ok, now we can return to non-unique world
|
|
755
|
+
shifted_channels = neighborhoods[neighborhood_ids]
|
|
756
|
+
|
|
757
|
+
return shifted_channels, neighborhoods, neighborhood_ids
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def grab_static(waveforms, shifted_channels, n_static_channels, fill_value=np.nan):
|
|
761
|
+
two_d = waveforms.ndim == 2
|
|
762
|
+
if two_d:
|
|
763
|
+
waveforms = waveforms[:, None, :]
|
|
764
|
+
n_spikes, t, c = waveforms.shape
|
|
765
|
+
if torch.is_tensor(waveforms):
|
|
766
|
+
static_waveforms = torch.full(
|
|
767
|
+
(n_spikes, t, n_static_channels + 1),
|
|
768
|
+
fill_value=fill_value,
|
|
769
|
+
dtype=waveforms.dtype,
|
|
770
|
+
device=waveforms.device,
|
|
771
|
+
)
|
|
772
|
+
else:
|
|
773
|
+
static_waveforms = np.full(
|
|
774
|
+
(n_spikes, t, n_static_channels + 1),
|
|
775
|
+
fill_value=fill_value,
|
|
776
|
+
dtype=waveforms.dtype,
|
|
777
|
+
)
|
|
778
|
+
spike_ix = np.arange(n_spikes)[:, None, None]
|
|
779
|
+
time_ix = np.arange(t)[None, :, None]
|
|
780
|
+
chan_ix = shifted_channels[:, None, :]
|
|
781
|
+
static_waveforms[spike_ix, time_ix, chan_ix] = waveforms
|
|
782
|
+
static_waveforms = static_waveforms[:, :, :n_static_channels]
|
|
783
|
+
if two_d:
|
|
784
|
+
static_waveforms = static_waveforms[:, 0, :]
|
|
785
|
+
|
|
786
|
+
return static_waveforms
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
# -- which templates appear at which shifts in a recording?
|
|
790
|
+
# and, which pairs of shifted templates appear together?
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
@dataclass
|
|
794
|
+
class TemplateShiftIndex:
|
|
795
|
+
"""Return value for get_shift_and_unit_pairs"""
|
|
796
|
+
|
|
797
|
+
n_shifted_templates: int
|
|
798
|
+
# shift index -> shift
|
|
799
|
+
all_pitch_shifts: np.ndarray
|
|
800
|
+
# (template ix, shift index) -> shifted template index
|
|
801
|
+
template_shift_index: np.ndarray
|
|
802
|
+
# (shifted temp ix, shifted temp ix) -> did these appear at the same time
|
|
803
|
+
shifted_temp_ix_to_temp_ix: np.ndarray
|
|
804
|
+
shifted_temp_ix_to_shift: np.ndarray
|
|
805
|
+
|
|
806
|
+
@classmethod
|
|
807
|
+
def from_shift_matrix(cls, shifts):
|
|
808
|
+
"""shift: n_times x n_templates"""
|
|
809
|
+
all_shifts = np.unique(shifts)
|
|
810
|
+
n_templates = shifts.shape[1]
|
|
811
|
+
pairs = np.stack(
|
|
812
|
+
np.broadcast_arrays(np.arange(n_templates)[None, :], shifts), axis=2
|
|
813
|
+
)
|
|
814
|
+
pairs = np.unique(pairs.reshape(shifts.size, 2), axis=0)
|
|
815
|
+
n_shifted_templates = len(pairs)
|
|
816
|
+
shift_ix = np.searchsorted(all_shifts, pairs[:, 1])
|
|
817
|
+
template_shift_index = np.full(
|
|
818
|
+
(n_templates, len(all_shifts)), n_shifted_templates
|
|
819
|
+
)
|
|
820
|
+
template_shift_index[pairs[:, 0], shift_ix] = np.arange(n_shifted_templates)
|
|
821
|
+
return cls(
|
|
822
|
+
n_shifted_templates,
|
|
823
|
+
all_shifts,
|
|
824
|
+
template_shift_index,
|
|
825
|
+
*pairs.T,
|
|
826
|
+
)
|
|
827
|
+
|
|
828
|
+
def shifts_to_shifted_ids(self, template_ids, shifts):
|
|
829
|
+
shift_ixs = np.searchsorted(self.all_pitch_shifts, shifts)
|
|
830
|
+
return self.template_shift_index[template_ids, shift_ixs]
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def static_template_shift_index(n_templates):
|
|
834
|
+
temp_ixs = np.arange(n_templates)
|
|
835
|
+
return TemplateShiftIndex(
|
|
836
|
+
n_templates,
|
|
837
|
+
np.zeros(1),
|
|
838
|
+
temp_ixs[:, None],
|
|
839
|
+
temp_ixs,
|
|
840
|
+
np.zeros_like(temp_ixs),
|
|
841
|
+
)
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def get_shift_and_unit_pairs(
|
|
845
|
+
chunk_time_centers_s,
|
|
846
|
+
geom,
|
|
847
|
+
template_data_a,
|
|
848
|
+
template_data_b=None,
|
|
849
|
+
motion_est=None,
|
|
850
|
+
):
|
|
851
|
+
if template_data_b is None:
|
|
852
|
+
template_data_b = template_data_a
|
|
853
|
+
|
|
854
|
+
na = template_data_a.templates.shape[0]
|
|
855
|
+
nb = template_data_b.templates.shape[0]
|
|
856
|
+
|
|
857
|
+
if motion_est is None:
|
|
858
|
+
shift_index_a = static_template_shift_index(na)
|
|
859
|
+
shift_index_b = static_template_shift_index(nb)
|
|
860
|
+
cooccurrence = np.ones((na, nb), dtype=bool)
|
|
861
|
+
return shift_index_a, shift_index_b, cooccurrence
|
|
862
|
+
|
|
863
|
+
reg_depths_um_a = template_data_a.registered_depths_um()
|
|
864
|
+
reg_depths_um_b = template_data_b.registered_depths_um()
|
|
865
|
+
same = np.array_equal(reg_depths_um_a, reg_depths_um_b)
|
|
866
|
+
if same:
|
|
867
|
+
reg_depths_um = reg_depths_um_a
|
|
868
|
+
else:
|
|
869
|
+
reg_depths_um = np.concatenate((reg_depths_um_a, reg_depths_um_b))
|
|
870
|
+
|
|
871
|
+
# figure out all shifts for all units at all times
|
|
872
|
+
unreg_depths_um = np.stack(
|
|
873
|
+
[
|
|
874
|
+
invert_motion_estimate(motion_est, t_s, reg_depths_um)
|
|
875
|
+
for t_s in chunk_time_centers_s
|
|
876
|
+
],
|
|
877
|
+
axis=0,
|
|
878
|
+
)
|
|
879
|
+
assert unreg_depths_um.shape == (len(chunk_time_centers_s), len(reg_depths_um))
|
|
880
|
+
diff = reg_depths_um - unreg_depths_um
|
|
881
|
+
pitch_shifts = get_spike_pitch_shifts(
|
|
882
|
+
depths_um=reg_depths_um,
|
|
883
|
+
pitch=get_pitch(geom),
|
|
884
|
+
registered_depths_um=unreg_depths_um,
|
|
885
|
+
)
|
|
886
|
+
if same:
|
|
887
|
+
shifts_a = shifts_b = pitch_shifts
|
|
888
|
+
else:
|
|
889
|
+
shifts_a = pitch_shifts[:, :na]
|
|
890
|
+
shifts_b = pitch_shifts[:, na:]
|
|
891
|
+
|
|
892
|
+
# assign ids to pitch/shift pairs
|
|
893
|
+
template_shift_index_a = TemplateShiftIndex.from_shift_matrix(shifts_a)
|
|
894
|
+
if same:
|
|
895
|
+
template_shift_index_b = template_shift_index_a
|
|
896
|
+
else:
|
|
897
|
+
template_shift_index_b = TemplateShiftIndex.from_shift_matrix(shifts_b)
|
|
898
|
+
|
|
899
|
+
# co-occurrence matrix: do these shifted templates appear together?
|
|
900
|
+
cooccurrence = np.zeros(
|
|
901
|
+
(
|
|
902
|
+
template_shift_index_a.n_shifted_templates,
|
|
903
|
+
template_shift_index_b.n_shifted_templates,
|
|
904
|
+
),
|
|
905
|
+
dtype=bool,
|
|
906
|
+
)
|
|
907
|
+
temps_a = np.arange(na)
|
|
908
|
+
temps_b = np.arange(nb)
|
|
909
|
+
for j in range(len(chunk_time_centers_s)):
|
|
910
|
+
shifted_ids_a = template_shift_index_a.shifts_to_shifted_ids(
|
|
911
|
+
temps_a, shifts_a[j]
|
|
912
|
+
)
|
|
913
|
+
if same:
|
|
914
|
+
shifted_ids_b = shifted_ids_a
|
|
915
|
+
else:
|
|
916
|
+
shifted_ids_b = template_shift_index_b.shifts_to_shifted_ids(
|
|
917
|
+
temps_b, shifts_b[j]
|
|
918
|
+
)
|
|
919
|
+
cooccurrence[shifted_ids_a[:, None], shifted_ids_b[None, :]] = 1
|
|
920
|
+
|
|
921
|
+
return template_shift_index_a, template_shift_index_b, cooccurrence
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def get_shift_info(
|
|
925
|
+
sorting, motion_est, geom, motion_depth_mode="channel", channels_mode="round"
|
|
926
|
+
):
|
|
927
|
+
"""
|
|
928
|
+
shifts = reg_depths - depths
|
|
929
|
+
reg_depths = depths + shifts
|
|
930
|
+
i.e., target_pos = source_pos + shifts
|
|
931
|
+
target_pos - shifts = source_pos
|
|
932
|
+
where by source pos i mean the true moving positions.
|
|
933
|
+
"""
|
|
934
|
+
times_s = sorting.times_seconds
|
|
935
|
+
channels = sorting.channels
|
|
936
|
+
if motion_depth_mode == "localization":
|
|
937
|
+
depths = sorting.point_source_localizations[:, 2]
|
|
938
|
+
elif motion_depth_mode == "channel":
|
|
939
|
+
depths = geom[:, 1][channels]
|
|
940
|
+
else:
|
|
941
|
+
assert False
|
|
942
|
+
|
|
943
|
+
rdepths = depths
|
|
944
|
+
if motion_est is not None:
|
|
945
|
+
rdepths = motion_est.correct_s(times_s, depths)
|
|
946
|
+
|
|
947
|
+
# shift = -displacement and rz = z - displacement
|
|
948
|
+
shifts = rdepths - depths
|
|
949
|
+
|
|
950
|
+
if motion_est is None:
|
|
951
|
+
n_pitches_shift = np.zeros(len(depths), dtype=np.int64)
|
|
952
|
+
else:
|
|
953
|
+
n_pitches_shift = get_spike_pitch_shifts(
|
|
954
|
+
depths,
|
|
955
|
+
geom=geom,
|
|
956
|
+
motion_est=motion_est,
|
|
957
|
+
times_s=times_s,
|
|
958
|
+
registered_depths_um=rdepths,
|
|
959
|
+
mode=channels_mode,
|
|
960
|
+
)
|
|
961
|
+
|
|
962
|
+
# pitch = drift_util.get_pitch(geom)
|
|
963
|
+
# remaining_shift = shifts - pitch * n_pitches_shift
|
|
964
|
+
|
|
965
|
+
return channels, shifts, n_pitches_shift
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def get_stable_channels(
|
|
969
|
+
geom,
|
|
970
|
+
channels,
|
|
971
|
+
channel_index,
|
|
972
|
+
registered_geom,
|
|
973
|
+
n_pitches_shift,
|
|
974
|
+
core_radius=None,
|
|
975
|
+
workers=-1,
|
|
976
|
+
device=None,
|
|
977
|
+
):
|
|
978
|
+
core_channel_index = None
|
|
979
|
+
if core_radius == "extract":
|
|
980
|
+
core_channel_index = channel_index
|
|
981
|
+
elif core_radius is not None:
|
|
982
|
+
core_channel_index = make_channel_index(geom, core_radius)
|
|
983
|
+
|
|
984
|
+
# extract the main unique chans computation
|
|
985
|
+
c = torch.asarray(channels, dtype=torch.int32)
|
|
986
|
+
s = torch.asarray(n_pitches_shift, dtype=torch.int32)
|
|
987
|
+
cs = torch.column_stack((c, s))
|
|
988
|
+
if device is not None:
|
|
989
|
+
cs = cs.to(device)
|
|
990
|
+
uniq_channels_and_shifts, uniq_inv = torch.unique(cs, dim=0, return_inverse=True)
|
|
991
|
+
uniq_channels_and_shifts = uniq_channels_and_shifts.numpy(force=True)
|
|
992
|
+
uniq_inv = uniq_inv.numpy(force=True)
|
|
993
|
+
|
|
994
|
+
# precompute these, shared across next two calls
|
|
995
|
+
pitch = get_pitch(geom)
|
|
996
|
+
registered_kdtree = KDTree(registered_geom)
|
|
997
|
+
match_distance = pdist(geom).min() / 1.5
|
|
998
|
+
|
|
999
|
+
extract_channels, extract_neighborhoods, extract_neighborhood_ids = (
|
|
1000
|
+
static_channel_neighborhoods(
|
|
1001
|
+
geom,
|
|
1002
|
+
channels,
|
|
1003
|
+
channel_index,
|
|
1004
|
+
pitch=pitch,
|
|
1005
|
+
n_pitches_shift=n_pitches_shift,
|
|
1006
|
+
registered_geom=registered_geom,
|
|
1007
|
+
target_kdtree=registered_kdtree,
|
|
1008
|
+
match_distance=match_distance,
|
|
1009
|
+
uniq_channels_and_shifts=uniq_channels_and_shifts,
|
|
1010
|
+
uniq_inv=uniq_inv,
|
|
1011
|
+
workers=workers,
|
|
1012
|
+
)
|
|
1013
|
+
)
|
|
1014
|
+
core_channels = core_neighborhoods = core_neighborhood_ids = None
|
|
1015
|
+
if core_radius == "extract":
|
|
1016
|
+
core_channels = extract_channels
|
|
1017
|
+
core_neighborhoods = extract_neighborhoods
|
|
1018
|
+
core_neighborhood_ids = extract_neighborhood_ids
|
|
1019
|
+
elif core_radius is not None:
|
|
1020
|
+
core_channels, core_neighborhoods, core_neighborhood_ids = (
|
|
1021
|
+
static_channel_neighborhoods(
|
|
1022
|
+
geom,
|
|
1023
|
+
channels,
|
|
1024
|
+
core_channel_index,
|
|
1025
|
+
pitch=pitch,
|
|
1026
|
+
n_pitches_shift=n_pitches_shift,
|
|
1027
|
+
registered_geom=registered_geom,
|
|
1028
|
+
target_kdtree=registered_kdtree,
|
|
1029
|
+
match_distance=match_distance,
|
|
1030
|
+
uniq_channels_and_shifts=uniq_channels_and_shifts,
|
|
1031
|
+
uniq_inv=uniq_inv,
|
|
1032
|
+
workers=workers,
|
|
1033
|
+
)
|
|
1034
|
+
)
|
|
1035
|
+
|
|
1036
|
+
return (
|
|
1037
|
+
extract_channels,
|
|
1038
|
+
extract_neighborhoods,
|
|
1039
|
+
extract_neighborhood_ids,
|
|
1040
|
+
core_channels,
|
|
1041
|
+
core_neighborhoods,
|
|
1042
|
+
core_neighborhood_ids,
|
|
1043
|
+
)
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
def unique_with_index(x, dim=0):
|
|
1047
|
+
# https://github.com/pytorch/pytorch/issues/36748
|
|
1048
|
+
unique, inverse, counts = torch.unique(
|
|
1049
|
+
x, dim=dim, sorted=True, return_inverse=True, return_counts=True
|
|
1050
|
+
)
|
|
1051
|
+
inv_sorted = inverse.argsort(stable=True)
|
|
1052
|
+
tot_counts = torch.cat((counts.new_zeros(1), counts.cumsum(dim=0)))[:-1]
|
|
1053
|
+
index = inv_sorted[tot_counts]
|
|
1054
|
+
return unique, inverse, counts, index
|