DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,518 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from dataclasses import dataclass, replace
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as F
|
|
7
|
+
from scipy.interpolate import interp1d
|
|
8
|
+
from scipy.spatial import KDTree
|
|
9
|
+
|
|
10
|
+
from ..util import drift_util, waveform_util
|
|
11
|
+
from ..util.job_util import ensure_computation_config
|
|
12
|
+
from ..util.logging_util import DARTSORTVERBOSE, get_logger
|
|
13
|
+
from ..util.spiketorch import ptp
|
|
14
|
+
from ..util.waveform_util import full_channel_index
|
|
15
|
+
|
|
16
|
+
logger = get_logger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def weighted_average(unit_ids, templates, weights):
|
|
20
|
+
n_out = unit_ids.max() + 1
|
|
21
|
+
n_in, t, c = templates.shape
|
|
22
|
+
out = np.zeros((n_out, t, c), dtype=templates.dtype)
|
|
23
|
+
weights = weights.astype(float)
|
|
24
|
+
for i in range(n_out):
|
|
25
|
+
which_in = np.flatnonzero(unit_ids == i)
|
|
26
|
+
if not which_in.size:
|
|
27
|
+
continue
|
|
28
|
+
|
|
29
|
+
w = weights[which_in][:, None, None]
|
|
30
|
+
if w.sum() == 0:
|
|
31
|
+
continue
|
|
32
|
+
w /= w.sum()
|
|
33
|
+
out[i] = (w * templates[which_in]).sum(0)
|
|
34
|
+
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# -- template drift handling
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def templates_at_time(
|
|
42
|
+
t_s,
|
|
43
|
+
registered_templates,
|
|
44
|
+
geom,
|
|
45
|
+
registered_template_depths_um=None,
|
|
46
|
+
registered_geom=None,
|
|
47
|
+
motion_est=None,
|
|
48
|
+
return_pitch_shifts=False,
|
|
49
|
+
geom_kdtree=None,
|
|
50
|
+
match_distance=None,
|
|
51
|
+
return_padded=False,
|
|
52
|
+
fill_value=torch.nan,
|
|
53
|
+
):
|
|
54
|
+
if registered_geom is None and not return_padded:
|
|
55
|
+
return registered_templates
|
|
56
|
+
if registered_geom is None and return_padded:
|
|
57
|
+
assert torch.is_tensor(registered_templates)
|
|
58
|
+
return F.pad(registered_templates, (0, 1), value=fill_value)
|
|
59
|
+
assert motion_est is not None
|
|
60
|
+
assert registered_template_depths_um is not None
|
|
61
|
+
|
|
62
|
+
# for each unit, extract relevant channels at time t_s
|
|
63
|
+
# how many pitches to shift each unit relative to registered pos at time t_s?
|
|
64
|
+
unregistered_depths_um = drift_util.invert_motion_estimate(
|
|
65
|
+
motion_est, t_s, registered_template_depths_um
|
|
66
|
+
)
|
|
67
|
+
# reverse arguments to pitch shifts since we are going the other direction
|
|
68
|
+
pitch_shifts = drift_util.get_spike_pitch_shifts(
|
|
69
|
+
depths_um=registered_template_depths_um,
|
|
70
|
+
geom=geom,
|
|
71
|
+
registered_depths_um=unregistered_depths_um,
|
|
72
|
+
)
|
|
73
|
+
# extract relevant channel neighborhoods, also by reversing args to a drift helper
|
|
74
|
+
unregistered_templates = drift_util.get_waveforms_on_static_channels(
|
|
75
|
+
registered_templates,
|
|
76
|
+
registered_geom,
|
|
77
|
+
n_pitches_shift=pitch_shifts,
|
|
78
|
+
registered_geom=geom,
|
|
79
|
+
target_kdtree=geom_kdtree,
|
|
80
|
+
match_distance=match_distance,
|
|
81
|
+
fill_value=fill_value,
|
|
82
|
+
return_padded=return_padded,
|
|
83
|
+
)
|
|
84
|
+
if return_pitch_shifts:
|
|
85
|
+
return pitch_shifts, unregistered_templates
|
|
86
|
+
return unregistered_templates
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def spatially_mask_templates(template_data, radius_um=0.0):
|
|
90
|
+
if not radius_um:
|
|
91
|
+
return template_data
|
|
92
|
+
|
|
93
|
+
tt = template_data.templates.copy()
|
|
94
|
+
ci = waveform_util.make_channel_index(template_data.registered_geom, radius_um)
|
|
95
|
+
chans = np.arange(ci.shape[0])
|
|
96
|
+
for j, t in enumerate(tt):
|
|
97
|
+
mask = ~np.isin(chans, ci[np.ptp(t, 0).argmax()])
|
|
98
|
+
tt[j, :, mask] = 0.0
|
|
99
|
+
|
|
100
|
+
return replace(template_data, templates=tt)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# -- template numerical processing
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass
|
|
107
|
+
class LowRankTemplates:
|
|
108
|
+
unit_ids: np.ndarray
|
|
109
|
+
temporal_components: np.ndarray
|
|
110
|
+
singular_values: np.ndarray
|
|
111
|
+
spatial_components: np.ndarray
|
|
112
|
+
spike_counts_by_channel: np.ndarray | None
|
|
113
|
+
|
|
114
|
+
def shift_to_best_channels(self, geom, registered_geom=None) -> "LowRankTemplates":
|
|
115
|
+
if registered_geom is None or registered_geom.shape == geom.shape:
|
|
116
|
+
return self
|
|
117
|
+
|
|
118
|
+
pitch = waveform_util.get_pitch(geom)
|
|
119
|
+
rgkdt = KDTree(registered_geom)
|
|
120
|
+
low = registered_geom[:, 1].min() - geom[:, 1].min()
|
|
121
|
+
high = registered_geom[:, 1].max() - geom[:, 1].max()
|
|
122
|
+
assert low <= 0 <= high
|
|
123
|
+
nlow = int(-low // pitch)
|
|
124
|
+
nhigh = int(high // pitch)
|
|
125
|
+
|
|
126
|
+
spatial_components = self.spatial_components
|
|
127
|
+
if torch.is_tensor(spatial_components):
|
|
128
|
+
spatial_components = spatial_components.numpy(force=True)
|
|
129
|
+
scores = np.full(len(self.spatial_components), -np.inf)
|
|
130
|
+
best_spatial = spatial_components[:, :, : len(geom)] + 0.0
|
|
131
|
+
|
|
132
|
+
for j in range(-nlow, nhigh + 1):
|
|
133
|
+
gshift = geom + [0, j * pitch]
|
|
134
|
+
d, regchans = rgkdt.query(gshift)
|
|
135
|
+
assert np.all(regchans < rgkdt.n)
|
|
136
|
+
|
|
137
|
+
spatial = spatial_components[:, :, regchans]
|
|
138
|
+
new_scores = np.square(spatial).sum((1, 2))
|
|
139
|
+
better = np.flatnonzero(new_scores > scores)
|
|
140
|
+
best_spatial[better] = spatial[better]
|
|
141
|
+
scores[better] = new_scores[better]
|
|
142
|
+
|
|
143
|
+
return replace(self, spatial_components=best_spatial)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def svd_compress_templates(
|
|
147
|
+
template_data,
|
|
148
|
+
min_channel_amplitude=0.0,
|
|
149
|
+
rank=5,
|
|
150
|
+
channel_sparse=True,
|
|
151
|
+
allow_na=False,
|
|
152
|
+
computation_cfg=None,
|
|
153
|
+
batch_size=32,
|
|
154
|
+
):
|
|
155
|
+
"""
|
|
156
|
+
Returns:
|
|
157
|
+
temporal_components: n_units, spike_length_samples, rank
|
|
158
|
+
singular_values: n_units, rank
|
|
159
|
+
spatial_components: n_units, rank, n_channels
|
|
160
|
+
"""
|
|
161
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
162
|
+
dev = computation_cfg.actual_device()
|
|
163
|
+
if hasattr(template_data, "templates"):
|
|
164
|
+
unit_ids = template_data.unit_ids
|
|
165
|
+
templates = template_data.templates
|
|
166
|
+
counts = template_data.spike_counts_by_channel
|
|
167
|
+
else:
|
|
168
|
+
templates = template_data
|
|
169
|
+
counts = None
|
|
170
|
+
unit_ids = np.arange(len(templates))
|
|
171
|
+
|
|
172
|
+
rank = min(rank, *templates.shape[1:])
|
|
173
|
+
|
|
174
|
+
amp_vecs = ptp(templates, dim=1, keepdims=True)
|
|
175
|
+
isna = np.isnan(amp_vecs)
|
|
176
|
+
if not allow_na:
|
|
177
|
+
assert not isna.any()
|
|
178
|
+
else:
|
|
179
|
+
amp_vecs = np.nan_to_num(amp_vecs, copy=False)
|
|
180
|
+
templates = np.nan_to_num(templates)
|
|
181
|
+
vis_mask = amp_vecs > min_channel_amplitude
|
|
182
|
+
dtype = templates.dtype
|
|
183
|
+
|
|
184
|
+
if not channel_sparse:
|
|
185
|
+
vis_templates = torch.as_tensor(templates * vis_mask)
|
|
186
|
+
U, S, Vh = _svd_helper(vis_templates)
|
|
187
|
+
# s is descending.
|
|
188
|
+
temporal_components = U[:, :, :rank].to(dtype).numpy(force=True)
|
|
189
|
+
singular_values = S[:, :rank].to(dtype).numpy(force=True)
|
|
190
|
+
spatial_components = Vh[:, :rank, :].to(dtype).numpy(force=True)
|
|
191
|
+
else:
|
|
192
|
+
# channel sparse: only SVD the nonzero channels
|
|
193
|
+
# this encodes the same exact subspace as above, and the reconstruction
|
|
194
|
+
# error is the same as above as a function of rank. it's just that
|
|
195
|
+
# we can zero out some spatial components, which is a useful property
|
|
196
|
+
# (used in pairwise convolutions for instance)
|
|
197
|
+
|
|
198
|
+
vis_mask = torch.as_tensor(vis_mask)
|
|
199
|
+
n, t, c = templates.shape
|
|
200
|
+
assert amp_vecs.shape == (n, 1, c)
|
|
201
|
+
templates = torch.as_tensor(templates)
|
|
202
|
+
uvec, groups = vis_mask[:, 0].unique(dim=0, return_inverse=True)
|
|
203
|
+
assert uvec.shape[1] == c
|
|
204
|
+
|
|
205
|
+
temporal_components = np.zeros((n, t, rank), dtype=dtype)
|
|
206
|
+
singular_values = np.zeros((n, rank), dtype=dtype)
|
|
207
|
+
spatial_components = np.zeros((n, rank, c), dtype=dtype)
|
|
208
|
+
|
|
209
|
+
for j in range(len(uvec)):
|
|
210
|
+
(in_j,) = (groups == j).nonzero(as_tuple=True)
|
|
211
|
+
nj = in_j.numel()
|
|
212
|
+
(mask,) = uvec[j].nonzero(as_tuple=True)
|
|
213
|
+
rankj = min(rank, mask.numel())
|
|
214
|
+
if not rankj:
|
|
215
|
+
continue
|
|
216
|
+
|
|
217
|
+
for bs in range(0, nj, batch_size):
|
|
218
|
+
be = min(bs + batch_size, nj)
|
|
219
|
+
binds = in_j[bs:be]
|
|
220
|
+
|
|
221
|
+
batch_x = templates[binds[:, None], :, mask[None, :]].to(dev)
|
|
222
|
+
# fancy ix comes to the front
|
|
223
|
+
assert batch_x.shape == (be - bs, mask.numel(), t)
|
|
224
|
+
U, S, Vh = _svd_helper(batch_x)
|
|
225
|
+
spatial_components[binds[:, None], :rankj, mask[None, :]] = U[
|
|
226
|
+
:, :, :rankj
|
|
227
|
+
].numpy(force=True)
|
|
228
|
+
singular_values[binds, :rankj] = S[:, :rankj].numpy(force=True)
|
|
229
|
+
assert np.isfinite(singular_values[binds, :rankj]).all()
|
|
230
|
+
temporal_components[binds, :, :rankj] = Vh[:, :rankj, :].mT.numpy(
|
|
231
|
+
force=True
|
|
232
|
+
)
|
|
233
|
+
assert np.isfinite(temporal_components[binds, :, :rankj]).all()
|
|
234
|
+
|
|
235
|
+
if logger.isEnabledFor(DARTSORTVERBOSE):
|
|
236
|
+
assert np.isfinite(temporal_components).all()
|
|
237
|
+
assert np.isfinite(singular_values).all()
|
|
238
|
+
assert np.isfinite(spatial_components).all()
|
|
239
|
+
recon = np.einsum(
|
|
240
|
+
"ntr,nr,nrc->ntc", temporal_components, singular_values, spatial_components
|
|
241
|
+
)
|
|
242
|
+
tp = templates.numpy(force=True)
|
|
243
|
+
err = np.square(recon - tp).sum(axis=(1, 2))
|
|
244
|
+
ss = np.square(tp).sum(axis=(1, 2))
|
|
245
|
+
r2 = 1.0 - err / ss
|
|
246
|
+
logger.dartsortverbose(
|
|
247
|
+
"Individual basis reconstructed templates with min, mean, max R^2s: "
|
|
248
|
+
f"{r2.min().item():0.3f}, {r2.mean().item():0.3f}, {r2.max().item():0.3f}."
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
if allow_na:
|
|
252
|
+
isna = np.broadcast_to(isna, spatial_components.shape)
|
|
253
|
+
spatial_components[isna] = np.nan
|
|
254
|
+
|
|
255
|
+
return LowRankTemplates(
|
|
256
|
+
unit_ids=unit_ids,
|
|
257
|
+
temporal_components=temporal_components,
|
|
258
|
+
singular_values=singular_values,
|
|
259
|
+
spatial_components=spatial_components,
|
|
260
|
+
spike_counts_by_channel=counts,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@dataclass
|
|
265
|
+
class SharedBasisTemplates:
|
|
266
|
+
unit_ids: np.ndarray
|
|
267
|
+
# rank, time
|
|
268
|
+
temporal_components: np.ndarray
|
|
269
|
+
# n, rank, chans
|
|
270
|
+
spatial_singular: np.ndarray
|
|
271
|
+
spike_counts_by_channel: np.ndarray | None
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def shared_basis_compress_templates(
|
|
275
|
+
template_data, min_channel_amplitude=1.0, rank=5, computation_cfg=None
|
|
276
|
+
):
|
|
277
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
278
|
+
dev = computation_cfg.actual_device()
|
|
279
|
+
if hasattr(template_data, "templates"):
|
|
280
|
+
unit_ids = template_data.unit_ids
|
|
281
|
+
templates = template_data.templates
|
|
282
|
+
counts = template_data.spike_counts_by_channel
|
|
283
|
+
else:
|
|
284
|
+
templates = template_data
|
|
285
|
+
counts = None
|
|
286
|
+
unit_ids = np.arange(len(templates))
|
|
287
|
+
n, t, c = templates.shape
|
|
288
|
+
rank = min(rank, t)
|
|
289
|
+
amp_vecs = ptp(templates, dim=1)
|
|
290
|
+
assert np.isfinite(amp_vecs).all()
|
|
291
|
+
visible = amp_vecs > min_channel_amplitude
|
|
292
|
+
uu, cc = np.nonzero(visible)
|
|
293
|
+
nvis = uu.shape[0]
|
|
294
|
+
|
|
295
|
+
# put time on the first axis
|
|
296
|
+
if nvis == visible.size:
|
|
297
|
+
to_compress = templates.transpose(1, 0, 2)
|
|
298
|
+
to_compress = to_compress.reshape(t, n * c)
|
|
299
|
+
else:
|
|
300
|
+
to_compress = templates[uu, :, cc]
|
|
301
|
+
assert to_compress.shape == (nvis, t)
|
|
302
|
+
to_compress = to_compress.T
|
|
303
|
+
|
|
304
|
+
# get the temporal basis
|
|
305
|
+
# we'll use cov->eigh here since we'd project onto U anyway
|
|
306
|
+
to_compress = torch.asarray(to_compress, device=dev, dtype=torch.double)
|
|
307
|
+
# U, S, Vh = _svd_helper(to_compress)
|
|
308
|
+
# assert U.shape == (t, min(t, nvis))
|
|
309
|
+
cov = torch.cov(to_compress, correction=0)
|
|
310
|
+
m = to_compress.mean(dim=1, keepdim=True)
|
|
311
|
+
cov += m * m.T
|
|
312
|
+
if nvis < t:
|
|
313
|
+
logger.warning(f"Had {nvis=} smaller than {t=} in shared basis compression.")
|
|
314
|
+
cov.diagonal().add_(1e-5)
|
|
315
|
+
vals, U = torch.linalg.eigh(cov)
|
|
316
|
+
temporal_comps = U[:, -rank:]
|
|
317
|
+
# to rank-major
|
|
318
|
+
temporal_comps = temporal_comps.T.contiguous()
|
|
319
|
+
temporal_comps = temporal_comps.numpy(force=True)
|
|
320
|
+
temporal_comps = temporal_comps[::-1]
|
|
321
|
+
temporal_comps = np.ascontiguousarray(temporal_comps, dtype=templates.dtype)
|
|
322
|
+
|
|
323
|
+
# project templates onto temporal comps (no sparsity here.)
|
|
324
|
+
# spatial_sing = np.einsum("ntc,rt->nrc", templates, temporal_comps)
|
|
325
|
+
templates_t = templates.transpose(0, 2, 1).reshape(n * c, t)
|
|
326
|
+
spatial_sing = templates_t @ (temporal_comps).T
|
|
327
|
+
spatial_sing = spatial_sing.reshape(n, c, rank).transpose(0, 2, 1)
|
|
328
|
+
spatial_sing = np.ascontiguousarray(spatial_sing)
|
|
329
|
+
|
|
330
|
+
assert np.isfinite(temporal_comps).all()
|
|
331
|
+
assert np.isfinite(spatial_sing).all()
|
|
332
|
+
|
|
333
|
+
if logger.isEnabledFor(DARTSORTVERBOSE):
|
|
334
|
+
recon = torch.asarray(np.einsum("rt,nrc->ntc", temporal_comps, spatial_sing))
|
|
335
|
+
templates = torch.asarray(templates)
|
|
336
|
+
err = recon.sub_(templates).square_().sum(dim=(1, 2)).cpu()
|
|
337
|
+
ss = templates.square().sum(dim=(1, 2)).cpu()
|
|
338
|
+
r2 = 1.0 - err / ss
|
|
339
|
+
logger.dartsortverbose(
|
|
340
|
+
"Shared basis reconstructed templates with min, mean, max R^2s: "
|
|
341
|
+
f"{r2.min().item():0.3f}, {r2.mean().item():0.3f}, {r2.max().item():0.3f}."
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
return SharedBasisTemplates(
|
|
345
|
+
unit_ids=unit_ids,
|
|
346
|
+
temporal_components=temporal_comps,
|
|
347
|
+
spatial_singular=spatial_sing,
|
|
348
|
+
spike_counts_by_channel=counts,
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def temporally_upsample_templates(
|
|
353
|
+
templates, temporal_upsampling_factor=8, kind="cubic"
|
|
354
|
+
):
|
|
355
|
+
"""Note, also works on temporal components thanks to compatible shape."""
|
|
356
|
+
n, t, c = templates.shape
|
|
357
|
+
tp = np.arange(t).astype(float)
|
|
358
|
+
erp = interp1d(tp, templates, axis=1, bounds_error=True, kind=kind)
|
|
359
|
+
tup = np.arange(t, step=1.0 / temporal_upsampling_factor) # type: ignore[reportCallIssue]
|
|
360
|
+
tup.clip(0, t - 1, out=tup)
|
|
361
|
+
upsampled_templates = erp(tup)
|
|
362
|
+
upsampled_templates = upsampled_templates.reshape(
|
|
363
|
+
n, t, temporal_upsampling_factor, c
|
|
364
|
+
)
|
|
365
|
+
upsampled_templates = upsampled_templates.astype(templates.dtype)
|
|
366
|
+
return upsampled_templates
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@dataclass
|
|
370
|
+
class CompressedUpsampledTemplates:
|
|
371
|
+
n_compressed_upsampled_templates: int
|
|
372
|
+
compressed_upsampled_templates: np.ndarray
|
|
373
|
+
compressed_upsampling_map: np.ndarray
|
|
374
|
+
compressed_upsampling_index: np.ndarray
|
|
375
|
+
compressed_index_to_template_index: np.ndarray
|
|
376
|
+
compressed_index_to_upsampling_index: np.ndarray
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def default_n_upsamples_map(ptps, max_upsample=8):
|
|
380
|
+
# avoid overflow in 4 ** by trimming ptp range in advance
|
|
381
|
+
max_ptp = 1 + 2 * math.log(max_upsample, 4)
|
|
382
|
+
ptps = np.minimum(ptps, max_ptp)
|
|
383
|
+
upsamples = 4 ** (ptps // 2)
|
|
384
|
+
upsamples = upsamples.astype(int)
|
|
385
|
+
np.clip(upsamples, 1, max_upsample, out=upsamples)
|
|
386
|
+
return upsamples
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def compressed_upsampled_templates(
|
|
390
|
+
templates,
|
|
391
|
+
*,
|
|
392
|
+
ptps=None,
|
|
393
|
+
max_upsample=8,
|
|
394
|
+
n_upsamples_map=default_n_upsamples_map,
|
|
395
|
+
kind="cubic",
|
|
396
|
+
):
|
|
397
|
+
"""compressedly store fewer temporally upsampled copies of lower amplitude templates
|
|
398
|
+
|
|
399
|
+
Returns
|
|
400
|
+
-------
|
|
401
|
+
A CompressedUpsampledTemplates object with fields:
|
|
402
|
+
compressed_upsampled_templates : array (n_compressed_upsampled_templates, spike_length_samples, n_channels)
|
|
403
|
+
compressed_upsampling_map : array (n_templates, max_upsample)
|
|
404
|
+
compressed_upsampled_templates[compressed_upsampling_map[unit, j]] is an approximation
|
|
405
|
+
of the jth upsampled template for this unit. for low-amplitude units,
|
|
406
|
+
compressed_upsampling_map[unit] will have fewer unique entries, corresponding
|
|
407
|
+
to fewer saved upsampled copies for that unit.
|
|
408
|
+
compressed_upsampling_index : array (n_templates, max_upsample)
|
|
409
|
+
A n_compressed_upsampled_templates-padded ragged array mapping each
|
|
410
|
+
template index to its compressed upsampled indices
|
|
411
|
+
compressed_index_to_template_index
|
|
412
|
+
compressed_index_to_upsampling_index
|
|
413
|
+
"""
|
|
414
|
+
n_templates = templates.shape[0]
|
|
415
|
+
assert templates.ndim == 3
|
|
416
|
+
if max_upsample == 1:
|
|
417
|
+
return CompressedUpsampledTemplates(
|
|
418
|
+
n_templates,
|
|
419
|
+
templates,
|
|
420
|
+
np.arange(n_templates)[:, None],
|
|
421
|
+
np.arange(n_templates)[:, None],
|
|
422
|
+
np.arange(n_templates),
|
|
423
|
+
np.zeros(n_templates, dtype=np.int64),
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
# how many copies should each unit get?
|
|
427
|
+
# sometimes users may pass temporal SVD components in instead of templates,
|
|
428
|
+
# so we allow them to pass in the amplitudes of the actual templates
|
|
429
|
+
if ptps is None:
|
|
430
|
+
ptps = np.ptp(templates, 1).max(1)
|
|
431
|
+
assert ptps.shape == (n_templates,)
|
|
432
|
+
|
|
433
|
+
if n_upsamples_map == "yass":
|
|
434
|
+
n_upsamples_map = default_n_upsamples_map
|
|
435
|
+
elif n_upsamples_map == "none":
|
|
436
|
+
n_upsamples_map = None
|
|
437
|
+
|
|
438
|
+
if n_upsamples_map is None:
|
|
439
|
+
n_upsamples = np.full(n_templates, max_upsample)
|
|
440
|
+
else:
|
|
441
|
+
n_upsamples = n_upsamples_map(ptps, max_upsample=max_upsample)
|
|
442
|
+
|
|
443
|
+
# build the compressed upsampling map
|
|
444
|
+
compressed_upsampling_map = np.full((n_templates, max_upsample), -1, dtype=np.int64)
|
|
445
|
+
compressed_upsampling_index = np.full(
|
|
446
|
+
(n_templates, max_upsample), -1, dtype=np.int64
|
|
447
|
+
)
|
|
448
|
+
template_indices = []
|
|
449
|
+
upsampling_indices = []
|
|
450
|
+
current_compressed_index = 0
|
|
451
|
+
for i, nup in enumerate(n_upsamples):
|
|
452
|
+
compression = max_upsample // nup
|
|
453
|
+
nup = max_upsample // compression # handle divisibility failure
|
|
454
|
+
|
|
455
|
+
# new compressed indices
|
|
456
|
+
compressed_upsampling_map[i] = current_compressed_index + np.arange(nup).repeat(
|
|
457
|
+
compression
|
|
458
|
+
)
|
|
459
|
+
compressed_upsampling_index[i, :nup] = current_compressed_index + np.arange(nup)
|
|
460
|
+
current_compressed_index += nup
|
|
461
|
+
|
|
462
|
+
# indices of the templates to keep in the full array of upsampled templates
|
|
463
|
+
template_indices.extend([i] * nup)
|
|
464
|
+
upsampling_indices.extend(compression * np.arange(nup))
|
|
465
|
+
assert (compressed_upsampling_map >= 0).all()
|
|
466
|
+
assert (
|
|
467
|
+
np.unique(compressed_upsampling_map).size
|
|
468
|
+
== (compressed_upsampling_index >= 0).sum()
|
|
469
|
+
== compressed_upsampling_map.max() + 1
|
|
470
|
+
== compressed_upsampling_index.max() + 1
|
|
471
|
+
== current_compressed_index
|
|
472
|
+
)
|
|
473
|
+
template_indices = np.array(template_indices)
|
|
474
|
+
upsampling_indices = np.array(upsampling_indices)
|
|
475
|
+
compressed_upsampling_index[compressed_upsampling_index < 0] = (
|
|
476
|
+
current_compressed_index
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
# get the upsampled templates
|
|
480
|
+
all_upsampled_templates = temporally_upsample_templates(
|
|
481
|
+
templates, temporal_upsampling_factor=max_upsample, kind=kind
|
|
482
|
+
)
|
|
483
|
+
# n, up, t, c
|
|
484
|
+
all_upsampled_templates = all_upsampled_templates.transpose(0, 2, 1, 3)
|
|
485
|
+
rix = np.ravel_multi_index(
|
|
486
|
+
(template_indices, upsampling_indices), all_upsampled_templates.shape[:2]
|
|
487
|
+
)
|
|
488
|
+
all_upsampled_templates = all_upsampled_templates.reshape(
|
|
489
|
+
n_templates * max_upsample, templates.shape[1], templates.shape[2]
|
|
490
|
+
)
|
|
491
|
+
compressed_upsampled_templates = all_upsampled_templates[rix]
|
|
492
|
+
|
|
493
|
+
return CompressedUpsampledTemplates(
|
|
494
|
+
current_compressed_index,
|
|
495
|
+
compressed_upsampled_templates,
|
|
496
|
+
compressed_upsampling_map,
|
|
497
|
+
compressed_upsampling_index,
|
|
498
|
+
template_indices,
|
|
499
|
+
upsampling_indices,
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _svd_helper(x, svd_dtype=torch.double):
|
|
504
|
+
"""This matches numpy's behavior in tests/test_matching.py."""
|
|
505
|
+
orig_dtype = x.dtype
|
|
506
|
+
x = x.to(dtype=svd_dtype)
|
|
507
|
+
if x.device.type == "cuda":
|
|
508
|
+
U, S, Vh = torch.linalg.svd(x, full_matrices=False, driver="gesvd")
|
|
509
|
+
U = U.to(dtype=orig_dtype)
|
|
510
|
+
S = S.to(dtype=orig_dtype)
|
|
511
|
+
Vh = Vh.to(dtype=orig_dtype)
|
|
512
|
+
return U, S, Vh
|
|
513
|
+
else:
|
|
514
|
+
U, S, Vh = np.linalg.svd(x.numpy(), full_matrices=False)
|
|
515
|
+
U = torch.asarray(U, dtype=orig_dtype)
|
|
516
|
+
S = torch.asarray(S, dtype=orig_dtype)
|
|
517
|
+
Vh = torch.asarray(Vh, dtype=orig_dtype)
|
|
518
|
+
return U, S, Vh
|