DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1461 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
5
|
+
import linear_operator
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import torch
|
|
9
|
+
import torch.nn.functional as F
|
|
10
|
+
from linear_operator import operators
|
|
11
|
+
from scipy.fftpack import next_fast_len
|
|
12
|
+
from scipy.spatial.distance import pdist, squareform
|
|
13
|
+
from scipy.stats import norm
|
|
14
|
+
from sklearn.covariance import GraphicalLassoCV, graphical_lasso
|
|
15
|
+
from tqdm.auto import trange
|
|
16
|
+
|
|
17
|
+
from ..util import more_operators, spiketorch
|
|
18
|
+
from ..util.interpolation_util import InterpolationParams, default_interpolation_params
|
|
19
|
+
from ..util.logging_util import DARTSORTDEBUG, get_logger
|
|
20
|
+
|
|
21
|
+
logger = get_logger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class FullNoise(torch.nn.Module):
|
|
25
|
+
"""Do not use this, it's just for comparison to the others."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, std, vt, nt, nc):
|
|
28
|
+
super().__init__()
|
|
29
|
+
self.std = std
|
|
30
|
+
self.vt = vt
|
|
31
|
+
self.nt = nt
|
|
32
|
+
self.nc = nc
|
|
33
|
+
|
|
34
|
+
def full_covariance(self):
|
|
35
|
+
return (self.vt.T * self.std.square()) @ self.vt
|
|
36
|
+
|
|
37
|
+
def simulate(self, size=1, t=None, generator=None):
|
|
38
|
+
device = cast(torch.device, self.spatial_std.device)
|
|
39
|
+
assert t == self.nt
|
|
40
|
+
samples = torch.randn(size, t * self.nc, generator=generator, device=device)
|
|
41
|
+
samples = torch.einsum(
|
|
42
|
+
"ni,i,ij->nj",
|
|
43
|
+
samples,
|
|
44
|
+
self.std,
|
|
45
|
+
self.vt,
|
|
46
|
+
)
|
|
47
|
+
return samples.view(size, self.nt, self.nc)
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def estimate(cls, snippets):
|
|
51
|
+
snippets = torch.asarray(snippets)
|
|
52
|
+
snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
|
|
53
|
+
|
|
54
|
+
n, t, c = snippets.shape
|
|
55
|
+
|
|
56
|
+
x = snippets.view(n, t * c)
|
|
57
|
+
# u_full, sing, vt_full = torch.linalg.svd(x, full_matrices=False)
|
|
58
|
+
# std = sing / torch.sqrt(torch.tensor(n - 1).to(snippets))
|
|
59
|
+
|
|
60
|
+
cov = torch.cov(x.T)
|
|
61
|
+
eigvals, v = torch.linalg.eigh(cov)
|
|
62
|
+
std = eigvals.sqrt()
|
|
63
|
+
vt = v.T.contiguous()
|
|
64
|
+
|
|
65
|
+
return cls(std, vt, nt=t, nc=c)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class FactorizedNoise(torch.nn.Module):
|
|
69
|
+
"""Spatial/temporal factorized noise. See .estimate()."""
|
|
70
|
+
|
|
71
|
+
def __init__(self, spatial_std, vt_spatial, temporal_std, vt_temporal):
|
|
72
|
+
super().__init__()
|
|
73
|
+
self.spatial_std = spatial_std
|
|
74
|
+
self.vt_spatial = vt_spatial
|
|
75
|
+
self.temporal_std = temporal_std
|
|
76
|
+
self.vt_temporal = vt_temporal
|
|
77
|
+
|
|
78
|
+
def full_covariance(self):
|
|
79
|
+
spatial_cov = self.spatial_std[:, None] * self.vt_spatial
|
|
80
|
+
spatial_cov = spatial_cov.T @ spatial_cov
|
|
81
|
+
temporal_cov = self.temporal_std[:, None] * self.vt_temporal
|
|
82
|
+
temporal_cov = temporal_cov.T @ temporal_cov
|
|
83
|
+
return torch.kron(temporal_cov, spatial_cov)
|
|
84
|
+
|
|
85
|
+
def simulate(self, size=1, t=None, generator=None):
|
|
86
|
+
c = self.spatial_std.numel()
|
|
87
|
+
t_ = self.temporal_std.numel()
|
|
88
|
+
device = self.spatial_std.device
|
|
89
|
+
if t is None:
|
|
90
|
+
t = t_
|
|
91
|
+
assert t == t_
|
|
92
|
+
samples = torch.randn(size, t, c, generator=generator, device=device)
|
|
93
|
+
return torch.einsum(
|
|
94
|
+
"ntc,t,tu,c,cd->nud",
|
|
95
|
+
samples,
|
|
96
|
+
self.temporal_std,
|
|
97
|
+
self.vt_temporal,
|
|
98
|
+
self.spatial_std,
|
|
99
|
+
self.vt_spatial,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
@classmethod
|
|
103
|
+
def estimate(cls, snippets):
|
|
104
|
+
"""Estimate factorized spatiotemporal noise covariance
|
|
105
|
+
|
|
106
|
+
If snippets has shape (n, t, c), the full t*c x t*c covariance is
|
|
107
|
+
too big. Let's say it factorizes as
|
|
108
|
+
spatial_cov[c, c'] * temporal_cov[t, t'].
|
|
109
|
+
This is not identifiable, but it's okay if we assume that the global
|
|
110
|
+
scale falls into the spatial part (say). So, we first estimate the
|
|
111
|
+
spatial cov, then apply spatial whitening to the snippets, and finally
|
|
112
|
+
estimate the temporal cov.
|
|
113
|
+
|
|
114
|
+
This is an SVD reimplementation of a Yass fn which used covariance matrices.
|
|
115
|
+
|
|
116
|
+
Arguments
|
|
117
|
+
---------
|
|
118
|
+
snippets : torch.Tensor
|
|
119
|
+
(n, t, c) array of noise snippets. If you don't want spikes in there,
|
|
120
|
+
this function won't help you with that.
|
|
121
|
+
"""
|
|
122
|
+
snippets = torch.asarray(snippets)
|
|
123
|
+
snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
|
|
124
|
+
|
|
125
|
+
n, t, c = snippets.shape
|
|
126
|
+
sqrt_nt_minus_1 = torch.tensor(n * t - 1, dtype=snippets.dtype).sqrt()
|
|
127
|
+
sqrt_nc_minus_1 = torch.tensor(n * c - 1, dtype=snippets.dtype).sqrt()
|
|
128
|
+
assert n * t > c**2
|
|
129
|
+
assert n * c > t**2
|
|
130
|
+
|
|
131
|
+
# estimate spatial covariance
|
|
132
|
+
x_spatial = snippets.view(n * t, c)
|
|
133
|
+
u_spatial, spatial_sing, vt_spatial = torch.linalg.svd(
|
|
134
|
+
x_spatial, full_matrices=False
|
|
135
|
+
)
|
|
136
|
+
spatial_std = spatial_sing / sqrt_nt_minus_1
|
|
137
|
+
|
|
138
|
+
# extract whitened temporal snips
|
|
139
|
+
x_temporal = u_spatial.view(n, t, c).permute(0, 2, 1).reshape(n * c, t)
|
|
140
|
+
x_temporal.mul_(sqrt_nt_minus_1)
|
|
141
|
+
_, temporal_sing, vt_temporal = torch.linalg.svd(
|
|
142
|
+
x_temporal, full_matrices=False
|
|
143
|
+
)
|
|
144
|
+
del _
|
|
145
|
+
temporal_std = temporal_sing / sqrt_nc_minus_1
|
|
146
|
+
|
|
147
|
+
return cls(spatial_std, vt_spatial, temporal_std, vt_temporal)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class WhiteNoise(torch.nn.Module):
|
|
151
|
+
"""White noise to mimic the StationaryFactorizedNoise for use in sims."""
|
|
152
|
+
|
|
153
|
+
margin = 0
|
|
154
|
+
|
|
155
|
+
def __init__(self, n_channels, scale=1.0):
|
|
156
|
+
super().__init__()
|
|
157
|
+
self.n_channels = n_channels
|
|
158
|
+
self.scale = scale
|
|
159
|
+
|
|
160
|
+
def unwhiten(self, snippet):
|
|
161
|
+
return snippet
|
|
162
|
+
|
|
163
|
+
def simulate(self, size=1, t=None, generator=None, chunk_t=None):
|
|
164
|
+
assert t is not None
|
|
165
|
+
x = torch.randn(size, t, self.n_channels, generator=generator)
|
|
166
|
+
if self.scale != 1.0:
|
|
167
|
+
x *= self.scale
|
|
168
|
+
return x
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class StationaryFactorizedNoise(torch.nn.Module):
|
|
172
|
+
def __init__(self, spatial_std, vt_spatial, kernel_fft, block_size, t):
|
|
173
|
+
super().__init__()
|
|
174
|
+
self.spatial_std = torch.asarray(spatial_std)
|
|
175
|
+
self.vt_spatial = torch.asarray(vt_spatial)
|
|
176
|
+
self.kernel_fft = torch.asarray(kernel_fft)
|
|
177
|
+
self.block_size = block_size
|
|
178
|
+
self.t = t
|
|
179
|
+
self.margin = (self.t - 1) // 2
|
|
180
|
+
|
|
181
|
+
def spatial_cov(self):
|
|
182
|
+
rt = self.spatial_std * self.vt_spatial.T
|
|
183
|
+
return rt @ rt.T
|
|
184
|
+
|
|
185
|
+
def whiten(self, snippet):
|
|
186
|
+
wsnip = snippet @ (self.vt_spatial / self.spatial_std[:, None])
|
|
187
|
+
wsnip = spiketorch.single_inv_oaconv1d(
|
|
188
|
+
wsnip.T,
|
|
189
|
+
s2=self.t,
|
|
190
|
+
f2=1 / self.kernel_fft,
|
|
191
|
+
block_size=self.block_size,
|
|
192
|
+
norm="ortho",
|
|
193
|
+
)
|
|
194
|
+
# zca
|
|
195
|
+
wsnip = wsnip.T @ self.vt_spatial.T
|
|
196
|
+
return wsnip
|
|
197
|
+
|
|
198
|
+
def unwhiten(self, snippet):
|
|
199
|
+
snippet = torch.asarray(
|
|
200
|
+
snippet, device=self.spatial_std.device, dtype=self.spatial_std.dtype
|
|
201
|
+
)
|
|
202
|
+
flat = snippet.ndim == 2
|
|
203
|
+
if flat:
|
|
204
|
+
snippet = snippet[None]
|
|
205
|
+
size, t_padded, c = snippet.shape
|
|
206
|
+
t = t_padded - self.t + 1
|
|
207
|
+
out = snippet.new_empty(size, c, t)
|
|
208
|
+
for j in range(size):
|
|
209
|
+
out[j] = spiketorch.single_inv_oaconv1d(
|
|
210
|
+
snippet[j].T,
|
|
211
|
+
s2=self.t,
|
|
212
|
+
f2=self.kernel_fft,
|
|
213
|
+
block_size=self.block_size,
|
|
214
|
+
norm="ortho",
|
|
215
|
+
)
|
|
216
|
+
spatial_part = self.spatial_std[:, None] * self.vt_spatial
|
|
217
|
+
out = torch.einsum("nct,cd->ntd", out, spatial_part)
|
|
218
|
+
if flat:
|
|
219
|
+
out = out[0]
|
|
220
|
+
return out
|
|
221
|
+
|
|
222
|
+
def simulate(self, size=1, t=None, generator=None, chunk_t=None):
|
|
223
|
+
"""Simulate stationary factorized noise
|
|
224
|
+
|
|
225
|
+
White noise is the same in FFT space or not. So we start there,
|
|
226
|
+
filter, inverse FFT, then spatially dewhiten. The filtering + inverse
|
|
227
|
+
FFT are basically overlap-add convolution, and thanks to Chris Langfield
|
|
228
|
+
for help with overlap-add.
|
|
229
|
+
"""
|
|
230
|
+
if t is None:
|
|
231
|
+
t = self.t
|
|
232
|
+
assert t >= self.t
|
|
233
|
+
c = self.spatial_std.numel()
|
|
234
|
+
device = self.spatial_std.device
|
|
235
|
+
|
|
236
|
+
if chunk_t:
|
|
237
|
+
out = torch.zeros((t + self.t, c))
|
|
238
|
+
taper = torch.linspace(0.0, 1.0, steps=self.t)
|
|
239
|
+
for bs in trange(0, t, chunk_t):
|
|
240
|
+
chunk = self.simulate(t=chunk_t + self.t, generator=generator)[0]
|
|
241
|
+
if bs > 0:
|
|
242
|
+
chunk[: self.t] *= taper[:, None]
|
|
243
|
+
chunk[-self.t :] *= (1 - taper)[:, None]
|
|
244
|
+
out[bs : bs + chunk_t + self.t] += chunk.cpu()
|
|
245
|
+
return out[None, : -self.t]
|
|
246
|
+
|
|
247
|
+
# need extra room at the edges to do valid convolution
|
|
248
|
+
t_padded = t + self.t - 1
|
|
249
|
+
noise = torch.randn(size, t_padded, c, generator=generator, device=device)
|
|
250
|
+
return self.unwhiten(noise)
|
|
251
|
+
|
|
252
|
+
@classmethod
|
|
253
|
+
def estimate(cls, snippets):
|
|
254
|
+
"""Estimate factorized temporally stationary noise
|
|
255
|
+
|
|
256
|
+
When simulating long samples, the above approach can't scale. Here let
|
|
257
|
+
spatial_cov[c, c'] * temporal_cov[t - t'].
|
|
258
|
+
|
|
259
|
+
Now, remember that stationary kernels diagonalize in Fourier basis. FFT
|
|
260
|
+
is the eigh of circulant things. So we use an FFT based version of the
|
|
261
|
+
SVD algorithm above.
|
|
262
|
+
|
|
263
|
+
Arguments
|
|
264
|
+
---------
|
|
265
|
+
snippets : torch.Tensor
|
|
266
|
+
(n, t, c) array of noise snippets. If you don't want spikes in there,
|
|
267
|
+
this function won't help you with that.
|
|
268
|
+
"""
|
|
269
|
+
snippets = torch.asarray(snippets)
|
|
270
|
+
snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
|
|
271
|
+
|
|
272
|
+
n, t, c = snippets.shape
|
|
273
|
+
sqrt_nt_minus_1 = torch.tensor(n * t - 1, dtype=snippets.dtype).sqrt()
|
|
274
|
+
assert n * t > c**2
|
|
275
|
+
assert n * c > t
|
|
276
|
+
|
|
277
|
+
# estimate spatial covariance
|
|
278
|
+
x_spatial = snippets.view(n * t, c)
|
|
279
|
+
u_spatial, spatial_sing, vt_spatial = torch.linalg.svd(
|
|
280
|
+
x_spatial, full_matrices=False
|
|
281
|
+
)
|
|
282
|
+
spatial_std = spatial_sing / sqrt_nt_minus_1
|
|
283
|
+
|
|
284
|
+
# extract whitened temporal snips
|
|
285
|
+
x_temporal = u_spatial.view(n, t, c).permute(0, 2, 1).reshape(n * c, t)
|
|
286
|
+
x_temporal.mul_(sqrt_nt_minus_1)
|
|
287
|
+
|
|
288
|
+
# estimate stationary temporal covariance via FFT
|
|
289
|
+
block_size = next_fast_len(t)
|
|
290
|
+
xt_fft = torch.fft.rfft(x_temporal, n=block_size, norm="ortho")
|
|
291
|
+
kernel_fft = (xt_fft * xt_fft.conj()).mean(0).sqrt_()
|
|
292
|
+
|
|
293
|
+
return cls(spatial_std, vt_spatial, kernel_fft, block_size, t)
|
|
294
|
+
|
|
295
|
+
def unit_false_positives(
|
|
296
|
+
self,
|
|
297
|
+
low_rank_templates,
|
|
298
|
+
min_threshold=5.0,
|
|
299
|
+
radius=10,
|
|
300
|
+
generator=None,
|
|
301
|
+
size=100,
|
|
302
|
+
t=4096,
|
|
303
|
+
):
|
|
304
|
+
"""Run template detection in noise residual to check for false positives
|
|
305
|
+
|
|
306
|
+
Given a collection of (SVD factorized) templates, compute their template
|
|
307
|
+
matching objective in simulated recording snippets drawn according to
|
|
308
|
+
this noise object's distribution. Then, detect peaks (with a refractory
|
|
309
|
+
condition) to estimate the false positive count.
|
|
310
|
+
|
|
311
|
+
Arguments
|
|
312
|
+
---------
|
|
313
|
+
low_rank_templates : LowRankTemplates
|
|
314
|
+
min_threshold : float
|
|
315
|
+
The bare minimum deconv objective
|
|
316
|
+
radius : int
|
|
317
|
+
Refractory period
|
|
318
|
+
generator : torch.Generator
|
|
319
|
+
For reproducibility.
|
|
320
|
+
size : int
|
|
321
|
+
Number of recording snipppets
|
|
322
|
+
t : int
|
|
323
|
+
Length of each recording snippet
|
|
324
|
+
|
|
325
|
+
Returns
|
|
326
|
+
-------
|
|
327
|
+
total_samples : int
|
|
328
|
+
Total number of valid temporal samples
|
|
329
|
+
fp_dataframe : pd.DataFrame
|
|
330
|
+
With columns `units` and `scores`
|
|
331
|
+
"""
|
|
332
|
+
from ..detect import detect_and_deduplicate
|
|
333
|
+
|
|
334
|
+
singular = torch.asarray(
|
|
335
|
+
low_rank_templates.singular_values,
|
|
336
|
+
device=self.spatial_std.device,
|
|
337
|
+
dtype=self.spatial_std.dtype,
|
|
338
|
+
)
|
|
339
|
+
spatial = torch.asarray(
|
|
340
|
+
low_rank_templates.spatial_components,
|
|
341
|
+
device=self.spatial_std.device,
|
|
342
|
+
dtype=self.spatial_std.dtype,
|
|
343
|
+
)
|
|
344
|
+
spatial_singular = singular.unsqueeze(-1) * spatial
|
|
345
|
+
temporal = torch.asarray(
|
|
346
|
+
low_rank_templates.temporal_components,
|
|
347
|
+
device=self.spatial_std.device,
|
|
348
|
+
dtype=self.spatial_std.dtype,
|
|
349
|
+
)
|
|
350
|
+
normsq = singular.square().sum(1)
|
|
351
|
+
negnormsq = normsq.neg().unsqueeze(1)
|
|
352
|
+
nu, nt = temporal.shape[:2]
|
|
353
|
+
obj = None # var for reusing buffers
|
|
354
|
+
units = []
|
|
355
|
+
scores = []
|
|
356
|
+
for j in trange(size, desc="False positives"):
|
|
357
|
+
# note: simulating with padding so that the valid conv has length t.
|
|
358
|
+
sample = self.simulate(t=t + nt - 1, generator=generator)[0].T
|
|
359
|
+
assert sample.isfinite().all()
|
|
360
|
+
obj = spiketorch.convolve_lowrank(
|
|
361
|
+
sample,
|
|
362
|
+
spatial_singular,
|
|
363
|
+
temporal,
|
|
364
|
+
out=obj,
|
|
365
|
+
padding=0,
|
|
366
|
+
)
|
|
367
|
+
assert obj.isfinite().all()
|
|
368
|
+
assert obj.shape == (nu, t)
|
|
369
|
+
obj = torch.add(negnormsq, obj, alpha=2.0, out=obj)
|
|
370
|
+
assert obj.isfinite().all()
|
|
371
|
+
|
|
372
|
+
# find peaks...
|
|
373
|
+
peak_times, peak_units, peak_energies = detect_and_deduplicate( # type: ignore
|
|
374
|
+
obj.T,
|
|
375
|
+
min_threshold,
|
|
376
|
+
peak_sign="pos",
|
|
377
|
+
relative_peak_radius=1,
|
|
378
|
+
dedup_temporal_radius=radius,
|
|
379
|
+
return_energies=True,
|
|
380
|
+
)
|
|
381
|
+
assert peak_energies.isfinite().all()
|
|
382
|
+
units.append(peak_units.numpy(force=True))
|
|
383
|
+
scores.append(peak_energies.numpy(force=True))
|
|
384
|
+
|
|
385
|
+
total_samples = size * t
|
|
386
|
+
data = dict(units=np.concatenate(units), scores=np.concatenate(scores))
|
|
387
|
+
fp_dataframe = pd.DataFrame(data)
|
|
388
|
+
|
|
389
|
+
return dict(
|
|
390
|
+
total_samples=total_samples,
|
|
391
|
+
fp_dataframe=fp_dataframe,
|
|
392
|
+
normsq=normsq.numpy(force=True),
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
class EmbeddedNoise(torch.nn.Module):
|
|
397
|
+
"""Handles computations related to noise in TPCA space.
|
|
398
|
+
|
|
399
|
+
Can have a couple of kinds of mean. mean_kind == ...
|
|
400
|
+
- "zero": noise was already centered, my mean is 0
|
|
401
|
+
- "by_rank": same mean on all channels
|
|
402
|
+
- "full": value per rank, chan
|
|
403
|
+
|
|
404
|
+
Default is zero, because we usually do not do "centering" in our TPCA
|
|
405
|
+
(i.e., it is just linear not affine). Since the channels are highpassed,
|
|
406
|
+
they have mean 0, and still do in TPCA space.
|
|
407
|
+
|
|
408
|
+
And cov_kind = ...
|
|
409
|
+
- "scalar": one global variance
|
|
410
|
+
- "diagonal_by_rank": same variance across chans, varies by rank
|
|
411
|
+
- "diagonal": value per rank, chan
|
|
412
|
+
- "factorized": kronecker prod of dense rank and chan factors
|
|
413
|
+
- "factorized_by_rank": same, but chan factor varies by rank
|
|
414
|
+
- "factorized_rank_diag" : factorized, but rank factor is diagonal
|
|
415
|
+
- "factorized_by_rank_rank_diag" : factorized_by_rank, but rank factor is diagonal
|
|
416
|
+
this one is block diagonal and therefore nicer than factorized_by_rank.
|
|
417
|
+
- "full": not generally practical.
|
|
418
|
+
"""
|
|
419
|
+
|
|
420
|
+
def __init__(
|
|
421
|
+
self,
|
|
422
|
+
rank,
|
|
423
|
+
n_channels,
|
|
424
|
+
mean_kind="zero",
|
|
425
|
+
cov_kind="factorized_by_rank_rank_diag",
|
|
426
|
+
mean=None,
|
|
427
|
+
global_std=None,
|
|
428
|
+
rank_std=None,
|
|
429
|
+
full_std=None,
|
|
430
|
+
rank_vt=None,
|
|
431
|
+
channel_std=None,
|
|
432
|
+
channel_vt=None,
|
|
433
|
+
full_cov=None,
|
|
434
|
+
zero_radius=None,
|
|
435
|
+
):
|
|
436
|
+
super().__init__()
|
|
437
|
+
self.rank = rank
|
|
438
|
+
self.n_channels = n_channels
|
|
439
|
+
self.mean_kind = mean_kind
|
|
440
|
+
self.cov_kind = cov_kind
|
|
441
|
+
self.D = rank * n_channels
|
|
442
|
+
self.zero_radius = zero_radius
|
|
443
|
+
|
|
444
|
+
self.register_buffer("chans_arange", torch.arange(n_channels))
|
|
445
|
+
self.register_buffer("global_std", global_std)
|
|
446
|
+
|
|
447
|
+
device = None
|
|
448
|
+
if global_std is not None:
|
|
449
|
+
device = global_std.device
|
|
450
|
+
|
|
451
|
+
if mean is not None:
|
|
452
|
+
self.register_buffer("mean", mean)
|
|
453
|
+
if rank_std is not None:
|
|
454
|
+
self.register_buffer("rank_std", rank_std)
|
|
455
|
+
if channel_std is not None:
|
|
456
|
+
self.register_buffer("channel_std", channel_std)
|
|
457
|
+
if full_std is not None:
|
|
458
|
+
self.register_buffer("full_std", full_std)
|
|
459
|
+
if rank_vt is not None:
|
|
460
|
+
self.register_buffer("rank_vt", rank_vt)
|
|
461
|
+
if channel_vt is not None:
|
|
462
|
+
self.register_buffer("channel_vt", channel_vt)
|
|
463
|
+
self.register_buffer("channel_vt_zpad", F.pad(channel_vt, (0, 1)))
|
|
464
|
+
if full_cov is not None:
|
|
465
|
+
self.register_buffer("full_cov", full_cov)
|
|
466
|
+
if device is None:
|
|
467
|
+
device = full_cov.device
|
|
468
|
+
|
|
469
|
+
# precompute stuff
|
|
470
|
+
self._full_cov = None
|
|
471
|
+
self._full_covinvcov = None
|
|
472
|
+
self._full_inverse = None
|
|
473
|
+
self._full_whitener = None
|
|
474
|
+
self._logdet = None
|
|
475
|
+
self.register_buffer("mean_full", self.mean_rc().clone().detach())
|
|
476
|
+
self.cache = {}
|
|
477
|
+
self.to(device)
|
|
478
|
+
|
|
479
|
+
@property
|
|
480
|
+
def logdet(self):
|
|
481
|
+
if self._logdet is None:
|
|
482
|
+
self.marginal_covariance()
|
|
483
|
+
return self._logdet
|
|
484
|
+
|
|
485
|
+
@property
|
|
486
|
+
def device(self):
|
|
487
|
+
return self.chans_arange.device
|
|
488
|
+
|
|
489
|
+
def mean_rc(self) -> torch.Tensor:
|
|
490
|
+
"""Return noise mean as a rank x channels tensor"""
|
|
491
|
+
shape = self.rank, self.n_channels
|
|
492
|
+
if self.mean_kind == "zero":
|
|
493
|
+
return torch.zeros(shape)
|
|
494
|
+
elif self.mean_kind == "by_rank":
|
|
495
|
+
return (
|
|
496
|
+
cast(torch.Tensor, self.mean)[:, None].broadcast_to(shape).contiguous()
|
|
497
|
+
)
|
|
498
|
+
elif self.mean_kind == "full":
|
|
499
|
+
return cast(torch.Tensor, self.mean)
|
|
500
|
+
else:
|
|
501
|
+
assert False
|
|
502
|
+
|
|
503
|
+
def marginal_mean(self):
|
|
504
|
+
"""Return noise mean as a rank x channels tensor"""
|
|
505
|
+
shape = self.rank, self.n_channels
|
|
506
|
+
if self.mean_kind == "zero":
|
|
507
|
+
return torch.zeros(shape)
|
|
508
|
+
if self.mean_kind == "by_rank":
|
|
509
|
+
return (
|
|
510
|
+
cast(torch.Tensor, self.mean)[:, None].broadcast_to(shape).contiguous()
|
|
511
|
+
)
|
|
512
|
+
if self.mean_kind == "full":
|
|
513
|
+
return self.mean
|
|
514
|
+
assert False
|
|
515
|
+
|
|
516
|
+
def whitener(self, channels=slice(None)):
|
|
517
|
+
cov = self.marginal_covariance(channels=channels)
|
|
518
|
+
chol = cov.cholesky().to_dense()
|
|
519
|
+
chans_eye = torch.eye(cov.shape[0], dtype=cov.dtype, device=cov.device)
|
|
520
|
+
whitener = torch.linalg.solve_triangular(chol, chans_eye, upper=False)
|
|
521
|
+
return whitener.reshape(cov.shape)
|
|
522
|
+
|
|
523
|
+
def whiten(self, data, channels=slice(None)):
|
|
524
|
+
assert self.mean_kind == "zero"
|
|
525
|
+
cov = self.marginal_covariance(channels=channels)
|
|
526
|
+
assert data.ndim == 3
|
|
527
|
+
data = data.reshape(len(data), -1)
|
|
528
|
+
chol = cov.cholesky().to_dense()
|
|
529
|
+
res = torch.linalg.solve_triangular(chol, data.T, upper=False)
|
|
530
|
+
res = res.T.unsqueeze(2)
|
|
531
|
+
|
|
532
|
+
# contour integral quad with lanczos is too fancy here...
|
|
533
|
+
# res = linear_operator.sqrt_inv_matmul(cov, data.unsqueeze(2))
|
|
534
|
+
assert res.ndim == 3 and res.shape == (*data.shape, 1)
|
|
535
|
+
return res
|
|
536
|
+
|
|
537
|
+
def full_dense_cov(self, device=None):
|
|
538
|
+
if self._full_cov is None:
|
|
539
|
+
self.marginal_covariance(device=device)
|
|
540
|
+
if device is not None:
|
|
541
|
+
self._full_cov_dense = self._full_cov_dense.to(device)
|
|
542
|
+
return self._full_cov_dense
|
|
543
|
+
|
|
544
|
+
def full_covinvcov(self, device=None):
|
|
545
|
+
if self._full_cov is None:
|
|
546
|
+
self.marginal_covariance(device=device)
|
|
547
|
+
assert self._full_cov is not None
|
|
548
|
+
if self._full_covinvcov is None:
|
|
549
|
+
self._full_covinvcov = self._full_cov.solve(self.full_dense_cov())
|
|
550
|
+
if device is not None:
|
|
551
|
+
self._full_covinvcov = self._full_covinvcov.to(device)
|
|
552
|
+
return self._full_covinvcov
|
|
553
|
+
|
|
554
|
+
def full_inverse(self, device=None):
|
|
555
|
+
if self._full_cov is None:
|
|
556
|
+
self.marginal_covariance(device=device)
|
|
557
|
+
assert self._full_cov is not None
|
|
558
|
+
if self._full_inverse is None:
|
|
559
|
+
self._full_inverse = self._full_cov.inverse().to_dense()
|
|
560
|
+
if device is not None:
|
|
561
|
+
self._full_inverse = self._full_inverse.to(device)
|
|
562
|
+
return self._full_inverse
|
|
563
|
+
|
|
564
|
+
def cov_batch_mul(
|
|
565
|
+
self, x: torch.Tensor, channels_src: torch.Tensor, channels_targ: torch.Tensor
|
|
566
|
+
):
|
|
567
|
+
assert x.ndim == 3
|
|
568
|
+
n = x.shape[0]
|
|
569
|
+
assert x.shape[2] == channels_src.shape[1]
|
|
570
|
+
assert channels_src.ndim == 2
|
|
571
|
+
assert channels_src.shape[0] == n
|
|
572
|
+
if channels_targ.ndim == 2:
|
|
573
|
+
assert channels_targ.shape[0] == n
|
|
574
|
+
else:
|
|
575
|
+
assert channels_targ.ndim == 1
|
|
576
|
+
|
|
577
|
+
if self.cov_kind == "factorized":
|
|
578
|
+
rank_root = self.rank_vt.T * self.rank_std # type: ignore
|
|
579
|
+
rank_cov = cast(torch.Tensor, rank_root @ rank_root.T)
|
|
580
|
+
rank_cov = rank_cov[None].broadcast_to(n, *rank_cov.shape)
|
|
581
|
+
x = rank_cov.bmm(x)
|
|
582
|
+
|
|
583
|
+
chan_root_left = self.channel_vt_zpad.T[channels_src] * self.channel_std # type: ignore
|
|
584
|
+
chan_root_right = self.channel_vt_zpad.T[channels_targ] * self.channel_std # type: ignore
|
|
585
|
+
if chan_root_right.ndim == 2:
|
|
586
|
+
targ_shp = (chan_root_left.shape[0], *chan_root_right.shape)
|
|
587
|
+
chan_root_right = chan_root_right[None].broadcast_to(targ_shp)
|
|
588
|
+
chan_cov = chan_root_left.bmm(chan_root_right.mT)
|
|
589
|
+
return x.bmm(chan_cov)
|
|
590
|
+
elif self.cov_kind == "scalar":
|
|
591
|
+
if channels_targ.ndim == 1:
|
|
592
|
+
channels_targ = channels_targ[None, :].broadcast_to(
|
|
593
|
+
(n, channels_targ.numel())
|
|
594
|
+
)
|
|
595
|
+
channels_src = torch.where(channels_src == self.n_channels, -2, channels_src)
|
|
596
|
+
eyes = (channels_src[:, :, None] == channels_targ[:, None, :]).float()
|
|
597
|
+
chan_cov = eyes * self.global_std**2 # type: ignore
|
|
598
|
+
return x.bmm(chan_cov)
|
|
599
|
+
elif self.cov_kind == "full":
|
|
600
|
+
# this branch is for debugging. it is slow.
|
|
601
|
+
if channels_targ.ndim == 1:
|
|
602
|
+
channels_targ = channels_targ[None, :].broadcast_to(
|
|
603
|
+
(n, channels_targ.numel())
|
|
604
|
+
)
|
|
605
|
+
cov_zpad = F.pad(self.full_cov, (0, 1, 0, 0, 0, 1))[None] # type: ignore
|
|
606
|
+
cov = cov_zpad.take_along_dim( # type: ignore
|
|
607
|
+
indices=channels_src[:, None, :, None, None], dim=2
|
|
608
|
+
)
|
|
609
|
+
assert cov.shape == (
|
|
610
|
+
n,
|
|
611
|
+
self.rank,
|
|
612
|
+
channels_src.shape[1],
|
|
613
|
+
self.rank,
|
|
614
|
+
self.n_channels + 1,
|
|
615
|
+
)
|
|
616
|
+
cov = cov.take_along_dim(
|
|
617
|
+
indices=channels_targ[:, None, None, None, :], dim=4
|
|
618
|
+
)
|
|
619
|
+
assert cov.shape == (
|
|
620
|
+
n,
|
|
621
|
+
self.rank,
|
|
622
|
+
channels_src.shape[1],
|
|
623
|
+
self.rank,
|
|
624
|
+
channels_targ.shape[1],
|
|
625
|
+
)
|
|
626
|
+
cov = cov.view(
|
|
627
|
+
n,
|
|
628
|
+
self.rank * channels_src.shape[1],
|
|
629
|
+
self.rank * channels_targ.shape[1],
|
|
630
|
+
)
|
|
631
|
+
x = x.view(n, 1, -1).bmm(cov)
|
|
632
|
+
return x.view(n, self.rank, channels_targ.shape[1])
|
|
633
|
+
else:
|
|
634
|
+
raise NotImplementedError(
|
|
635
|
+
f"Need to implement cov_batch_mul for {self.cov_kind=}."
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
def cov_batch(self, channels_left: torch.Tensor, channels_right: torch.Tensor):
|
|
639
|
+
if self.cov_kind != "factorized":
|
|
640
|
+
raise NotImplementedError(
|
|
641
|
+
f"Need to implement cov_batch for {self.cov_kind=}."
|
|
642
|
+
)
|
|
643
|
+
rank_root = self.rank_vt.T * self.rank_std
|
|
644
|
+
rank_cov = rank_root @ rank_root.T
|
|
645
|
+
chan_root_left = self.channel_vt_zpad.T[channels_left] * self.channel_std
|
|
646
|
+
chan_root_right = self.channel_vt_zpad.T[channels_right] * self.channel_std
|
|
647
|
+
if chan_root_right.ndim == 2:
|
|
648
|
+
chan_root_right = chan_root_right[None].broadcast_to(
|
|
649
|
+
chan_root_left.shape[0], *chan_root_right.shape
|
|
650
|
+
)
|
|
651
|
+
chan_cov = chan_root_left.bmm(chan_root_right.mT)
|
|
652
|
+
rank_cov = rank_cov[None].broadcast_to(
|
|
653
|
+
channels_left.shape[0], self.rank, self.rank
|
|
654
|
+
)
|
|
655
|
+
return operators.KroneckerProductLinearOperator(rank_cov, chan_cov)
|
|
656
|
+
|
|
657
|
+
def marginal_covariance(
|
|
658
|
+
self,
|
|
659
|
+
channels: torch.Tensor | slice | None = slice(None),
|
|
660
|
+
cache_prefix=None,
|
|
661
|
+
cache_key=None,
|
|
662
|
+
device=None,
|
|
663
|
+
):
|
|
664
|
+
if device is not None:
|
|
665
|
+
device = torch.device(device)
|
|
666
|
+
if cache_key is not None and torch.is_tensor(cache_key):
|
|
667
|
+
cache_key = cache_key.item()
|
|
668
|
+
if cache_prefix is not None:
|
|
669
|
+
cache_key = (cache_prefix, cache_key)
|
|
670
|
+
if cache_key is not None and cache_key in self.cache:
|
|
671
|
+
res = self.cache[cache_key]
|
|
672
|
+
if device is not None and device != res.device:
|
|
673
|
+
res = res.to(device)
|
|
674
|
+
self.cache[cache_key] = res
|
|
675
|
+
# TODO: why is this necessary? I thought that calling solve would cache this.
|
|
676
|
+
if not hasattr(res, "_memoize_cache"):
|
|
677
|
+
res.cholesky()
|
|
678
|
+
return res
|
|
679
|
+
if channels is None or channels == slice(None):
|
|
680
|
+
if self._full_cov is None:
|
|
681
|
+
fcov = self._marginal_covariance()
|
|
682
|
+
self._full_cov_dense = fcov.to_dense()
|
|
683
|
+
if not isinstance(fcov, operators.ConstantDiagLinearOperator):
|
|
684
|
+
fcov = operators.CholLinearOperator(fcov.cholesky())
|
|
685
|
+
self._full_cov = fcov
|
|
686
|
+
self._logdet = self._full_cov.logdet()
|
|
687
|
+
if device is not None and device != self._full_cov.device:
|
|
688
|
+
self._full_cov = self._full_cov.to(device)
|
|
689
|
+
self._logdet = self._logdet.to(device)
|
|
690
|
+
return self._full_cov
|
|
691
|
+
cov = self._marginal_covariance(channels)
|
|
692
|
+
if device is not None and device != cov.device:
|
|
693
|
+
cov = cov.to(device)
|
|
694
|
+
if cache_key is not None:
|
|
695
|
+
self.cache[cache_key] = cov
|
|
696
|
+
return cov
|
|
697
|
+
|
|
698
|
+
def offdiag_covariance(
|
|
699
|
+
self, channels_left=slice(None), channels_right=slice(None), device=None
|
|
700
|
+
):
|
|
701
|
+
odc = self._marginal_covariance(
|
|
702
|
+
channels=channels_right, channels_left=channels_left
|
|
703
|
+
)
|
|
704
|
+
if device is not None:
|
|
705
|
+
odc = odc.to(device)
|
|
706
|
+
return odc
|
|
707
|
+
|
|
708
|
+
def _marginal_covariance(self, channels=slice(None), channels_left=None):
|
|
709
|
+
channels = self.chans_arange[channels]
|
|
710
|
+
have_left = channels_left is not None
|
|
711
|
+
if channels_left is None:
|
|
712
|
+
channels_left = channels
|
|
713
|
+
nc = channels.numel()
|
|
714
|
+
ncl = channels_left.numel()
|
|
715
|
+
|
|
716
|
+
if have_left:
|
|
717
|
+
if self.cov_kind in ("scalar", "diagonal_by_rank", "diagonal"):
|
|
718
|
+
# if torch.isin(channels_left.to(channels), channels).any():
|
|
719
|
+
# raise ValueError(
|
|
720
|
+
# "Don't know how to get non-marginal covariance block "
|
|
721
|
+
# f"for cov_kind={self.cov_kind} when the block overlaps "
|
|
722
|
+
# "with the diagonal."
|
|
723
|
+
# )
|
|
724
|
+
sz = self.rank * ncl, self.rank * nc
|
|
725
|
+
return operators.ZeroLinearOperator(*sz, dtype=self.global_std.dtype)
|
|
726
|
+
|
|
727
|
+
if self.cov_kind == "scalar":
|
|
728
|
+
eye = operators.IdentityLinearOperator(self.rank * nc, device=self.device)
|
|
729
|
+
return self.global_std.square() * eye
|
|
730
|
+
|
|
731
|
+
if self.cov_kind == "diagonal_by_rank":
|
|
732
|
+
rank_diag = operators.DiagLinearOperator(self.rank_std**2)
|
|
733
|
+
chans_eye = operators.IdentityLinearOperator(nc, device=self.device)
|
|
734
|
+
return torch.kron(rank_diag, chans_eye)
|
|
735
|
+
|
|
736
|
+
if self.cov_kind == "diagonal":
|
|
737
|
+
chans_std = self.full_std[:, channels]
|
|
738
|
+
return operators.DiagLinearOperator(chans_std**2)
|
|
739
|
+
|
|
740
|
+
if self.cov_kind == "full":
|
|
741
|
+
marg_cov = self.full_cov[:, channels_left][..., channels]
|
|
742
|
+
r = marg_cov.shape[0]
|
|
743
|
+
marg_cov = marg_cov.reshape(r * ncl, r * nc)
|
|
744
|
+
return linear_operator.to_linear_operator(marg_cov)
|
|
745
|
+
|
|
746
|
+
if self.cov_kind in "factorized":
|
|
747
|
+
rank_root = self.rank_vt.T * self.rank_std
|
|
748
|
+
rank_cov = rank_root @ rank_root.T
|
|
749
|
+
chan_root = self.channel_vt.T[channels] * self.channel_std
|
|
750
|
+
chan_root_right = chan_root.T
|
|
751
|
+
if have_left:
|
|
752
|
+
chan_root = self.channel_vt.T[channels_left] * self.channel_std
|
|
753
|
+
chan_cov = chan_root @ chan_root_right
|
|
754
|
+
return operators.KroneckerProductLinearOperator(rank_cov, chan_cov)
|
|
755
|
+
|
|
756
|
+
if self.cov_kind == "factorized_rank_diag":
|
|
757
|
+
rank_cov = operators.DiagLinearOperator(self.rank_std.square())
|
|
758
|
+
chan_root = self.channel_vt.T[channels] * self.channel_std
|
|
759
|
+
chan_root_right = chan_root.T
|
|
760
|
+
if have_left:
|
|
761
|
+
chan_root = self.channel_vt.T[channels_left] * self.channel_std
|
|
762
|
+
chan_cov = chan_root @ chan_root_right
|
|
763
|
+
return torch.kron(rank_cov, chan_cov)
|
|
764
|
+
|
|
765
|
+
if self.cov_kind == "factorized_by_rank_rank_diag":
|
|
766
|
+
rank_std = self.rank_std.view(self.rank, 1, 1)
|
|
767
|
+
chans_std = self.channel_std[:, :, None] # no slice here!
|
|
768
|
+
rc_std = rank_std * chans_std
|
|
769
|
+
chans_vt = self.channel_vt[:, :, channels]
|
|
770
|
+
chan_rootl = chan_root = rc_std * chans_vt
|
|
771
|
+
if have_left:
|
|
772
|
+
chans_vtl = self.channel_vt[:, :, channels_left]
|
|
773
|
+
chan_rootl = rc_std * chans_vtl
|
|
774
|
+
blocks = torch.bmm(chan_rootl.mT, chan_root)
|
|
775
|
+
blocks = linear_operator.to_linear_operator(blocks)
|
|
776
|
+
if have_left:
|
|
777
|
+
return more_operators.NonSquareBlockLinearOperator(blocks)
|
|
778
|
+
return operators.BlockDiagLinearOperator(blocks)
|
|
779
|
+
|
|
780
|
+
assert False
|
|
781
|
+
|
|
782
|
+
@classmethod
|
|
783
|
+
def estimate(
|
|
784
|
+
cls,
|
|
785
|
+
snippets,
|
|
786
|
+
mean_kind="zero",
|
|
787
|
+
cov_kind="scalar",
|
|
788
|
+
shrinkage=0.0,
|
|
789
|
+
glasso_alpha: int | float | None = None,
|
|
790
|
+
eps=1e-4,
|
|
791
|
+
zero_radius: float | None = None,
|
|
792
|
+
svd_batch_size=256,
|
|
793
|
+
rgeom=None,
|
|
794
|
+
):
|
|
795
|
+
"""Factory method to estimate noise model from TPCA snippets
|
|
796
|
+
|
|
797
|
+
Arguments
|
|
798
|
+
---------
|
|
799
|
+
snippets : torch.Tensor
|
|
800
|
+
(n, rank, c) array of tpca-embedded noise snippets
|
|
801
|
+
missing values are okay, indicate by NaN please
|
|
802
|
+
"""
|
|
803
|
+
n, rank, n_channels = snippets.shape
|
|
804
|
+
init_kw = dict(
|
|
805
|
+
rank=rank, n_channels=n_channels, mean_kind=mean_kind, cov_kind=cov_kind
|
|
806
|
+
)
|
|
807
|
+
x = torch.asarray(snippets)
|
|
808
|
+
x = x.to(torch.promote_types(x.dtype, torch.float))
|
|
809
|
+
if zero_radius is not None or shrinkage:
|
|
810
|
+
assert cov_kind.startswith("factorized")
|
|
811
|
+
assert rgeom is not None
|
|
812
|
+
|
|
813
|
+
# estimate mean and center data
|
|
814
|
+
if mean_kind == "zero":
|
|
815
|
+
mean = None
|
|
816
|
+
elif mean_kind == "by_rank":
|
|
817
|
+
mean = torch.nanmean(x, dim=(0, 2))
|
|
818
|
+
assert mean.isfinite().all()
|
|
819
|
+
x = x - mean.unsqueeze(1)
|
|
820
|
+
elif mean_kind == "full":
|
|
821
|
+
mean = torch.nanmean(x, dim=0)
|
|
822
|
+
mean = torch.where(
|
|
823
|
+
mean.isnan().all(1).unsqueeze(1),
|
|
824
|
+
torch.nanmean(mean, dim=1).unsqueeze(1),
|
|
825
|
+
mean,
|
|
826
|
+
)
|
|
827
|
+
x = x - mean
|
|
828
|
+
else:
|
|
829
|
+
assert False
|
|
830
|
+
|
|
831
|
+
# estimate covs
|
|
832
|
+
# still n, rank, n_channels
|
|
833
|
+
dxsq = x.square()
|
|
834
|
+
full_var = torch.nanmean(dxsq, dim=0)
|
|
835
|
+
rank_var = torch.nanmean(full_var, dim=1)
|
|
836
|
+
rank_std = rank_var.sqrt_()
|
|
837
|
+
assert rank_var.isfinite().all()
|
|
838
|
+
global_var = torch.nanmean(rank_var)
|
|
839
|
+
global_std = global_var.sqrt()
|
|
840
|
+
|
|
841
|
+
if cov_kind == "scalar":
|
|
842
|
+
return cls(mean=mean, global_std=global_std, **init_kw)
|
|
843
|
+
|
|
844
|
+
if cov_kind == "by_rank":
|
|
845
|
+
return cls(mean=mean, global_std=global_std, rank_std=rank_std, **init_kw)
|
|
846
|
+
|
|
847
|
+
if cov_kind == "diagonal":
|
|
848
|
+
full_var = torch.where(
|
|
849
|
+
full_var.isnan().all(1).unsqueeze(1),
|
|
850
|
+
rank_var.unsqueeze(1),
|
|
851
|
+
full_var,
|
|
852
|
+
)
|
|
853
|
+
full_std = full_var.sqrt()
|
|
854
|
+
return cls(mean=mean, global_std=global_std, full_std=full_std, **init_kw)
|
|
855
|
+
|
|
856
|
+
if cov_kind == "full":
|
|
857
|
+
x = x.view(n, rank * n_channels)
|
|
858
|
+
present = torch.isfinite(x).any(dim=0)
|
|
859
|
+
cov = torch.eye(x.shape[1], device=x.device, dtype=x.dtype)
|
|
860
|
+
vcov = spiketorch.nancov(x[:, present], force_posdef=True, eps=eps)
|
|
861
|
+
cov[present[:, None] & present[None, :]] = vcov.view(-1)
|
|
862
|
+
cov = cov.reshape(rank, n_channels, rank, n_channels)
|
|
863
|
+
return cls(mean=mean, global_std=global_std, full_cov=cov, **init_kw)
|
|
864
|
+
|
|
865
|
+
assert cov_kind.startswith("factorized")
|
|
866
|
+
# handle rank part first, then the spatial part
|
|
867
|
+
|
|
868
|
+
# start by getting the rank part of the cov, if necessary, and whitening
|
|
869
|
+
# the ranks to leave behind x_spatial
|
|
870
|
+
if "rank_diag" in cov_kind:
|
|
871
|
+
# rank part is diagonal
|
|
872
|
+
rank_vt = None
|
|
873
|
+
x_spatial = x.div_(rank_std.unsqueeze(1))
|
|
874
|
+
del x
|
|
875
|
+
else:
|
|
876
|
+
# full cov estimation for rank part via svd
|
|
877
|
+
# we have NaNs, but we can get rid of them because channels are either all
|
|
878
|
+
# NaN or not. below, for the spatial part, no such luck and we have to
|
|
879
|
+
# evaluate the covariance in a masked way
|
|
880
|
+
|
|
881
|
+
# doing this on CPU since the SVD can use a lot of memory.
|
|
882
|
+
x_rank = x.permute(0, 2, 1)
|
|
883
|
+
x_rank = x_rank.cpu().reshape(n * n_channels, rank)
|
|
884
|
+
(valid,) = x_rank.isfinite().all(dim=1).nonzero(as_tuple=True)
|
|
885
|
+
x_rankv = x_rank[valid]
|
|
886
|
+
orig_device = x.device
|
|
887
|
+
del x
|
|
888
|
+
assert not x_rankv.requires_grad
|
|
889
|
+
u_rankv, rank_sing, rank_vt = torch.linalg.svd(x_rankv, full_matrices=False)
|
|
890
|
+
|
|
891
|
+
correction = torch.tensor(len(x_rankv) - 1.0).sqrt()
|
|
892
|
+
rank_std = rank_sing / correction
|
|
893
|
+
rank_std = rank_std.to(orig_device)
|
|
894
|
+
correction = correction.to(orig_device)
|
|
895
|
+
|
|
896
|
+
# whitened spatial part -- reuse storage
|
|
897
|
+
x_spatial = x_rank
|
|
898
|
+
del x_rank
|
|
899
|
+
x_spatial[valid] = u_rankv
|
|
900
|
+
x_spatial = x_spatial.reshape(n, n_channels, rank).permute(0, 2, 1)
|
|
901
|
+
x_spatial = x_spatial.to(orig_device)
|
|
902
|
+
x_spatial.mul_(correction)
|
|
903
|
+
|
|
904
|
+
spatial_mask = None
|
|
905
|
+
if zero_radius:
|
|
906
|
+
assert rgeom is not None
|
|
907
|
+
assert rgeom.shape[0] == n_channels
|
|
908
|
+
rg_np = rgeom.numpy(force=True) if torch.is_tensor(rgeom) else rgeom
|
|
909
|
+
spatial_mask = squareform(pdist(rg_np)) < zero_radius
|
|
910
|
+
|
|
911
|
+
# spatial part could be "by rank" or same for all ranks
|
|
912
|
+
# either way, there are nans afoot
|
|
913
|
+
if "by_rank" in cov_kind:
|
|
914
|
+
channel_std = torch.ones_like(x_spatial[0])
|
|
915
|
+
channel_vt = torch.zeros((rank, n_channels, n_channels)).to(channel_std)
|
|
916
|
+
for q in range(rank):
|
|
917
|
+
xq = x_spatial[:, q]
|
|
918
|
+
validq = xq.isfinite().any(0)
|
|
919
|
+
covq = spiketorch.nancov(xq[:, validq])
|
|
920
|
+
if shrinkage:
|
|
921
|
+
covq = F.softshrink(covq, shrinkage)
|
|
922
|
+
fullcovq = torch.eye(xq.shape[1], dtype=covq.dtype, device=covq.device)
|
|
923
|
+
fullcovq[validq[:, None] & validq[None, :]] = covq.view(-1)
|
|
924
|
+
if spatial_mask is not None:
|
|
925
|
+
fullcovq *= torch.asarray(spatial_mask).to(fullcovq)
|
|
926
|
+
if glasso_alpha:
|
|
927
|
+
res = graphical_lasso(
|
|
928
|
+
fullcovq.numpy(force=True),
|
|
929
|
+
alpha=glasso_alpha,
|
|
930
|
+
mode="cd",
|
|
931
|
+
max_iter=100,
|
|
932
|
+
verbose=logger.isEnabledFor(DARTSORTDEBUG),
|
|
933
|
+
)
|
|
934
|
+
fullcovq = torch.from_numpy(res[0]).to(fullcovq)
|
|
935
|
+
qeig, qv = torch.linalg.eigh(fullcovq)
|
|
936
|
+
channel_std[q] = qeig.sqrt()
|
|
937
|
+
channel_vt[q] = qv.T
|
|
938
|
+
else:
|
|
939
|
+
x_spatial = x_spatial.reshape(n * rank, n_channels)
|
|
940
|
+
valid = x_spatial.isfinite().any(0)
|
|
941
|
+
|
|
942
|
+
if "noise" in cov_kind:
|
|
943
|
+
xx = x_spatial.numpy(force=True)
|
|
944
|
+
invalid = np.isnan(xx)
|
|
945
|
+
xx[invalid] = (
|
|
946
|
+
np.random.default_rng(0).normal(size=invalid.sum()).astype(xx.dtype)
|
|
947
|
+
)
|
|
948
|
+
x_spatial = torch.from_numpy(xx).to(x_spatial)
|
|
949
|
+
valid = slice(None)
|
|
950
|
+
init_kw["cov_kind"] = cov_kind.removesuffix("noise")
|
|
951
|
+
|
|
952
|
+
cov = spiketorch.nancov(x_spatial[:, valid].double(), force_posdef=True)
|
|
953
|
+
cov.diagonal().add_(eps)
|
|
954
|
+
if shrinkage:
|
|
955
|
+
cov = F.softshrink(cov, shrinkage)
|
|
956
|
+
|
|
957
|
+
if glasso_alpha and isinstance(glasso_alpha, int):
|
|
958
|
+
logger.dartsortdebug(
|
|
959
|
+
f"Run glasso cv on {x_spatial.shape=} {cov.abs().max()=}"
|
|
960
|
+
)
|
|
961
|
+
# todo: clean up, propagate, ...
|
|
962
|
+
glasso = GraphicalLassoCV(
|
|
963
|
+
alphas=glasso_alpha,
|
|
964
|
+
n_jobs=1,
|
|
965
|
+
verbose=logger.isEnabledFor(DARTSORTDEBUG),
|
|
966
|
+
assume_centered=True,
|
|
967
|
+
eps=eps,
|
|
968
|
+
)
|
|
969
|
+
xx = x_spatial[:, valid].double().numpy(force=True)
|
|
970
|
+
invalid = np.isnan(xx)
|
|
971
|
+
xx[invalid] = np.random.default_rng(0).normal(size=invalid.sum())
|
|
972
|
+
with warnings.catch_warnings(action="ignore"):
|
|
973
|
+
glasso.fit(xx)
|
|
974
|
+
logger.dartsortdebug(f"Best alpha was {glasso.alpha_=}")
|
|
975
|
+
glasso_alpha = glasso.alpha_
|
|
976
|
+
|
|
977
|
+
if glasso_alpha and isinstance(glasso_alpha, float):
|
|
978
|
+
logger.dartsortdebug(f"Run glasso on {cov.shape=}")
|
|
979
|
+
res = graphical_lasso(
|
|
980
|
+
cov.numpy(force=True),
|
|
981
|
+
alpha=glasso_alpha,
|
|
982
|
+
mode="cd",
|
|
983
|
+
max_iter=100,
|
|
984
|
+
verbose=logger.isEnabledFor(DARTSORTDEBUG),
|
|
985
|
+
)
|
|
986
|
+
cov = torch.from_numpy(res[0]).to(cov)
|
|
987
|
+
|
|
988
|
+
cov_spatial = cov = cov.to(x_spatial)
|
|
989
|
+
assert cov_spatial.isfinite().all()
|
|
990
|
+
if valid != slice(None):
|
|
991
|
+
cov_spatial = torch.eye(
|
|
992
|
+
x_spatial.shape[1], dtype=x_spatial.dtype, device=x_spatial.device
|
|
993
|
+
)
|
|
994
|
+
cov_spatial[valid[:, None] & valid[None, :]] = cov.view(-1)
|
|
995
|
+
assert cov_spatial.isfinite().all()
|
|
996
|
+
if spatial_mask is not None:
|
|
997
|
+
cov_spatial *= torch.asarray(spatial_mask).to(cov_spatial)
|
|
998
|
+
assert cov_spatial.isfinite().all()
|
|
999
|
+
channel_eig, channel_v = torch.linalg.eigh(cov_spatial.double())
|
|
1000
|
+
if spatial_mask is None:
|
|
1001
|
+
assert channel_eig.isfinite().all()
|
|
1002
|
+
else:
|
|
1003
|
+
# the spatial mask is, i guess, not so plausible as a cov...
|
|
1004
|
+
channel_eig[channel_eig < eps] = eps
|
|
1005
|
+
channel_std = channel_eig.sqrt().float()
|
|
1006
|
+
channel_vt = channel_v.T.contiguous().float()
|
|
1007
|
+
assert channel_std.isfinite().all()
|
|
1008
|
+
assert channel_vt.isfinite().all()
|
|
1009
|
+
|
|
1010
|
+
return cls(
|
|
1011
|
+
mean=mean,
|
|
1012
|
+
global_std=global_std,
|
|
1013
|
+
rank_std=rank_std,
|
|
1014
|
+
rank_vt=rank_vt,
|
|
1015
|
+
channel_std=channel_std,
|
|
1016
|
+
channel_vt=channel_vt,
|
|
1017
|
+
zero_radius=zero_radius,
|
|
1018
|
+
**init_kw,
|
|
1019
|
+
)
|
|
1020
|
+
|
|
1021
|
+
@classmethod
|
|
1022
|
+
def estimate_from_hdf5(
|
|
1023
|
+
cls,
|
|
1024
|
+
hdf5_path,
|
|
1025
|
+
mean_kind="zero",
|
|
1026
|
+
cov_kind="factorizednoise",
|
|
1027
|
+
motion_est=None,
|
|
1028
|
+
interp_params: InterpolationParams = default_interpolation_params,
|
|
1029
|
+
device=None,
|
|
1030
|
+
shrinkage=0.0,
|
|
1031
|
+
glasso_alpha: int | float | None = None,
|
|
1032
|
+
zero_radius: float | None = None,
|
|
1033
|
+
rgeom=None,
|
|
1034
|
+
):
|
|
1035
|
+
from dartsort.util.drift_util import registered_geometry
|
|
1036
|
+
|
|
1037
|
+
logger.dartsortdebug(
|
|
1038
|
+
f"Estimate embedded noise with {mean_kind=} {cov_kind=} {motion_est is None=} {glasso_alpha=}"
|
|
1039
|
+
)
|
|
1040
|
+
|
|
1041
|
+
with h5py.File(hdf5_path, "r", locking=False) as h5:
|
|
1042
|
+
geom = h5["geom"][:]
|
|
1043
|
+
if rgeom is None:
|
|
1044
|
+
rgeom = geom
|
|
1045
|
+
if motion_est is not None:
|
|
1046
|
+
rgeom = registered_geometry(geom, motion_est=motion_est)
|
|
1047
|
+
snippets = interpolate_residual_snippets(
|
|
1048
|
+
motion_est,
|
|
1049
|
+
hdf5_path,
|
|
1050
|
+
geom,
|
|
1051
|
+
rgeom,
|
|
1052
|
+
interp_params=interp_params,
|
|
1053
|
+
device=device,
|
|
1054
|
+
)
|
|
1055
|
+
return cls.estimate(
|
|
1056
|
+
snippets,
|
|
1057
|
+
shrinkage=shrinkage,
|
|
1058
|
+
mean_kind=mean_kind,
|
|
1059
|
+
cov_kind=cov_kind,
|
|
1060
|
+
glasso_alpha=glasso_alpha,
|
|
1061
|
+
zero_radius=zero_radius,
|
|
1062
|
+
rgeom=rgeom,
|
|
1063
|
+
)
|
|
1064
|
+
|
|
1065
|
+
def detection_prior_log_prob(self, templates_pca_projected, threshold=10.0):
|
|
1066
|
+
"""
|
|
1067
|
+
Computes:
|
|
1068
|
+
z(T) = log[p(noise det | T)]
|
|
1069
|
+
= log[P(|N|^2 - |N - T|^2 > threshold^2)]
|
|
1070
|
+
= log[log P(2N.T > threshold^2 + |T|^2)]
|
|
1071
|
+
|
|
1072
|
+
Then, later, in mixture modeling one can compute
|
|
1073
|
+
p(l = noise | x, T) = log pi_noise + log N(x | noise) - z(T)
|
|
1074
|
+
|
|
1075
|
+
It's a little bit counterintuitive: this boosts the noise probability more
|
|
1076
|
+
for units with higher signal templates, since their FP probability is lower.
|
|
1077
|
+
|
|
1078
|
+
Note that since N ~ N(0, C), N.T ~ N(0, tr TCT'). Or, whatever version of
|
|
1079
|
+
that makes the dimensions work out. Thus we need to compute
|
|
1080
|
+
log normal_sf(0.5 * (thresh^2 + |T|^2) ; mean=0, scale=sqrt(tr TCT))
|
|
1081
|
+
"""
|
|
1082
|
+
C = self.full_dense_cov()
|
|
1083
|
+
templates_pca_projected = torch.asarray(
|
|
1084
|
+
templates_pca_projected, device=C.device
|
|
1085
|
+
)
|
|
1086
|
+
T = templates_pca_projected.reshape(len(templates_pca_projected), -1)
|
|
1087
|
+
assert C.shape == (T.shape[1], T.shape[1])
|
|
1088
|
+
tr = torch.einsum("nc,cd,nd->n", T, C, T)
|
|
1089
|
+
scale = tr.sqrt_()
|
|
1090
|
+
crit = T.square().sum(dim=1).add_(threshold**2).mul_(0.5)
|
|
1091
|
+
return norm.logsf(crit.numpy(force=True), scale=scale.numpy(force=True))
|
|
1092
|
+
|
|
1093
|
+
def channelwise_detection_prior_log_prob(
|
|
1094
|
+
self, threshold=4.0, n_samples=8192, seed=0
|
|
1095
|
+
):
|
|
1096
|
+
"""Compute the marginal probability that the norm is larger than threshold on each channel.
|
|
1097
|
+
|
|
1098
|
+
TPCA is isometry, so no need to know the basis when working with the norm. Further, we can
|
|
1099
|
+
sample independent normals using the eigs.
|
|
1100
|
+
"""
|
|
1101
|
+
probs = np.zeros(self.n_channels)
|
|
1102
|
+
rg = np.random.default_rng(seed)
|
|
1103
|
+
samples = rg.normal(size=(n_samples, self.rank))
|
|
1104
|
+
tmp = samples.copy()
|
|
1105
|
+
for channel in range(self.n_channels):
|
|
1106
|
+
marginal_cov = self.marginal_covariance(channels=[channel]).to_dense()
|
|
1107
|
+
marginal_cov = marginal_cov.numpy(force=True).astype(np.float64)
|
|
1108
|
+
eigs = np.linalg.eigvalsh(marginal_cov)
|
|
1109
|
+
stds = np.sqrt(eigs)
|
|
1110
|
+
norms = np.linalg.norm(np.multiply(samples, stds, out=tmp), axis=1)
|
|
1111
|
+
probs[channel] = np.mean(norms > threshold)
|
|
1112
|
+
return np.log(probs)
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def interpolate_residual_snippets(
|
|
1116
|
+
motion_est,
|
|
1117
|
+
hdf5_path,
|
|
1118
|
+
geom,
|
|
1119
|
+
registered_geom,
|
|
1120
|
+
residual_times_s_dataset_name="residual_times_seconds",
|
|
1121
|
+
residual_dataset_name="residual",
|
|
1122
|
+
interp_params: InterpolationParams = default_interpolation_params,
|
|
1123
|
+
do_tpca=True,
|
|
1124
|
+
workers=None,
|
|
1125
|
+
device=None,
|
|
1126
|
+
batch_size=16,
|
|
1127
|
+
):
|
|
1128
|
+
"""PCA-embed and interpolate residual snippets to the registered probe"""
|
|
1129
|
+
from dartsort.util import data_util, drift_util, interpolation_util
|
|
1130
|
+
|
|
1131
|
+
assert geom.shape[1] == 2, "Haven't implemented 3d probes here."
|
|
1132
|
+
|
|
1133
|
+
if device is None:
|
|
1134
|
+
device = "cuda" if torch.cuda.is_available() else None
|
|
1135
|
+
device = torch.device(device)
|
|
1136
|
+
|
|
1137
|
+
with h5py.File(hdf5_path, "r", locking=False) as h5:
|
|
1138
|
+
channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
|
|
1139
|
+
snippets = cast(h5py.Dataset, h5[residual_dataset_name])[:]
|
|
1140
|
+
times_s = cast(h5py.Dataset, h5[residual_times_s_dataset_name])[:]
|
|
1141
|
+
channel_index = torch.from_numpy(channel_index)
|
|
1142
|
+
snippets = torch.from_numpy(snippets)
|
|
1143
|
+
times_s = torch.from_numpy(times_s)
|
|
1144
|
+
|
|
1145
|
+
# tpca project
|
|
1146
|
+
if do_tpca:
|
|
1147
|
+
tpca = data_util.get_tpca(hdf5_path)
|
|
1148
|
+
if device is not None:
|
|
1149
|
+
tpca = tpca.to(tpca.b.components.dtype)
|
|
1150
|
+
tpca = tpca.to(device=device)
|
|
1151
|
+
if tpca.temporal_slice is not None:
|
|
1152
|
+
snippets = snippets[:, tpca.temporal_slice]
|
|
1153
|
+
n, t, c = snippets.shape
|
|
1154
|
+
snippets_proj = snippets.new_zeros(n, tpca.rank, c)
|
|
1155
|
+
for i0 in range(0, n, batch_size):
|
|
1156
|
+
i1 = min(n, i0 + batch_size)
|
|
1157
|
+
nb = i1 - i0
|
|
1158
|
+
b = snippets[i0:i1].permute(0, 2, 1).reshape(nb * c, t)
|
|
1159
|
+
b = tpca._transform_in_probe(b.to(tpca.components))
|
|
1160
|
+
snippets_proj[i0:i1] = b.reshape(nb, c, -1).permute(0, 2, 1)
|
|
1161
|
+
snippets = snippets_proj
|
|
1162
|
+
n, dim, c = snippets.shape
|
|
1163
|
+
assert snippets.isfinite().all()
|
|
1164
|
+
|
|
1165
|
+
# -- interpolate
|
|
1166
|
+
# fill in the registered probe with residual snippets using interpolation,
|
|
1167
|
+
# with possible missing values
|
|
1168
|
+
source_geom = torch.asarray(geom).to(snippets).to(device)
|
|
1169
|
+
source_pos = source_geom[None].broadcast_to(n, *geom.shape).contiguous()
|
|
1170
|
+
|
|
1171
|
+
# - precompute
|
|
1172
|
+
interp_params = interp_params.normalize()
|
|
1173
|
+
precomputed_data = interpolation_util.full_probe_precompute(
|
|
1174
|
+
source_geom=source_geom.to(device),
|
|
1175
|
+
channel_index=channel_index.to(device),
|
|
1176
|
+
params=interp_params,
|
|
1177
|
+
)
|
|
1178
|
+
if precomputed_data is not None:
|
|
1179
|
+
precomputed_data = precomputed_data.to(device)
|
|
1180
|
+
if motion_est is None:
|
|
1181
|
+
# no drift case, no missing values, but still interpolate to avoid
|
|
1182
|
+
# statistical differences between drifty/no drift versions of the sorter
|
|
1183
|
+
target_geom = torch.asarray(registered_geom).to(device=device)
|
|
1184
|
+
assert torch.equal(source_geom, target_geom)
|
|
1185
|
+
target_pos = target_geom[None].broadcast_to(n, *geom.shape).contiguous()
|
|
1186
|
+
|
|
1187
|
+
for bs in range(0, n, batch_size):
|
|
1188
|
+
sl = slice(bs, min(n, bs + batch_size))
|
|
1189
|
+
snippets[sl] = interpolation_util.kernel_interpolate(
|
|
1190
|
+
snippets[sl].to(device),
|
|
1191
|
+
source_pos[sl].to(device),
|
|
1192
|
+
target_pos[sl].to(device),
|
|
1193
|
+
params=interp_params,
|
|
1194
|
+
precomputed_data=precomputed_data,
|
|
1195
|
+
allow_destroy=True,
|
|
1196
|
+
).to(snippets)
|
|
1197
|
+
return snippets
|
|
1198
|
+
|
|
1199
|
+
# goal
|
|
1200
|
+
# - determine what registered channels the shifting source positions
|
|
1201
|
+
# would land on
|
|
1202
|
+
# - interpolate residual snippets from the shifted source position
|
|
1203
|
+
# to the target positions
|
|
1204
|
+
# except that's slow, because kriging interpolator would require
|
|
1205
|
+
# inverting a matrix for each shifted source geom. so instead,
|
|
1206
|
+
# let's shift the targets inversely and use the same source geom,
|
|
1207
|
+
# even if that has a bit of a different meaning.
|
|
1208
|
+
|
|
1209
|
+
# determine each channel's drift over time
|
|
1210
|
+
source_depths = source_pos[:, :, 1].reshape(-1)
|
|
1211
|
+
source_t = times_s[:, None].broadcast_to(source_pos[:, :, 1].shape).reshape(-1)
|
|
1212
|
+
source_shifts = motion_est.disp_at_s(source_t.cpu(), source_depths.cpu())
|
|
1213
|
+
source_shifts = source_shifts.reshape(source_pos[:, :, 1].shape).astype("float32")
|
|
1214
|
+
assert np.isfinite(source_shifts).all()
|
|
1215
|
+
source_shifts_xy = np.stack([np.zeros_like(source_shifts), source_shifts], axis=-1)
|
|
1216
|
+
source_pos_shifted = source_pos.numpy(force=True) - source_shifts_xy
|
|
1217
|
+
|
|
1218
|
+
# query the target geom for the closest source pos, within reason
|
|
1219
|
+
rg_np = registered_geom
|
|
1220
|
+
if torch.is_tensor(rg_np):
|
|
1221
|
+
rg_np = rg_np.numpy(force=True)
|
|
1222
|
+
kdtree = drift_util.KDTree(rg_np)
|
|
1223
|
+
match_distance = drift_util.pdist(geom).min()
|
|
1224
|
+
_, targ_inds = kdtree.query(
|
|
1225
|
+
source_pos_shifted.reshape(-1, geom.shape[1]),
|
|
1226
|
+
distance_upper_bound=match_distance,
|
|
1227
|
+
workers=workers or -1,
|
|
1228
|
+
)
|
|
1229
|
+
targ_inds = torch.from_numpy(targ_inds).reshape(source_pos.shape[:2])
|
|
1230
|
+
assert (targ_inds < kdtree.n).all()
|
|
1231
|
+
target_pos = torch.asarray(registered_geom[targ_inds], device=device)
|
|
1232
|
+
target_pos_shifted = target_pos + torch.asarray(source_shifts_xy).to(target_pos)
|
|
1233
|
+
target_pos_shifted = torch.asarray(target_pos_shifted).to(snippets)
|
|
1234
|
+
|
|
1235
|
+
# allocate output storage with an extra channel of NaN needed later
|
|
1236
|
+
for bs in range(0, n, batch_size):
|
|
1237
|
+
sl = slice(bs, min(n, bs + batch_size))
|
|
1238
|
+
snippets[sl] = interpolation_util.kernel_interpolate(
|
|
1239
|
+
snippets[sl].to(device),
|
|
1240
|
+
source_pos[sl].to(device),
|
|
1241
|
+
target_pos_shifted[sl].to(device),
|
|
1242
|
+
params=interp_params,
|
|
1243
|
+
precomputed_data=precomputed_data,
|
|
1244
|
+
allow_destroy=True,
|
|
1245
|
+
).to(snippets)
|
|
1246
|
+
assert snippets.isfinite().all()
|
|
1247
|
+
|
|
1248
|
+
# now, let's embed these into the full registered probe
|
|
1249
|
+
snippets_full = snippets.new_full((n, dim, len(registered_geom)), torch.nan)
|
|
1250
|
+
targ_inds = targ_inds[:, None].broadcast_to(snippets.shape)
|
|
1251
|
+
snippets_full.scatter_(2, targ_inds.to(snippets.device), snippets)
|
|
1252
|
+
|
|
1253
|
+
return snippets_full
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
def fp_control_threshold(
|
|
1257
|
+
fp_dataframe,
|
|
1258
|
+
fp_num_frames,
|
|
1259
|
+
tp_unit_ids,
|
|
1260
|
+
tp_counts,
|
|
1261
|
+
clustering_num_frames,
|
|
1262
|
+
template_normsqs,
|
|
1263
|
+
clustering_subsampling_rate=1.0,
|
|
1264
|
+
max_fp_per_input_spike=1.0,
|
|
1265
|
+
resolution=1.0,
|
|
1266
|
+
min_threshold_factor=0.1,
|
|
1267
|
+
min_threshold=5.0,
|
|
1268
|
+
):
|
|
1269
|
+
"""Global threshold for template matching to control the false positive rate
|
|
1270
|
+
|
|
1271
|
+
For each unit, as a function of the threshold, we can estimate the number
|
|
1272
|
+
of false positives per frame using `fp_dataframe` and `fp_num_frames`. We
|
|
1273
|
+
can also estimate the true positive rate using `labels` and
|
|
1274
|
+
`clustering_num_frames`.
|
|
1275
|
+
|
|
1276
|
+
Then, for each unit, we can find the largest threshold such that the false
|
|
1277
|
+
positive rate divided by the true positive rate is <max_fp_per_input_spike.
|
|
1278
|
+
This controls the FPR after deconv per unit to be
|
|
1279
|
+
<max_fp_per_input_spike/2*max_fp_per_input_spike.
|
|
1280
|
+
We can then take the max over units to get a global threshold.
|
|
1281
|
+
|
|
1282
|
+
Arguments
|
|
1283
|
+
---------
|
|
1284
|
+
fp_dataframe: pd.DataFrame
|
|
1285
|
+
fp_num_frames : int
|
|
1286
|
+
Above two as returned by .unit_false_positives()
|
|
1287
|
+
tp_unit_ids, tp_counts : np.array
|
|
1288
|
+
Unique of clustering labels
|
|
1289
|
+
clustering_num_frames : int
|
|
1290
|
+
Number of valid recording frames used during clustering
|
|
1291
|
+
clustering_subsampling_rate : float
|
|
1292
|
+
If the full set of detected spikes from the whole `clustering_num_frames`
|
|
1293
|
+
frame collection was subsampled, then we should calculate rates per frame
|
|
1294
|
+
by dividing by clustering_subsampling_rate * clustering_num_frames
|
|
1295
|
+
max_fp_per_input_spike : float
|
|
1296
|
+
The control parameter described above.
|
|
1297
|
+
resolution : float
|
|
1298
|
+
The rates as a function of threshold will be estimated as a function of
|
|
1299
|
+
(squared) deconv threshold at this resolution.
|
|
1300
|
+
|
|
1301
|
+
Returns
|
|
1302
|
+
-------
|
|
1303
|
+
threshold : float
|
|
1304
|
+
Note: this is in squared objective units.
|
|
1305
|
+
"""
|
|
1306
|
+
tp_counts = tp_counts[tp_unit_ids >= 0]
|
|
1307
|
+
tp_unit_ids = tp_unit_ids[tp_unit_ids >= 0]
|
|
1308
|
+
fp_units = np.unique(fp_dataframe.units)
|
|
1309
|
+
assert np.isin(fp_units, tp_unit_ids).all()
|
|
1310
|
+
has_fp = np.isin(tp_unit_ids, fp_units)
|
|
1311
|
+
logger.dartsortdebug(f"{has_fp.mean() * 100:.1f}% of units had FPs")
|
|
1312
|
+
|
|
1313
|
+
# initialize the threshold with a min factor
|
|
1314
|
+
threshold = max(min_threshold, min_threshold_factor * template_normsqs.min())
|
|
1315
|
+
logger.dartsortdebug(f"fp control: min possible {threshold=}")
|
|
1316
|
+
if not len(fp_dataframe):
|
|
1317
|
+
return threshold
|
|
1318
|
+
|
|
1319
|
+
# resolution-spaced grid which will be used for searches below
|
|
1320
|
+
start = resolution * np.ceil(threshold / resolution)
|
|
1321
|
+
end = resolution * np.ceil(fp_dataframe.scores.max() / resolution)
|
|
1322
|
+
domain = np.arange(start, end + resolution, resolution)
|
|
1323
|
+
domaini = 0
|
|
1324
|
+
|
|
1325
|
+
# account for different time ranges
|
|
1326
|
+
fp_per_tp = fp_num_frames / (clustering_subsampling_rate * clustering_num_frames)
|
|
1327
|
+
|
|
1328
|
+
for uid, tp in zip(tp_unit_ids[has_fp], tp_counts[has_fp]):
|
|
1329
|
+
in_uid = fp_dataframe.units == uid
|
|
1330
|
+
if not in_uid.any():
|
|
1331
|
+
continue
|
|
1332
|
+
|
|
1333
|
+
# does the unit even have any fps at this threshold?
|
|
1334
|
+
fp_scores = fp_dataframe[in_uid].scores.values
|
|
1335
|
+
if threshold > fp_scores.max():
|
|
1336
|
+
continue
|
|
1337
|
+
|
|
1338
|
+
# largest number of false positives we can allow in this unit
|
|
1339
|
+
max_fp = max_fp_per_input_spike * tp * fp_per_tp
|
|
1340
|
+
|
|
1341
|
+
# check if we are already under the limit
|
|
1342
|
+
if fp_scores.size <= max_fp:
|
|
1343
|
+
continue
|
|
1344
|
+
|
|
1345
|
+
# pick the smallest threshold such that the estimated fp is < max_fp
|
|
1346
|
+
fp_scores.sort()
|
|
1347
|
+
fp_ix = 0
|
|
1348
|
+
for i, x in enumerate(domain[domaini:]):
|
|
1349
|
+
rel_ix = np.searchsorted(fp_scores[fp_ix:], x, side="left")
|
|
1350
|
+
if rel_ix == 0:
|
|
1351
|
+
continue
|
|
1352
|
+
fp_ix += rel_ix
|
|
1353
|
+
n_fp = fp_scores.size - fp_ix
|
|
1354
|
+
if n_fp <= max_fp:
|
|
1355
|
+
assert x >= threshold
|
|
1356
|
+
domaini += i
|
|
1357
|
+
threshold = x
|
|
1358
|
+
logger.dartsortdebug(f"fp control: new {threshold=}")
|
|
1359
|
+
break
|
|
1360
|
+
else:
|
|
1361
|
+
# a break should always be hit thanks to the continues above
|
|
1362
|
+
assert False
|
|
1363
|
+
|
|
1364
|
+
return threshold
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
def fp_control_threshold_from_h5(
|
|
1368
|
+
hdf5_path,
|
|
1369
|
+
low_rank_templates,
|
|
1370
|
+
unit_ids=None,
|
|
1371
|
+
spike_counts=None,
|
|
1372
|
+
rg: int | np.random.Generator = 0,
|
|
1373
|
+
refractory_radius_frames=10,
|
|
1374
|
+
num_frames=0,
|
|
1375
|
+
max_fp_per_input_spike=1.0,
|
|
1376
|
+
):
|
|
1377
|
+
from ..util.data_util import get_residual_snips
|
|
1378
|
+
|
|
1379
|
+
resids = get_residual_snips(hdf5_path)
|
|
1380
|
+
noise = StationaryFactorizedNoise.estimate(resids)
|
|
1381
|
+
|
|
1382
|
+
# be reproducible
|
|
1383
|
+
rg = np.random.default_rng(rg)
|
|
1384
|
+
generator = spiketorch.spawn_torch_rg(rg)
|
|
1385
|
+
|
|
1386
|
+
# restrict my low rank templates to usual geom...
|
|
1387
|
+
# TODO: this is not a very logical way to handle drift here...
|
|
1388
|
+
|
|
1389
|
+
# simulate fps
|
|
1390
|
+
fp_res = noise.unit_false_positives(
|
|
1391
|
+
low_rank_templates,
|
|
1392
|
+
radius=refractory_radius_frames,
|
|
1393
|
+
generator=generator,
|
|
1394
|
+
)
|
|
1395
|
+
return fp_control_threshold(
|
|
1396
|
+
fp_res["fp_dataframe"],
|
|
1397
|
+
fp_res["total_samples"],
|
|
1398
|
+
tp_unit_ids=unit_ids,
|
|
1399
|
+
tp_counts=spike_counts,
|
|
1400
|
+
clustering_num_frames=num_frames,
|
|
1401
|
+
template_normsqs=fp_res["normsq"],
|
|
1402
|
+
max_fp_per_input_spike=max_fp_per_input_spike,
|
|
1403
|
+
)
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def whitener_from_hdf5(
|
|
1407
|
+
hdf5_path,
|
|
1408
|
+
motion_est=None,
|
|
1409
|
+
interp_params: InterpolationParams = default_interpolation_params,
|
|
1410
|
+
device=None,
|
|
1411
|
+
rgeom=None,
|
|
1412
|
+
eps=1e-4,
|
|
1413
|
+
corr_mode=False,
|
|
1414
|
+
):
|
|
1415
|
+
from dartsort.util.drift_util import registered_geometry
|
|
1416
|
+
|
|
1417
|
+
with h5py.File(hdf5_path, "r", locking=False) as h5:
|
|
1418
|
+
geom = cast(h5py.Dataset, h5["geom"])[:]
|
|
1419
|
+
if rgeom is None:
|
|
1420
|
+
rgeom = geom
|
|
1421
|
+
if motion_est is not None:
|
|
1422
|
+
rgeom = registered_geometry(geom, motion_est=motion_est)
|
|
1423
|
+
|
|
1424
|
+
snippets = interpolate_residual_snippets(
|
|
1425
|
+
motion_est,
|
|
1426
|
+
hdf5_path,
|
|
1427
|
+
geom.astype(np.float32),
|
|
1428
|
+
rgeom.astype(np.float32),
|
|
1429
|
+
interp_params=interp_params,
|
|
1430
|
+
device=device,
|
|
1431
|
+
do_tpca=False,
|
|
1432
|
+
)
|
|
1433
|
+
|
|
1434
|
+
x = np.ascontiguousarray(snippets.cpu(), dtype=np.float64)
|
|
1435
|
+
x = x.reshape(-1)
|
|
1436
|
+
invalid = np.flatnonzero(np.isnan(x))
|
|
1437
|
+
if invalid.size:
|
|
1438
|
+
rg = np.random.default_rng(0)
|
|
1439
|
+
x[invalid] = rg.normal(size=invalid.size).astype(x.dtype)
|
|
1440
|
+
x = x.reshape(-1, len(rgeom))
|
|
1441
|
+
if corr_mode:
|
|
1442
|
+
std = np.std(x, axis=0)
|
|
1443
|
+
x /= std
|
|
1444
|
+
_, S, Vh = np.linalg.svd(x, full_matrices=False)
|
|
1445
|
+
assert Vh.shape == (len(rgeom), len(rgeom))
|
|
1446
|
+
assert S.shape == (len(rgeom),)
|
|
1447
|
+
std = S / np.sqrt(len(x) - 1.0) + eps
|
|
1448
|
+
logger.dartsortdebug(
|
|
1449
|
+
"Whitening standard deviations: min,mean,max="
|
|
1450
|
+
f"{std.min().item():0.5f},{std.mean().item():0.5f},{std.max().item():0.5f}"
|
|
1451
|
+
)
|
|
1452
|
+
|
|
1453
|
+
wstd = 1.0 / std
|
|
1454
|
+
# life is easier if we try not to change the units
|
|
1455
|
+
# too much in the whitened data domain
|
|
1456
|
+
wstd /= np.median(wstd)
|
|
1457
|
+
|
|
1458
|
+
# using the zca whitener here. symmetry takes a load off the mind.
|
|
1459
|
+
whitener = (Vh * wstd) @ Vh.T
|
|
1460
|
+
|
|
1461
|
+
return whitener.astype(np.float32)
|