DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1461 @@
1
+ import warnings
2
+ from typing import cast
3
+
4
+ import h5py
5
+ import linear_operator
6
+ import numpy as np
7
+ import pandas as pd
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from linear_operator import operators
11
+ from scipy.fftpack import next_fast_len
12
+ from scipy.spatial.distance import pdist, squareform
13
+ from scipy.stats import norm
14
+ from sklearn.covariance import GraphicalLassoCV, graphical_lasso
15
+ from tqdm.auto import trange
16
+
17
+ from ..util import more_operators, spiketorch
18
+ from ..util.interpolation_util import InterpolationParams, default_interpolation_params
19
+ from ..util.logging_util import DARTSORTDEBUG, get_logger
20
+
21
+ logger = get_logger(__name__)
22
+
23
+
24
+ class FullNoise(torch.nn.Module):
25
+ """Do not use this, it's just for comparison to the others."""
26
+
27
+ def __init__(self, std, vt, nt, nc):
28
+ super().__init__()
29
+ self.std = std
30
+ self.vt = vt
31
+ self.nt = nt
32
+ self.nc = nc
33
+
34
+ def full_covariance(self):
35
+ return (self.vt.T * self.std.square()) @ self.vt
36
+
37
+ def simulate(self, size=1, t=None, generator=None):
38
+ device = cast(torch.device, self.spatial_std.device)
39
+ assert t == self.nt
40
+ samples = torch.randn(size, t * self.nc, generator=generator, device=device)
41
+ samples = torch.einsum(
42
+ "ni,i,ij->nj",
43
+ samples,
44
+ self.std,
45
+ self.vt,
46
+ )
47
+ return samples.view(size, self.nt, self.nc)
48
+
49
+ @classmethod
50
+ def estimate(cls, snippets):
51
+ snippets = torch.asarray(snippets)
52
+ snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
53
+
54
+ n, t, c = snippets.shape
55
+
56
+ x = snippets.view(n, t * c)
57
+ # u_full, sing, vt_full = torch.linalg.svd(x, full_matrices=False)
58
+ # std = sing / torch.sqrt(torch.tensor(n - 1).to(snippets))
59
+
60
+ cov = torch.cov(x.T)
61
+ eigvals, v = torch.linalg.eigh(cov)
62
+ std = eigvals.sqrt()
63
+ vt = v.T.contiguous()
64
+
65
+ return cls(std, vt, nt=t, nc=c)
66
+
67
+
68
+ class FactorizedNoise(torch.nn.Module):
69
+ """Spatial/temporal factorized noise. See .estimate()."""
70
+
71
+ def __init__(self, spatial_std, vt_spatial, temporal_std, vt_temporal):
72
+ super().__init__()
73
+ self.spatial_std = spatial_std
74
+ self.vt_spatial = vt_spatial
75
+ self.temporal_std = temporal_std
76
+ self.vt_temporal = vt_temporal
77
+
78
+ def full_covariance(self):
79
+ spatial_cov = self.spatial_std[:, None] * self.vt_spatial
80
+ spatial_cov = spatial_cov.T @ spatial_cov
81
+ temporal_cov = self.temporal_std[:, None] * self.vt_temporal
82
+ temporal_cov = temporal_cov.T @ temporal_cov
83
+ return torch.kron(temporal_cov, spatial_cov)
84
+
85
+ def simulate(self, size=1, t=None, generator=None):
86
+ c = self.spatial_std.numel()
87
+ t_ = self.temporal_std.numel()
88
+ device = self.spatial_std.device
89
+ if t is None:
90
+ t = t_
91
+ assert t == t_
92
+ samples = torch.randn(size, t, c, generator=generator, device=device)
93
+ return torch.einsum(
94
+ "ntc,t,tu,c,cd->nud",
95
+ samples,
96
+ self.temporal_std,
97
+ self.vt_temporal,
98
+ self.spatial_std,
99
+ self.vt_spatial,
100
+ )
101
+
102
+ @classmethod
103
+ def estimate(cls, snippets):
104
+ """Estimate factorized spatiotemporal noise covariance
105
+
106
+ If snippets has shape (n, t, c), the full t*c x t*c covariance is
107
+ too big. Let's say it factorizes as
108
+ spatial_cov[c, c'] * temporal_cov[t, t'].
109
+ This is not identifiable, but it's okay if we assume that the global
110
+ scale falls into the spatial part (say). So, we first estimate the
111
+ spatial cov, then apply spatial whitening to the snippets, and finally
112
+ estimate the temporal cov.
113
+
114
+ This is an SVD reimplementation of a Yass fn which used covariance matrices.
115
+
116
+ Arguments
117
+ ---------
118
+ snippets : torch.Tensor
119
+ (n, t, c) array of noise snippets. If you don't want spikes in there,
120
+ this function won't help you with that.
121
+ """
122
+ snippets = torch.asarray(snippets)
123
+ snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
124
+
125
+ n, t, c = snippets.shape
126
+ sqrt_nt_minus_1 = torch.tensor(n * t - 1, dtype=snippets.dtype).sqrt()
127
+ sqrt_nc_minus_1 = torch.tensor(n * c - 1, dtype=snippets.dtype).sqrt()
128
+ assert n * t > c**2
129
+ assert n * c > t**2
130
+
131
+ # estimate spatial covariance
132
+ x_spatial = snippets.view(n * t, c)
133
+ u_spatial, spatial_sing, vt_spatial = torch.linalg.svd(
134
+ x_spatial, full_matrices=False
135
+ )
136
+ spatial_std = spatial_sing / sqrt_nt_minus_1
137
+
138
+ # extract whitened temporal snips
139
+ x_temporal = u_spatial.view(n, t, c).permute(0, 2, 1).reshape(n * c, t)
140
+ x_temporal.mul_(sqrt_nt_minus_1)
141
+ _, temporal_sing, vt_temporal = torch.linalg.svd(
142
+ x_temporal, full_matrices=False
143
+ )
144
+ del _
145
+ temporal_std = temporal_sing / sqrt_nc_minus_1
146
+
147
+ return cls(spatial_std, vt_spatial, temporal_std, vt_temporal)
148
+
149
+
150
+ class WhiteNoise(torch.nn.Module):
151
+ """White noise to mimic the StationaryFactorizedNoise for use in sims."""
152
+
153
+ margin = 0
154
+
155
+ def __init__(self, n_channels, scale=1.0):
156
+ super().__init__()
157
+ self.n_channels = n_channels
158
+ self.scale = scale
159
+
160
+ def unwhiten(self, snippet):
161
+ return snippet
162
+
163
+ def simulate(self, size=1, t=None, generator=None, chunk_t=None):
164
+ assert t is not None
165
+ x = torch.randn(size, t, self.n_channels, generator=generator)
166
+ if self.scale != 1.0:
167
+ x *= self.scale
168
+ return x
169
+
170
+
171
+ class StationaryFactorizedNoise(torch.nn.Module):
172
+ def __init__(self, spatial_std, vt_spatial, kernel_fft, block_size, t):
173
+ super().__init__()
174
+ self.spatial_std = torch.asarray(spatial_std)
175
+ self.vt_spatial = torch.asarray(vt_spatial)
176
+ self.kernel_fft = torch.asarray(kernel_fft)
177
+ self.block_size = block_size
178
+ self.t = t
179
+ self.margin = (self.t - 1) // 2
180
+
181
+ def spatial_cov(self):
182
+ rt = self.spatial_std * self.vt_spatial.T
183
+ return rt @ rt.T
184
+
185
+ def whiten(self, snippet):
186
+ wsnip = snippet @ (self.vt_spatial / self.spatial_std[:, None])
187
+ wsnip = spiketorch.single_inv_oaconv1d(
188
+ wsnip.T,
189
+ s2=self.t,
190
+ f2=1 / self.kernel_fft,
191
+ block_size=self.block_size,
192
+ norm="ortho",
193
+ )
194
+ # zca
195
+ wsnip = wsnip.T @ self.vt_spatial.T
196
+ return wsnip
197
+
198
+ def unwhiten(self, snippet):
199
+ snippet = torch.asarray(
200
+ snippet, device=self.spatial_std.device, dtype=self.spatial_std.dtype
201
+ )
202
+ flat = snippet.ndim == 2
203
+ if flat:
204
+ snippet = snippet[None]
205
+ size, t_padded, c = snippet.shape
206
+ t = t_padded - self.t + 1
207
+ out = snippet.new_empty(size, c, t)
208
+ for j in range(size):
209
+ out[j] = spiketorch.single_inv_oaconv1d(
210
+ snippet[j].T,
211
+ s2=self.t,
212
+ f2=self.kernel_fft,
213
+ block_size=self.block_size,
214
+ norm="ortho",
215
+ )
216
+ spatial_part = self.spatial_std[:, None] * self.vt_spatial
217
+ out = torch.einsum("nct,cd->ntd", out, spatial_part)
218
+ if flat:
219
+ out = out[0]
220
+ return out
221
+
222
+ def simulate(self, size=1, t=None, generator=None, chunk_t=None):
223
+ """Simulate stationary factorized noise
224
+
225
+ White noise is the same in FFT space or not. So we start there,
226
+ filter, inverse FFT, then spatially dewhiten. The filtering + inverse
227
+ FFT are basically overlap-add convolution, and thanks to Chris Langfield
228
+ for help with overlap-add.
229
+ """
230
+ if t is None:
231
+ t = self.t
232
+ assert t >= self.t
233
+ c = self.spatial_std.numel()
234
+ device = self.spatial_std.device
235
+
236
+ if chunk_t:
237
+ out = torch.zeros((t + self.t, c))
238
+ taper = torch.linspace(0.0, 1.0, steps=self.t)
239
+ for bs in trange(0, t, chunk_t):
240
+ chunk = self.simulate(t=chunk_t + self.t, generator=generator)[0]
241
+ if bs > 0:
242
+ chunk[: self.t] *= taper[:, None]
243
+ chunk[-self.t :] *= (1 - taper)[:, None]
244
+ out[bs : bs + chunk_t + self.t] += chunk.cpu()
245
+ return out[None, : -self.t]
246
+
247
+ # need extra room at the edges to do valid convolution
248
+ t_padded = t + self.t - 1
249
+ noise = torch.randn(size, t_padded, c, generator=generator, device=device)
250
+ return self.unwhiten(noise)
251
+
252
+ @classmethod
253
+ def estimate(cls, snippets):
254
+ """Estimate factorized temporally stationary noise
255
+
256
+ When simulating long samples, the above approach can't scale. Here let
257
+ spatial_cov[c, c'] * temporal_cov[t - t'].
258
+
259
+ Now, remember that stationary kernels diagonalize in Fourier basis. FFT
260
+ is the eigh of circulant things. So we use an FFT based version of the
261
+ SVD algorithm above.
262
+
263
+ Arguments
264
+ ---------
265
+ snippets : torch.Tensor
266
+ (n, t, c) array of noise snippets. If you don't want spikes in there,
267
+ this function won't help you with that.
268
+ """
269
+ snippets = torch.asarray(snippets)
270
+ snippets = snippets.to(torch.promote_types(snippets.dtype, torch.float))
271
+
272
+ n, t, c = snippets.shape
273
+ sqrt_nt_minus_1 = torch.tensor(n * t - 1, dtype=snippets.dtype).sqrt()
274
+ assert n * t > c**2
275
+ assert n * c > t
276
+
277
+ # estimate spatial covariance
278
+ x_spatial = snippets.view(n * t, c)
279
+ u_spatial, spatial_sing, vt_spatial = torch.linalg.svd(
280
+ x_spatial, full_matrices=False
281
+ )
282
+ spatial_std = spatial_sing / sqrt_nt_minus_1
283
+
284
+ # extract whitened temporal snips
285
+ x_temporal = u_spatial.view(n, t, c).permute(0, 2, 1).reshape(n * c, t)
286
+ x_temporal.mul_(sqrt_nt_minus_1)
287
+
288
+ # estimate stationary temporal covariance via FFT
289
+ block_size = next_fast_len(t)
290
+ xt_fft = torch.fft.rfft(x_temporal, n=block_size, norm="ortho")
291
+ kernel_fft = (xt_fft * xt_fft.conj()).mean(0).sqrt_()
292
+
293
+ return cls(spatial_std, vt_spatial, kernel_fft, block_size, t)
294
+
295
+ def unit_false_positives(
296
+ self,
297
+ low_rank_templates,
298
+ min_threshold=5.0,
299
+ radius=10,
300
+ generator=None,
301
+ size=100,
302
+ t=4096,
303
+ ):
304
+ """Run template detection in noise residual to check for false positives
305
+
306
+ Given a collection of (SVD factorized) templates, compute their template
307
+ matching objective in simulated recording snippets drawn according to
308
+ this noise object's distribution. Then, detect peaks (with a refractory
309
+ condition) to estimate the false positive count.
310
+
311
+ Arguments
312
+ ---------
313
+ low_rank_templates : LowRankTemplates
314
+ min_threshold : float
315
+ The bare minimum deconv objective
316
+ radius : int
317
+ Refractory period
318
+ generator : torch.Generator
319
+ For reproducibility.
320
+ size : int
321
+ Number of recording snipppets
322
+ t : int
323
+ Length of each recording snippet
324
+
325
+ Returns
326
+ -------
327
+ total_samples : int
328
+ Total number of valid temporal samples
329
+ fp_dataframe : pd.DataFrame
330
+ With columns `units` and `scores`
331
+ """
332
+ from ..detect import detect_and_deduplicate
333
+
334
+ singular = torch.asarray(
335
+ low_rank_templates.singular_values,
336
+ device=self.spatial_std.device,
337
+ dtype=self.spatial_std.dtype,
338
+ )
339
+ spatial = torch.asarray(
340
+ low_rank_templates.spatial_components,
341
+ device=self.spatial_std.device,
342
+ dtype=self.spatial_std.dtype,
343
+ )
344
+ spatial_singular = singular.unsqueeze(-1) * spatial
345
+ temporal = torch.asarray(
346
+ low_rank_templates.temporal_components,
347
+ device=self.spatial_std.device,
348
+ dtype=self.spatial_std.dtype,
349
+ )
350
+ normsq = singular.square().sum(1)
351
+ negnormsq = normsq.neg().unsqueeze(1)
352
+ nu, nt = temporal.shape[:2]
353
+ obj = None # var for reusing buffers
354
+ units = []
355
+ scores = []
356
+ for j in trange(size, desc="False positives"):
357
+ # note: simulating with padding so that the valid conv has length t.
358
+ sample = self.simulate(t=t + nt - 1, generator=generator)[0].T
359
+ assert sample.isfinite().all()
360
+ obj = spiketorch.convolve_lowrank(
361
+ sample,
362
+ spatial_singular,
363
+ temporal,
364
+ out=obj,
365
+ padding=0,
366
+ )
367
+ assert obj.isfinite().all()
368
+ assert obj.shape == (nu, t)
369
+ obj = torch.add(negnormsq, obj, alpha=2.0, out=obj)
370
+ assert obj.isfinite().all()
371
+
372
+ # find peaks...
373
+ peak_times, peak_units, peak_energies = detect_and_deduplicate( # type: ignore
374
+ obj.T,
375
+ min_threshold,
376
+ peak_sign="pos",
377
+ relative_peak_radius=1,
378
+ dedup_temporal_radius=radius,
379
+ return_energies=True,
380
+ )
381
+ assert peak_energies.isfinite().all()
382
+ units.append(peak_units.numpy(force=True))
383
+ scores.append(peak_energies.numpy(force=True))
384
+
385
+ total_samples = size * t
386
+ data = dict(units=np.concatenate(units), scores=np.concatenate(scores))
387
+ fp_dataframe = pd.DataFrame(data)
388
+
389
+ return dict(
390
+ total_samples=total_samples,
391
+ fp_dataframe=fp_dataframe,
392
+ normsq=normsq.numpy(force=True),
393
+ )
394
+
395
+
396
+ class EmbeddedNoise(torch.nn.Module):
397
+ """Handles computations related to noise in TPCA space.
398
+
399
+ Can have a couple of kinds of mean. mean_kind == ...
400
+ - "zero": noise was already centered, my mean is 0
401
+ - "by_rank": same mean on all channels
402
+ - "full": value per rank, chan
403
+
404
+ Default is zero, because we usually do not do "centering" in our TPCA
405
+ (i.e., it is just linear not affine). Since the channels are highpassed,
406
+ they have mean 0, and still do in TPCA space.
407
+
408
+ And cov_kind = ...
409
+ - "scalar": one global variance
410
+ - "diagonal_by_rank": same variance across chans, varies by rank
411
+ - "diagonal": value per rank, chan
412
+ - "factorized": kronecker prod of dense rank and chan factors
413
+ - "factorized_by_rank": same, but chan factor varies by rank
414
+ - "factorized_rank_diag" : factorized, but rank factor is diagonal
415
+ - "factorized_by_rank_rank_diag" : factorized_by_rank, but rank factor is diagonal
416
+ this one is block diagonal and therefore nicer than factorized_by_rank.
417
+ - "full": not generally practical.
418
+ """
419
+
420
+ def __init__(
421
+ self,
422
+ rank,
423
+ n_channels,
424
+ mean_kind="zero",
425
+ cov_kind="factorized_by_rank_rank_diag",
426
+ mean=None,
427
+ global_std=None,
428
+ rank_std=None,
429
+ full_std=None,
430
+ rank_vt=None,
431
+ channel_std=None,
432
+ channel_vt=None,
433
+ full_cov=None,
434
+ zero_radius=None,
435
+ ):
436
+ super().__init__()
437
+ self.rank = rank
438
+ self.n_channels = n_channels
439
+ self.mean_kind = mean_kind
440
+ self.cov_kind = cov_kind
441
+ self.D = rank * n_channels
442
+ self.zero_radius = zero_radius
443
+
444
+ self.register_buffer("chans_arange", torch.arange(n_channels))
445
+ self.register_buffer("global_std", global_std)
446
+
447
+ device = None
448
+ if global_std is not None:
449
+ device = global_std.device
450
+
451
+ if mean is not None:
452
+ self.register_buffer("mean", mean)
453
+ if rank_std is not None:
454
+ self.register_buffer("rank_std", rank_std)
455
+ if channel_std is not None:
456
+ self.register_buffer("channel_std", channel_std)
457
+ if full_std is not None:
458
+ self.register_buffer("full_std", full_std)
459
+ if rank_vt is not None:
460
+ self.register_buffer("rank_vt", rank_vt)
461
+ if channel_vt is not None:
462
+ self.register_buffer("channel_vt", channel_vt)
463
+ self.register_buffer("channel_vt_zpad", F.pad(channel_vt, (0, 1)))
464
+ if full_cov is not None:
465
+ self.register_buffer("full_cov", full_cov)
466
+ if device is None:
467
+ device = full_cov.device
468
+
469
+ # precompute stuff
470
+ self._full_cov = None
471
+ self._full_covinvcov = None
472
+ self._full_inverse = None
473
+ self._full_whitener = None
474
+ self._logdet = None
475
+ self.register_buffer("mean_full", self.mean_rc().clone().detach())
476
+ self.cache = {}
477
+ self.to(device)
478
+
479
+ @property
480
+ def logdet(self):
481
+ if self._logdet is None:
482
+ self.marginal_covariance()
483
+ return self._logdet
484
+
485
+ @property
486
+ def device(self):
487
+ return self.chans_arange.device
488
+
489
+ def mean_rc(self) -> torch.Tensor:
490
+ """Return noise mean as a rank x channels tensor"""
491
+ shape = self.rank, self.n_channels
492
+ if self.mean_kind == "zero":
493
+ return torch.zeros(shape)
494
+ elif self.mean_kind == "by_rank":
495
+ return (
496
+ cast(torch.Tensor, self.mean)[:, None].broadcast_to(shape).contiguous()
497
+ )
498
+ elif self.mean_kind == "full":
499
+ return cast(torch.Tensor, self.mean)
500
+ else:
501
+ assert False
502
+
503
+ def marginal_mean(self):
504
+ """Return noise mean as a rank x channels tensor"""
505
+ shape = self.rank, self.n_channels
506
+ if self.mean_kind == "zero":
507
+ return torch.zeros(shape)
508
+ if self.mean_kind == "by_rank":
509
+ return (
510
+ cast(torch.Tensor, self.mean)[:, None].broadcast_to(shape).contiguous()
511
+ )
512
+ if self.mean_kind == "full":
513
+ return self.mean
514
+ assert False
515
+
516
+ def whitener(self, channels=slice(None)):
517
+ cov = self.marginal_covariance(channels=channels)
518
+ chol = cov.cholesky().to_dense()
519
+ chans_eye = torch.eye(cov.shape[0], dtype=cov.dtype, device=cov.device)
520
+ whitener = torch.linalg.solve_triangular(chol, chans_eye, upper=False)
521
+ return whitener.reshape(cov.shape)
522
+
523
+ def whiten(self, data, channels=slice(None)):
524
+ assert self.mean_kind == "zero"
525
+ cov = self.marginal_covariance(channels=channels)
526
+ assert data.ndim == 3
527
+ data = data.reshape(len(data), -1)
528
+ chol = cov.cholesky().to_dense()
529
+ res = torch.linalg.solve_triangular(chol, data.T, upper=False)
530
+ res = res.T.unsqueeze(2)
531
+
532
+ # contour integral quad with lanczos is too fancy here...
533
+ # res = linear_operator.sqrt_inv_matmul(cov, data.unsqueeze(2))
534
+ assert res.ndim == 3 and res.shape == (*data.shape, 1)
535
+ return res
536
+
537
+ def full_dense_cov(self, device=None):
538
+ if self._full_cov is None:
539
+ self.marginal_covariance(device=device)
540
+ if device is not None:
541
+ self._full_cov_dense = self._full_cov_dense.to(device)
542
+ return self._full_cov_dense
543
+
544
+ def full_covinvcov(self, device=None):
545
+ if self._full_cov is None:
546
+ self.marginal_covariance(device=device)
547
+ assert self._full_cov is not None
548
+ if self._full_covinvcov is None:
549
+ self._full_covinvcov = self._full_cov.solve(self.full_dense_cov())
550
+ if device is not None:
551
+ self._full_covinvcov = self._full_covinvcov.to(device)
552
+ return self._full_covinvcov
553
+
554
+ def full_inverse(self, device=None):
555
+ if self._full_cov is None:
556
+ self.marginal_covariance(device=device)
557
+ assert self._full_cov is not None
558
+ if self._full_inverse is None:
559
+ self._full_inverse = self._full_cov.inverse().to_dense()
560
+ if device is not None:
561
+ self._full_inverse = self._full_inverse.to(device)
562
+ return self._full_inverse
563
+
564
+ def cov_batch_mul(
565
+ self, x: torch.Tensor, channels_src: torch.Tensor, channels_targ: torch.Tensor
566
+ ):
567
+ assert x.ndim == 3
568
+ n = x.shape[0]
569
+ assert x.shape[2] == channels_src.shape[1]
570
+ assert channels_src.ndim == 2
571
+ assert channels_src.shape[0] == n
572
+ if channels_targ.ndim == 2:
573
+ assert channels_targ.shape[0] == n
574
+ else:
575
+ assert channels_targ.ndim == 1
576
+
577
+ if self.cov_kind == "factorized":
578
+ rank_root = self.rank_vt.T * self.rank_std # type: ignore
579
+ rank_cov = cast(torch.Tensor, rank_root @ rank_root.T)
580
+ rank_cov = rank_cov[None].broadcast_to(n, *rank_cov.shape)
581
+ x = rank_cov.bmm(x)
582
+
583
+ chan_root_left = self.channel_vt_zpad.T[channels_src] * self.channel_std # type: ignore
584
+ chan_root_right = self.channel_vt_zpad.T[channels_targ] * self.channel_std # type: ignore
585
+ if chan_root_right.ndim == 2:
586
+ targ_shp = (chan_root_left.shape[0], *chan_root_right.shape)
587
+ chan_root_right = chan_root_right[None].broadcast_to(targ_shp)
588
+ chan_cov = chan_root_left.bmm(chan_root_right.mT)
589
+ return x.bmm(chan_cov)
590
+ elif self.cov_kind == "scalar":
591
+ if channels_targ.ndim == 1:
592
+ channels_targ = channels_targ[None, :].broadcast_to(
593
+ (n, channels_targ.numel())
594
+ )
595
+ channels_src = torch.where(channels_src == self.n_channels, -2, channels_src)
596
+ eyes = (channels_src[:, :, None] == channels_targ[:, None, :]).float()
597
+ chan_cov = eyes * self.global_std**2 # type: ignore
598
+ return x.bmm(chan_cov)
599
+ elif self.cov_kind == "full":
600
+ # this branch is for debugging. it is slow.
601
+ if channels_targ.ndim == 1:
602
+ channels_targ = channels_targ[None, :].broadcast_to(
603
+ (n, channels_targ.numel())
604
+ )
605
+ cov_zpad = F.pad(self.full_cov, (0, 1, 0, 0, 0, 1))[None] # type: ignore
606
+ cov = cov_zpad.take_along_dim( # type: ignore
607
+ indices=channels_src[:, None, :, None, None], dim=2
608
+ )
609
+ assert cov.shape == (
610
+ n,
611
+ self.rank,
612
+ channels_src.shape[1],
613
+ self.rank,
614
+ self.n_channels + 1,
615
+ )
616
+ cov = cov.take_along_dim(
617
+ indices=channels_targ[:, None, None, None, :], dim=4
618
+ )
619
+ assert cov.shape == (
620
+ n,
621
+ self.rank,
622
+ channels_src.shape[1],
623
+ self.rank,
624
+ channels_targ.shape[1],
625
+ )
626
+ cov = cov.view(
627
+ n,
628
+ self.rank * channels_src.shape[1],
629
+ self.rank * channels_targ.shape[1],
630
+ )
631
+ x = x.view(n, 1, -1).bmm(cov)
632
+ return x.view(n, self.rank, channels_targ.shape[1])
633
+ else:
634
+ raise NotImplementedError(
635
+ f"Need to implement cov_batch_mul for {self.cov_kind=}."
636
+ )
637
+
638
+ def cov_batch(self, channels_left: torch.Tensor, channels_right: torch.Tensor):
639
+ if self.cov_kind != "factorized":
640
+ raise NotImplementedError(
641
+ f"Need to implement cov_batch for {self.cov_kind=}."
642
+ )
643
+ rank_root = self.rank_vt.T * self.rank_std
644
+ rank_cov = rank_root @ rank_root.T
645
+ chan_root_left = self.channel_vt_zpad.T[channels_left] * self.channel_std
646
+ chan_root_right = self.channel_vt_zpad.T[channels_right] * self.channel_std
647
+ if chan_root_right.ndim == 2:
648
+ chan_root_right = chan_root_right[None].broadcast_to(
649
+ chan_root_left.shape[0], *chan_root_right.shape
650
+ )
651
+ chan_cov = chan_root_left.bmm(chan_root_right.mT)
652
+ rank_cov = rank_cov[None].broadcast_to(
653
+ channels_left.shape[0], self.rank, self.rank
654
+ )
655
+ return operators.KroneckerProductLinearOperator(rank_cov, chan_cov)
656
+
657
+ def marginal_covariance(
658
+ self,
659
+ channels: torch.Tensor | slice | None = slice(None),
660
+ cache_prefix=None,
661
+ cache_key=None,
662
+ device=None,
663
+ ):
664
+ if device is not None:
665
+ device = torch.device(device)
666
+ if cache_key is not None and torch.is_tensor(cache_key):
667
+ cache_key = cache_key.item()
668
+ if cache_prefix is not None:
669
+ cache_key = (cache_prefix, cache_key)
670
+ if cache_key is not None and cache_key in self.cache:
671
+ res = self.cache[cache_key]
672
+ if device is not None and device != res.device:
673
+ res = res.to(device)
674
+ self.cache[cache_key] = res
675
+ # TODO: why is this necessary? I thought that calling solve would cache this.
676
+ if not hasattr(res, "_memoize_cache"):
677
+ res.cholesky()
678
+ return res
679
+ if channels is None or channels == slice(None):
680
+ if self._full_cov is None:
681
+ fcov = self._marginal_covariance()
682
+ self._full_cov_dense = fcov.to_dense()
683
+ if not isinstance(fcov, operators.ConstantDiagLinearOperator):
684
+ fcov = operators.CholLinearOperator(fcov.cholesky())
685
+ self._full_cov = fcov
686
+ self._logdet = self._full_cov.logdet()
687
+ if device is not None and device != self._full_cov.device:
688
+ self._full_cov = self._full_cov.to(device)
689
+ self._logdet = self._logdet.to(device)
690
+ return self._full_cov
691
+ cov = self._marginal_covariance(channels)
692
+ if device is not None and device != cov.device:
693
+ cov = cov.to(device)
694
+ if cache_key is not None:
695
+ self.cache[cache_key] = cov
696
+ return cov
697
+
698
+ def offdiag_covariance(
699
+ self, channels_left=slice(None), channels_right=slice(None), device=None
700
+ ):
701
+ odc = self._marginal_covariance(
702
+ channels=channels_right, channels_left=channels_left
703
+ )
704
+ if device is not None:
705
+ odc = odc.to(device)
706
+ return odc
707
+
708
+ def _marginal_covariance(self, channels=slice(None), channels_left=None):
709
+ channels = self.chans_arange[channels]
710
+ have_left = channels_left is not None
711
+ if channels_left is None:
712
+ channels_left = channels
713
+ nc = channels.numel()
714
+ ncl = channels_left.numel()
715
+
716
+ if have_left:
717
+ if self.cov_kind in ("scalar", "diagonal_by_rank", "diagonal"):
718
+ # if torch.isin(channels_left.to(channels), channels).any():
719
+ # raise ValueError(
720
+ # "Don't know how to get non-marginal covariance block "
721
+ # f"for cov_kind={self.cov_kind} when the block overlaps "
722
+ # "with the diagonal."
723
+ # )
724
+ sz = self.rank * ncl, self.rank * nc
725
+ return operators.ZeroLinearOperator(*sz, dtype=self.global_std.dtype)
726
+
727
+ if self.cov_kind == "scalar":
728
+ eye = operators.IdentityLinearOperator(self.rank * nc, device=self.device)
729
+ return self.global_std.square() * eye
730
+
731
+ if self.cov_kind == "diagonal_by_rank":
732
+ rank_diag = operators.DiagLinearOperator(self.rank_std**2)
733
+ chans_eye = operators.IdentityLinearOperator(nc, device=self.device)
734
+ return torch.kron(rank_diag, chans_eye)
735
+
736
+ if self.cov_kind == "diagonal":
737
+ chans_std = self.full_std[:, channels]
738
+ return operators.DiagLinearOperator(chans_std**2)
739
+
740
+ if self.cov_kind == "full":
741
+ marg_cov = self.full_cov[:, channels_left][..., channels]
742
+ r = marg_cov.shape[0]
743
+ marg_cov = marg_cov.reshape(r * ncl, r * nc)
744
+ return linear_operator.to_linear_operator(marg_cov)
745
+
746
+ if self.cov_kind in "factorized":
747
+ rank_root = self.rank_vt.T * self.rank_std
748
+ rank_cov = rank_root @ rank_root.T
749
+ chan_root = self.channel_vt.T[channels] * self.channel_std
750
+ chan_root_right = chan_root.T
751
+ if have_left:
752
+ chan_root = self.channel_vt.T[channels_left] * self.channel_std
753
+ chan_cov = chan_root @ chan_root_right
754
+ return operators.KroneckerProductLinearOperator(rank_cov, chan_cov)
755
+
756
+ if self.cov_kind == "factorized_rank_diag":
757
+ rank_cov = operators.DiagLinearOperator(self.rank_std.square())
758
+ chan_root = self.channel_vt.T[channels] * self.channel_std
759
+ chan_root_right = chan_root.T
760
+ if have_left:
761
+ chan_root = self.channel_vt.T[channels_left] * self.channel_std
762
+ chan_cov = chan_root @ chan_root_right
763
+ return torch.kron(rank_cov, chan_cov)
764
+
765
+ if self.cov_kind == "factorized_by_rank_rank_diag":
766
+ rank_std = self.rank_std.view(self.rank, 1, 1)
767
+ chans_std = self.channel_std[:, :, None] # no slice here!
768
+ rc_std = rank_std * chans_std
769
+ chans_vt = self.channel_vt[:, :, channels]
770
+ chan_rootl = chan_root = rc_std * chans_vt
771
+ if have_left:
772
+ chans_vtl = self.channel_vt[:, :, channels_left]
773
+ chan_rootl = rc_std * chans_vtl
774
+ blocks = torch.bmm(chan_rootl.mT, chan_root)
775
+ blocks = linear_operator.to_linear_operator(blocks)
776
+ if have_left:
777
+ return more_operators.NonSquareBlockLinearOperator(blocks)
778
+ return operators.BlockDiagLinearOperator(blocks)
779
+
780
+ assert False
781
+
782
+ @classmethod
783
+ def estimate(
784
+ cls,
785
+ snippets,
786
+ mean_kind="zero",
787
+ cov_kind="scalar",
788
+ shrinkage=0.0,
789
+ glasso_alpha: int | float | None = None,
790
+ eps=1e-4,
791
+ zero_radius: float | None = None,
792
+ svd_batch_size=256,
793
+ rgeom=None,
794
+ ):
795
+ """Factory method to estimate noise model from TPCA snippets
796
+
797
+ Arguments
798
+ ---------
799
+ snippets : torch.Tensor
800
+ (n, rank, c) array of tpca-embedded noise snippets
801
+ missing values are okay, indicate by NaN please
802
+ """
803
+ n, rank, n_channels = snippets.shape
804
+ init_kw = dict(
805
+ rank=rank, n_channels=n_channels, mean_kind=mean_kind, cov_kind=cov_kind
806
+ )
807
+ x = torch.asarray(snippets)
808
+ x = x.to(torch.promote_types(x.dtype, torch.float))
809
+ if zero_radius is not None or shrinkage:
810
+ assert cov_kind.startswith("factorized")
811
+ assert rgeom is not None
812
+
813
+ # estimate mean and center data
814
+ if mean_kind == "zero":
815
+ mean = None
816
+ elif mean_kind == "by_rank":
817
+ mean = torch.nanmean(x, dim=(0, 2))
818
+ assert mean.isfinite().all()
819
+ x = x - mean.unsqueeze(1)
820
+ elif mean_kind == "full":
821
+ mean = torch.nanmean(x, dim=0)
822
+ mean = torch.where(
823
+ mean.isnan().all(1).unsqueeze(1),
824
+ torch.nanmean(mean, dim=1).unsqueeze(1),
825
+ mean,
826
+ )
827
+ x = x - mean
828
+ else:
829
+ assert False
830
+
831
+ # estimate covs
832
+ # still n, rank, n_channels
833
+ dxsq = x.square()
834
+ full_var = torch.nanmean(dxsq, dim=0)
835
+ rank_var = torch.nanmean(full_var, dim=1)
836
+ rank_std = rank_var.sqrt_()
837
+ assert rank_var.isfinite().all()
838
+ global_var = torch.nanmean(rank_var)
839
+ global_std = global_var.sqrt()
840
+
841
+ if cov_kind == "scalar":
842
+ return cls(mean=mean, global_std=global_std, **init_kw)
843
+
844
+ if cov_kind == "by_rank":
845
+ return cls(mean=mean, global_std=global_std, rank_std=rank_std, **init_kw)
846
+
847
+ if cov_kind == "diagonal":
848
+ full_var = torch.where(
849
+ full_var.isnan().all(1).unsqueeze(1),
850
+ rank_var.unsqueeze(1),
851
+ full_var,
852
+ )
853
+ full_std = full_var.sqrt()
854
+ return cls(mean=mean, global_std=global_std, full_std=full_std, **init_kw)
855
+
856
+ if cov_kind == "full":
857
+ x = x.view(n, rank * n_channels)
858
+ present = torch.isfinite(x).any(dim=0)
859
+ cov = torch.eye(x.shape[1], device=x.device, dtype=x.dtype)
860
+ vcov = spiketorch.nancov(x[:, present], force_posdef=True, eps=eps)
861
+ cov[present[:, None] & present[None, :]] = vcov.view(-1)
862
+ cov = cov.reshape(rank, n_channels, rank, n_channels)
863
+ return cls(mean=mean, global_std=global_std, full_cov=cov, **init_kw)
864
+
865
+ assert cov_kind.startswith("factorized")
866
+ # handle rank part first, then the spatial part
867
+
868
+ # start by getting the rank part of the cov, if necessary, and whitening
869
+ # the ranks to leave behind x_spatial
870
+ if "rank_diag" in cov_kind:
871
+ # rank part is diagonal
872
+ rank_vt = None
873
+ x_spatial = x.div_(rank_std.unsqueeze(1))
874
+ del x
875
+ else:
876
+ # full cov estimation for rank part via svd
877
+ # we have NaNs, but we can get rid of them because channels are either all
878
+ # NaN or not. below, for the spatial part, no such luck and we have to
879
+ # evaluate the covariance in a masked way
880
+
881
+ # doing this on CPU since the SVD can use a lot of memory.
882
+ x_rank = x.permute(0, 2, 1)
883
+ x_rank = x_rank.cpu().reshape(n * n_channels, rank)
884
+ (valid,) = x_rank.isfinite().all(dim=1).nonzero(as_tuple=True)
885
+ x_rankv = x_rank[valid]
886
+ orig_device = x.device
887
+ del x
888
+ assert not x_rankv.requires_grad
889
+ u_rankv, rank_sing, rank_vt = torch.linalg.svd(x_rankv, full_matrices=False)
890
+
891
+ correction = torch.tensor(len(x_rankv) - 1.0).sqrt()
892
+ rank_std = rank_sing / correction
893
+ rank_std = rank_std.to(orig_device)
894
+ correction = correction.to(orig_device)
895
+
896
+ # whitened spatial part -- reuse storage
897
+ x_spatial = x_rank
898
+ del x_rank
899
+ x_spatial[valid] = u_rankv
900
+ x_spatial = x_spatial.reshape(n, n_channels, rank).permute(0, 2, 1)
901
+ x_spatial = x_spatial.to(orig_device)
902
+ x_spatial.mul_(correction)
903
+
904
+ spatial_mask = None
905
+ if zero_radius:
906
+ assert rgeom is not None
907
+ assert rgeom.shape[0] == n_channels
908
+ rg_np = rgeom.numpy(force=True) if torch.is_tensor(rgeom) else rgeom
909
+ spatial_mask = squareform(pdist(rg_np)) < zero_radius
910
+
911
+ # spatial part could be "by rank" or same for all ranks
912
+ # either way, there are nans afoot
913
+ if "by_rank" in cov_kind:
914
+ channel_std = torch.ones_like(x_spatial[0])
915
+ channel_vt = torch.zeros((rank, n_channels, n_channels)).to(channel_std)
916
+ for q in range(rank):
917
+ xq = x_spatial[:, q]
918
+ validq = xq.isfinite().any(0)
919
+ covq = spiketorch.nancov(xq[:, validq])
920
+ if shrinkage:
921
+ covq = F.softshrink(covq, shrinkage)
922
+ fullcovq = torch.eye(xq.shape[1], dtype=covq.dtype, device=covq.device)
923
+ fullcovq[validq[:, None] & validq[None, :]] = covq.view(-1)
924
+ if spatial_mask is not None:
925
+ fullcovq *= torch.asarray(spatial_mask).to(fullcovq)
926
+ if glasso_alpha:
927
+ res = graphical_lasso(
928
+ fullcovq.numpy(force=True),
929
+ alpha=glasso_alpha,
930
+ mode="cd",
931
+ max_iter=100,
932
+ verbose=logger.isEnabledFor(DARTSORTDEBUG),
933
+ )
934
+ fullcovq = torch.from_numpy(res[0]).to(fullcovq)
935
+ qeig, qv = torch.linalg.eigh(fullcovq)
936
+ channel_std[q] = qeig.sqrt()
937
+ channel_vt[q] = qv.T
938
+ else:
939
+ x_spatial = x_spatial.reshape(n * rank, n_channels)
940
+ valid = x_spatial.isfinite().any(0)
941
+
942
+ if "noise" in cov_kind:
943
+ xx = x_spatial.numpy(force=True)
944
+ invalid = np.isnan(xx)
945
+ xx[invalid] = (
946
+ np.random.default_rng(0).normal(size=invalid.sum()).astype(xx.dtype)
947
+ )
948
+ x_spatial = torch.from_numpy(xx).to(x_spatial)
949
+ valid = slice(None)
950
+ init_kw["cov_kind"] = cov_kind.removesuffix("noise")
951
+
952
+ cov = spiketorch.nancov(x_spatial[:, valid].double(), force_posdef=True)
953
+ cov.diagonal().add_(eps)
954
+ if shrinkage:
955
+ cov = F.softshrink(cov, shrinkage)
956
+
957
+ if glasso_alpha and isinstance(glasso_alpha, int):
958
+ logger.dartsortdebug(
959
+ f"Run glasso cv on {x_spatial.shape=} {cov.abs().max()=}"
960
+ )
961
+ # todo: clean up, propagate, ...
962
+ glasso = GraphicalLassoCV(
963
+ alphas=glasso_alpha,
964
+ n_jobs=1,
965
+ verbose=logger.isEnabledFor(DARTSORTDEBUG),
966
+ assume_centered=True,
967
+ eps=eps,
968
+ )
969
+ xx = x_spatial[:, valid].double().numpy(force=True)
970
+ invalid = np.isnan(xx)
971
+ xx[invalid] = np.random.default_rng(0).normal(size=invalid.sum())
972
+ with warnings.catch_warnings(action="ignore"):
973
+ glasso.fit(xx)
974
+ logger.dartsortdebug(f"Best alpha was {glasso.alpha_=}")
975
+ glasso_alpha = glasso.alpha_
976
+
977
+ if glasso_alpha and isinstance(glasso_alpha, float):
978
+ logger.dartsortdebug(f"Run glasso on {cov.shape=}")
979
+ res = graphical_lasso(
980
+ cov.numpy(force=True),
981
+ alpha=glasso_alpha,
982
+ mode="cd",
983
+ max_iter=100,
984
+ verbose=logger.isEnabledFor(DARTSORTDEBUG),
985
+ )
986
+ cov = torch.from_numpy(res[0]).to(cov)
987
+
988
+ cov_spatial = cov = cov.to(x_spatial)
989
+ assert cov_spatial.isfinite().all()
990
+ if valid != slice(None):
991
+ cov_spatial = torch.eye(
992
+ x_spatial.shape[1], dtype=x_spatial.dtype, device=x_spatial.device
993
+ )
994
+ cov_spatial[valid[:, None] & valid[None, :]] = cov.view(-1)
995
+ assert cov_spatial.isfinite().all()
996
+ if spatial_mask is not None:
997
+ cov_spatial *= torch.asarray(spatial_mask).to(cov_spatial)
998
+ assert cov_spatial.isfinite().all()
999
+ channel_eig, channel_v = torch.linalg.eigh(cov_spatial.double())
1000
+ if spatial_mask is None:
1001
+ assert channel_eig.isfinite().all()
1002
+ else:
1003
+ # the spatial mask is, i guess, not so plausible as a cov...
1004
+ channel_eig[channel_eig < eps] = eps
1005
+ channel_std = channel_eig.sqrt().float()
1006
+ channel_vt = channel_v.T.contiguous().float()
1007
+ assert channel_std.isfinite().all()
1008
+ assert channel_vt.isfinite().all()
1009
+
1010
+ return cls(
1011
+ mean=mean,
1012
+ global_std=global_std,
1013
+ rank_std=rank_std,
1014
+ rank_vt=rank_vt,
1015
+ channel_std=channel_std,
1016
+ channel_vt=channel_vt,
1017
+ zero_radius=zero_radius,
1018
+ **init_kw,
1019
+ )
1020
+
1021
+ @classmethod
1022
+ def estimate_from_hdf5(
1023
+ cls,
1024
+ hdf5_path,
1025
+ mean_kind="zero",
1026
+ cov_kind="factorizednoise",
1027
+ motion_est=None,
1028
+ interp_params: InterpolationParams = default_interpolation_params,
1029
+ device=None,
1030
+ shrinkage=0.0,
1031
+ glasso_alpha: int | float | None = None,
1032
+ zero_radius: float | None = None,
1033
+ rgeom=None,
1034
+ ):
1035
+ from dartsort.util.drift_util import registered_geometry
1036
+
1037
+ logger.dartsortdebug(
1038
+ f"Estimate embedded noise with {mean_kind=} {cov_kind=} {motion_est is None=} {glasso_alpha=}"
1039
+ )
1040
+
1041
+ with h5py.File(hdf5_path, "r", locking=False) as h5:
1042
+ geom = h5["geom"][:]
1043
+ if rgeom is None:
1044
+ rgeom = geom
1045
+ if motion_est is not None:
1046
+ rgeom = registered_geometry(geom, motion_est=motion_est)
1047
+ snippets = interpolate_residual_snippets(
1048
+ motion_est,
1049
+ hdf5_path,
1050
+ geom,
1051
+ rgeom,
1052
+ interp_params=interp_params,
1053
+ device=device,
1054
+ )
1055
+ return cls.estimate(
1056
+ snippets,
1057
+ shrinkage=shrinkage,
1058
+ mean_kind=mean_kind,
1059
+ cov_kind=cov_kind,
1060
+ glasso_alpha=glasso_alpha,
1061
+ zero_radius=zero_radius,
1062
+ rgeom=rgeom,
1063
+ )
1064
+
1065
+ def detection_prior_log_prob(self, templates_pca_projected, threshold=10.0):
1066
+ """
1067
+ Computes:
1068
+ z(T) = log[p(noise det | T)]
1069
+ = log[P(|N|^2 - |N - T|^2 > threshold^2)]
1070
+ = log[log P(2N.T > threshold^2 + |T|^2)]
1071
+
1072
+ Then, later, in mixture modeling one can compute
1073
+ p(l = noise | x, T) = log pi_noise + log N(x | noise) - z(T)
1074
+
1075
+ It's a little bit counterintuitive: this boosts the noise probability more
1076
+ for units with higher signal templates, since their FP probability is lower.
1077
+
1078
+ Note that since N ~ N(0, C), N.T ~ N(0, tr TCT'). Or, whatever version of
1079
+ that makes the dimensions work out. Thus we need to compute
1080
+ log normal_sf(0.5 * (thresh^2 + |T|^2) ; mean=0, scale=sqrt(tr TCT))
1081
+ """
1082
+ C = self.full_dense_cov()
1083
+ templates_pca_projected = torch.asarray(
1084
+ templates_pca_projected, device=C.device
1085
+ )
1086
+ T = templates_pca_projected.reshape(len(templates_pca_projected), -1)
1087
+ assert C.shape == (T.shape[1], T.shape[1])
1088
+ tr = torch.einsum("nc,cd,nd->n", T, C, T)
1089
+ scale = tr.sqrt_()
1090
+ crit = T.square().sum(dim=1).add_(threshold**2).mul_(0.5)
1091
+ return norm.logsf(crit.numpy(force=True), scale=scale.numpy(force=True))
1092
+
1093
+ def channelwise_detection_prior_log_prob(
1094
+ self, threshold=4.0, n_samples=8192, seed=0
1095
+ ):
1096
+ """Compute the marginal probability that the norm is larger than threshold on each channel.
1097
+
1098
+ TPCA is isometry, so no need to know the basis when working with the norm. Further, we can
1099
+ sample independent normals using the eigs.
1100
+ """
1101
+ probs = np.zeros(self.n_channels)
1102
+ rg = np.random.default_rng(seed)
1103
+ samples = rg.normal(size=(n_samples, self.rank))
1104
+ tmp = samples.copy()
1105
+ for channel in range(self.n_channels):
1106
+ marginal_cov = self.marginal_covariance(channels=[channel]).to_dense()
1107
+ marginal_cov = marginal_cov.numpy(force=True).astype(np.float64)
1108
+ eigs = np.linalg.eigvalsh(marginal_cov)
1109
+ stds = np.sqrt(eigs)
1110
+ norms = np.linalg.norm(np.multiply(samples, stds, out=tmp), axis=1)
1111
+ probs[channel] = np.mean(norms > threshold)
1112
+ return np.log(probs)
1113
+
1114
+
1115
+ def interpolate_residual_snippets(
1116
+ motion_est,
1117
+ hdf5_path,
1118
+ geom,
1119
+ registered_geom,
1120
+ residual_times_s_dataset_name="residual_times_seconds",
1121
+ residual_dataset_name="residual",
1122
+ interp_params: InterpolationParams = default_interpolation_params,
1123
+ do_tpca=True,
1124
+ workers=None,
1125
+ device=None,
1126
+ batch_size=16,
1127
+ ):
1128
+ """PCA-embed and interpolate residual snippets to the registered probe"""
1129
+ from dartsort.util import data_util, drift_util, interpolation_util
1130
+
1131
+ assert geom.shape[1] == 2, "Haven't implemented 3d probes here."
1132
+
1133
+ if device is None:
1134
+ device = "cuda" if torch.cuda.is_available() else None
1135
+ device = torch.device(device)
1136
+
1137
+ with h5py.File(hdf5_path, "r", locking=False) as h5:
1138
+ channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
1139
+ snippets = cast(h5py.Dataset, h5[residual_dataset_name])[:]
1140
+ times_s = cast(h5py.Dataset, h5[residual_times_s_dataset_name])[:]
1141
+ channel_index = torch.from_numpy(channel_index)
1142
+ snippets = torch.from_numpy(snippets)
1143
+ times_s = torch.from_numpy(times_s)
1144
+
1145
+ # tpca project
1146
+ if do_tpca:
1147
+ tpca = data_util.get_tpca(hdf5_path)
1148
+ if device is not None:
1149
+ tpca = tpca.to(tpca.b.components.dtype)
1150
+ tpca = tpca.to(device=device)
1151
+ if tpca.temporal_slice is not None:
1152
+ snippets = snippets[:, tpca.temporal_slice]
1153
+ n, t, c = snippets.shape
1154
+ snippets_proj = snippets.new_zeros(n, tpca.rank, c)
1155
+ for i0 in range(0, n, batch_size):
1156
+ i1 = min(n, i0 + batch_size)
1157
+ nb = i1 - i0
1158
+ b = snippets[i0:i1].permute(0, 2, 1).reshape(nb * c, t)
1159
+ b = tpca._transform_in_probe(b.to(tpca.components))
1160
+ snippets_proj[i0:i1] = b.reshape(nb, c, -1).permute(0, 2, 1)
1161
+ snippets = snippets_proj
1162
+ n, dim, c = snippets.shape
1163
+ assert snippets.isfinite().all()
1164
+
1165
+ # -- interpolate
1166
+ # fill in the registered probe with residual snippets using interpolation,
1167
+ # with possible missing values
1168
+ source_geom = torch.asarray(geom).to(snippets).to(device)
1169
+ source_pos = source_geom[None].broadcast_to(n, *geom.shape).contiguous()
1170
+
1171
+ # - precompute
1172
+ interp_params = interp_params.normalize()
1173
+ precomputed_data = interpolation_util.full_probe_precompute(
1174
+ source_geom=source_geom.to(device),
1175
+ channel_index=channel_index.to(device),
1176
+ params=interp_params,
1177
+ )
1178
+ if precomputed_data is not None:
1179
+ precomputed_data = precomputed_data.to(device)
1180
+ if motion_est is None:
1181
+ # no drift case, no missing values, but still interpolate to avoid
1182
+ # statistical differences between drifty/no drift versions of the sorter
1183
+ target_geom = torch.asarray(registered_geom).to(device=device)
1184
+ assert torch.equal(source_geom, target_geom)
1185
+ target_pos = target_geom[None].broadcast_to(n, *geom.shape).contiguous()
1186
+
1187
+ for bs in range(0, n, batch_size):
1188
+ sl = slice(bs, min(n, bs + batch_size))
1189
+ snippets[sl] = interpolation_util.kernel_interpolate(
1190
+ snippets[sl].to(device),
1191
+ source_pos[sl].to(device),
1192
+ target_pos[sl].to(device),
1193
+ params=interp_params,
1194
+ precomputed_data=precomputed_data,
1195
+ allow_destroy=True,
1196
+ ).to(snippets)
1197
+ return snippets
1198
+
1199
+ # goal
1200
+ # - determine what registered channels the shifting source positions
1201
+ # would land on
1202
+ # - interpolate residual snippets from the shifted source position
1203
+ # to the target positions
1204
+ # except that's slow, because kriging interpolator would require
1205
+ # inverting a matrix for each shifted source geom. so instead,
1206
+ # let's shift the targets inversely and use the same source geom,
1207
+ # even if that has a bit of a different meaning.
1208
+
1209
+ # determine each channel's drift over time
1210
+ source_depths = source_pos[:, :, 1].reshape(-1)
1211
+ source_t = times_s[:, None].broadcast_to(source_pos[:, :, 1].shape).reshape(-1)
1212
+ source_shifts = motion_est.disp_at_s(source_t.cpu(), source_depths.cpu())
1213
+ source_shifts = source_shifts.reshape(source_pos[:, :, 1].shape).astype("float32")
1214
+ assert np.isfinite(source_shifts).all()
1215
+ source_shifts_xy = np.stack([np.zeros_like(source_shifts), source_shifts], axis=-1)
1216
+ source_pos_shifted = source_pos.numpy(force=True) - source_shifts_xy
1217
+
1218
+ # query the target geom for the closest source pos, within reason
1219
+ rg_np = registered_geom
1220
+ if torch.is_tensor(rg_np):
1221
+ rg_np = rg_np.numpy(force=True)
1222
+ kdtree = drift_util.KDTree(rg_np)
1223
+ match_distance = drift_util.pdist(geom).min()
1224
+ _, targ_inds = kdtree.query(
1225
+ source_pos_shifted.reshape(-1, geom.shape[1]),
1226
+ distance_upper_bound=match_distance,
1227
+ workers=workers or -1,
1228
+ )
1229
+ targ_inds = torch.from_numpy(targ_inds).reshape(source_pos.shape[:2])
1230
+ assert (targ_inds < kdtree.n).all()
1231
+ target_pos = torch.asarray(registered_geom[targ_inds], device=device)
1232
+ target_pos_shifted = target_pos + torch.asarray(source_shifts_xy).to(target_pos)
1233
+ target_pos_shifted = torch.asarray(target_pos_shifted).to(snippets)
1234
+
1235
+ # allocate output storage with an extra channel of NaN needed later
1236
+ for bs in range(0, n, batch_size):
1237
+ sl = slice(bs, min(n, bs + batch_size))
1238
+ snippets[sl] = interpolation_util.kernel_interpolate(
1239
+ snippets[sl].to(device),
1240
+ source_pos[sl].to(device),
1241
+ target_pos_shifted[sl].to(device),
1242
+ params=interp_params,
1243
+ precomputed_data=precomputed_data,
1244
+ allow_destroy=True,
1245
+ ).to(snippets)
1246
+ assert snippets.isfinite().all()
1247
+
1248
+ # now, let's embed these into the full registered probe
1249
+ snippets_full = snippets.new_full((n, dim, len(registered_geom)), torch.nan)
1250
+ targ_inds = targ_inds[:, None].broadcast_to(snippets.shape)
1251
+ snippets_full.scatter_(2, targ_inds.to(snippets.device), snippets)
1252
+
1253
+ return snippets_full
1254
+
1255
+
1256
+ def fp_control_threshold(
1257
+ fp_dataframe,
1258
+ fp_num_frames,
1259
+ tp_unit_ids,
1260
+ tp_counts,
1261
+ clustering_num_frames,
1262
+ template_normsqs,
1263
+ clustering_subsampling_rate=1.0,
1264
+ max_fp_per_input_spike=1.0,
1265
+ resolution=1.0,
1266
+ min_threshold_factor=0.1,
1267
+ min_threshold=5.0,
1268
+ ):
1269
+ """Global threshold for template matching to control the false positive rate
1270
+
1271
+ For each unit, as a function of the threshold, we can estimate the number
1272
+ of false positives per frame using `fp_dataframe` and `fp_num_frames`. We
1273
+ can also estimate the true positive rate using `labels` and
1274
+ `clustering_num_frames`.
1275
+
1276
+ Then, for each unit, we can find the largest threshold such that the false
1277
+ positive rate divided by the true positive rate is <max_fp_per_input_spike.
1278
+ This controls the FPR after deconv per unit to be
1279
+ <max_fp_per_input_spike/2*max_fp_per_input_spike.
1280
+ We can then take the max over units to get a global threshold.
1281
+
1282
+ Arguments
1283
+ ---------
1284
+ fp_dataframe: pd.DataFrame
1285
+ fp_num_frames : int
1286
+ Above two as returned by .unit_false_positives()
1287
+ tp_unit_ids, tp_counts : np.array
1288
+ Unique of clustering labels
1289
+ clustering_num_frames : int
1290
+ Number of valid recording frames used during clustering
1291
+ clustering_subsampling_rate : float
1292
+ If the full set of detected spikes from the whole `clustering_num_frames`
1293
+ frame collection was subsampled, then we should calculate rates per frame
1294
+ by dividing by clustering_subsampling_rate * clustering_num_frames
1295
+ max_fp_per_input_spike : float
1296
+ The control parameter described above.
1297
+ resolution : float
1298
+ The rates as a function of threshold will be estimated as a function of
1299
+ (squared) deconv threshold at this resolution.
1300
+
1301
+ Returns
1302
+ -------
1303
+ threshold : float
1304
+ Note: this is in squared objective units.
1305
+ """
1306
+ tp_counts = tp_counts[tp_unit_ids >= 0]
1307
+ tp_unit_ids = tp_unit_ids[tp_unit_ids >= 0]
1308
+ fp_units = np.unique(fp_dataframe.units)
1309
+ assert np.isin(fp_units, tp_unit_ids).all()
1310
+ has_fp = np.isin(tp_unit_ids, fp_units)
1311
+ logger.dartsortdebug(f"{has_fp.mean() * 100:.1f}% of units had FPs")
1312
+
1313
+ # initialize the threshold with a min factor
1314
+ threshold = max(min_threshold, min_threshold_factor * template_normsqs.min())
1315
+ logger.dartsortdebug(f"fp control: min possible {threshold=}")
1316
+ if not len(fp_dataframe):
1317
+ return threshold
1318
+
1319
+ # resolution-spaced grid which will be used for searches below
1320
+ start = resolution * np.ceil(threshold / resolution)
1321
+ end = resolution * np.ceil(fp_dataframe.scores.max() / resolution)
1322
+ domain = np.arange(start, end + resolution, resolution)
1323
+ domaini = 0
1324
+
1325
+ # account for different time ranges
1326
+ fp_per_tp = fp_num_frames / (clustering_subsampling_rate * clustering_num_frames)
1327
+
1328
+ for uid, tp in zip(tp_unit_ids[has_fp], tp_counts[has_fp]):
1329
+ in_uid = fp_dataframe.units == uid
1330
+ if not in_uid.any():
1331
+ continue
1332
+
1333
+ # does the unit even have any fps at this threshold?
1334
+ fp_scores = fp_dataframe[in_uid].scores.values
1335
+ if threshold > fp_scores.max():
1336
+ continue
1337
+
1338
+ # largest number of false positives we can allow in this unit
1339
+ max_fp = max_fp_per_input_spike * tp * fp_per_tp
1340
+
1341
+ # check if we are already under the limit
1342
+ if fp_scores.size <= max_fp:
1343
+ continue
1344
+
1345
+ # pick the smallest threshold such that the estimated fp is < max_fp
1346
+ fp_scores.sort()
1347
+ fp_ix = 0
1348
+ for i, x in enumerate(domain[domaini:]):
1349
+ rel_ix = np.searchsorted(fp_scores[fp_ix:], x, side="left")
1350
+ if rel_ix == 0:
1351
+ continue
1352
+ fp_ix += rel_ix
1353
+ n_fp = fp_scores.size - fp_ix
1354
+ if n_fp <= max_fp:
1355
+ assert x >= threshold
1356
+ domaini += i
1357
+ threshold = x
1358
+ logger.dartsortdebug(f"fp control: new {threshold=}")
1359
+ break
1360
+ else:
1361
+ # a break should always be hit thanks to the continues above
1362
+ assert False
1363
+
1364
+ return threshold
1365
+
1366
+
1367
+ def fp_control_threshold_from_h5(
1368
+ hdf5_path,
1369
+ low_rank_templates,
1370
+ unit_ids=None,
1371
+ spike_counts=None,
1372
+ rg: int | np.random.Generator = 0,
1373
+ refractory_radius_frames=10,
1374
+ num_frames=0,
1375
+ max_fp_per_input_spike=1.0,
1376
+ ):
1377
+ from ..util.data_util import get_residual_snips
1378
+
1379
+ resids = get_residual_snips(hdf5_path)
1380
+ noise = StationaryFactorizedNoise.estimate(resids)
1381
+
1382
+ # be reproducible
1383
+ rg = np.random.default_rng(rg)
1384
+ generator = spiketorch.spawn_torch_rg(rg)
1385
+
1386
+ # restrict my low rank templates to usual geom...
1387
+ # TODO: this is not a very logical way to handle drift here...
1388
+
1389
+ # simulate fps
1390
+ fp_res = noise.unit_false_positives(
1391
+ low_rank_templates,
1392
+ radius=refractory_radius_frames,
1393
+ generator=generator,
1394
+ )
1395
+ return fp_control_threshold(
1396
+ fp_res["fp_dataframe"],
1397
+ fp_res["total_samples"],
1398
+ tp_unit_ids=unit_ids,
1399
+ tp_counts=spike_counts,
1400
+ clustering_num_frames=num_frames,
1401
+ template_normsqs=fp_res["normsq"],
1402
+ max_fp_per_input_spike=max_fp_per_input_spike,
1403
+ )
1404
+
1405
+
1406
+ def whitener_from_hdf5(
1407
+ hdf5_path,
1408
+ motion_est=None,
1409
+ interp_params: InterpolationParams = default_interpolation_params,
1410
+ device=None,
1411
+ rgeom=None,
1412
+ eps=1e-4,
1413
+ corr_mode=False,
1414
+ ):
1415
+ from dartsort.util.drift_util import registered_geometry
1416
+
1417
+ with h5py.File(hdf5_path, "r", locking=False) as h5:
1418
+ geom = cast(h5py.Dataset, h5["geom"])[:]
1419
+ if rgeom is None:
1420
+ rgeom = geom
1421
+ if motion_est is not None:
1422
+ rgeom = registered_geometry(geom, motion_est=motion_est)
1423
+
1424
+ snippets = interpolate_residual_snippets(
1425
+ motion_est,
1426
+ hdf5_path,
1427
+ geom.astype(np.float32),
1428
+ rgeom.astype(np.float32),
1429
+ interp_params=interp_params,
1430
+ device=device,
1431
+ do_tpca=False,
1432
+ )
1433
+
1434
+ x = np.ascontiguousarray(snippets.cpu(), dtype=np.float64)
1435
+ x = x.reshape(-1)
1436
+ invalid = np.flatnonzero(np.isnan(x))
1437
+ if invalid.size:
1438
+ rg = np.random.default_rng(0)
1439
+ x[invalid] = rg.normal(size=invalid.size).astype(x.dtype)
1440
+ x = x.reshape(-1, len(rgeom))
1441
+ if corr_mode:
1442
+ std = np.std(x, axis=0)
1443
+ x /= std
1444
+ _, S, Vh = np.linalg.svd(x, full_matrices=False)
1445
+ assert Vh.shape == (len(rgeom), len(rgeom))
1446
+ assert S.shape == (len(rgeom),)
1447
+ std = S / np.sqrt(len(x) - 1.0) + eps
1448
+ logger.dartsortdebug(
1449
+ "Whitening standard deviations: min,mean,max="
1450
+ f"{std.min().item():0.5f},{std.mean().item():0.5f},{std.max().item():0.5f}"
1451
+ )
1452
+
1453
+ wstd = 1.0 / std
1454
+ # life is easier if we try not to change the units
1455
+ # too much in the whitened data domain
1456
+ wstd /= np.median(wstd)
1457
+
1458
+ # using the zca whitener here. symmetry takes a load off the mind.
1459
+ whitener = (Vh * wstd) @ Vh.T
1460
+
1461
+ return whitener.astype(np.float32)