DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,413 @@
1
+ import numpy as np
2
+ import torch
3
+ from sklearn.decomposition import PCA, TruncatedSVD
4
+
5
+ from ..util.waveform_util import (
6
+ channel_subset_by_radius,
7
+ get_channels_in_probe,
8
+ set_channels_in_probe,
9
+ )
10
+ from ..util.spiketorch import svd_lowrank_helper
11
+ from .transform_base import (
12
+ BaseWaveformAutoencoder,
13
+ BaseWaveformDenoiser,
14
+ BaseWaveformFeaturizer,
15
+ BaseWaveformModule,
16
+ )
17
+
18
+
19
+ class BaseTemporalPCA(BaseWaveformModule):
20
+ default_name = "basis"
21
+
22
+ def __init__(
23
+ self,
24
+ channel_index,
25
+ geom=None,
26
+ rank=8,
27
+ whiten=False,
28
+ centered=False,
29
+ fit_radius=None,
30
+ random_state=0,
31
+ name=None,
32
+ name_prefix="",
33
+ temporal_slice: slice | None = None,
34
+ n_oversamples=10,
35
+ niter=21,
36
+ max_waveforms=20_000,
37
+ batch_size=4096,
38
+ fit_dtype=torch.double,
39
+ ):
40
+ if fit_radius is not None:
41
+ if geom is None or channel_index is None:
42
+ raise ValueError("TemporalPCA with fit_radius!=None requires geom.")
43
+
44
+ super().__init__(
45
+ channel_index=channel_index, geom=geom, name=name, name_prefix=name_prefix
46
+ )
47
+
48
+ # behavior
49
+ self.rank = rank
50
+ self.centered = centered
51
+ self.whiten = whiten
52
+ self.temporal_slice = temporal_slice
53
+ self.batch_size = batch_size
54
+
55
+ # fit control
56
+ self.fit_radius = fit_radius
57
+ self.random_state = random_state
58
+ self.n_oversamples = n_oversamples
59
+ self.niter = niter
60
+ self.fit_dtype = fit_dtype
61
+ self.max_waveforms = max_waveforms
62
+
63
+ # gizmo
64
+ self._needs_fit = True
65
+ self.shape = (rank, channel_index.shape[1])
66
+ self.nt = None
67
+
68
+ def initialize_spike_length_dependent_params(self):
69
+ nt = self.spike_length_samples
70
+ assert nt is not None
71
+ if self.temporal_slice is None:
72
+ self._temporal_ix = None
73
+ else:
74
+ self.register_buffer("_temporal_ix", torch.arange(nt)[self.temporal_slice])
75
+ nt = self.b._temporal_ix.numel()
76
+ self.nt = nt
77
+ self.register_buffer("mean", torch.zeros(nt))
78
+ self.register_buffer("components", torch.zeros(self.rank, nt))
79
+ self.register_buffer("whitener", torch.zeros(self.rank))
80
+ self.to(self.b.channel_index.device)
81
+
82
+ def fit(
83
+ self,
84
+ recording,
85
+ waveforms,
86
+ *,
87
+ channels,
88
+ weights=None,
89
+ time_shifts=None,
90
+ **unused,
91
+ ):
92
+ super().fit(recording, waveforms, channels=channels, weights=weights)
93
+ if weights is not None and waveforms.shape[0] > self.max_waveforms:
94
+ self.random_state = np.random.default_rng(self.random_state)
95
+ weights = weights.numpy(force=True) if torch.is_tensor(weights) else weights
96
+ weights = weights.astype(np.float64)
97
+ weights = weights / weights.sum()
98
+ choices = self.random_state.choice(
99
+ len(weights), p=weights, size=self.max_waveforms
100
+ )
101
+ choices.sort()
102
+ choices = torch.from_numpy(choices)
103
+ waveforms = waveforms[choices]
104
+ channels = channels[choices]
105
+ waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
106
+ self.dtype = waveforms.dtype
107
+ if self.fit_radius is not None:
108
+ waveforms, train_channel_index = channel_subset_by_radius(
109
+ waveforms,
110
+ channels,
111
+ self.channel_index.to(device=waveforms.device),
112
+ self.geom.to(device=waveforms.device),
113
+ self.fit_radius,
114
+ )
115
+ else:
116
+ train_channel_index = self.b.channel_index.to(waveforms.device)
117
+ _, waveforms_fit = get_channels_in_probe(
118
+ waveforms, channels, train_channel_index
119
+ )
120
+ waveforms_fit = waveforms_fit.to(self.b.channel_index.device)
121
+
122
+ if self.centered:
123
+ mean = waveforms_fit.mean(0)
124
+ else:
125
+ mean = torch.zeros_like(waveforms_fit[0])
126
+
127
+ if self.centered:
128
+ # torch does not seem always to want to broadcast M as advertised?
129
+ M = mean[None].broadcast_to(waveforms_fit.shape)
130
+ else:
131
+ M = None
132
+
133
+ _, components, _, whitener = svd_lowrank_helper(
134
+ x=waveforms_fit,
135
+ rank=self.rank,
136
+ n_oversamples=self.n_oversamples,
137
+ fit_dtype=self.fit_dtype,
138
+ niter=self.niter,
139
+ M=M,
140
+ with_loadings=False,
141
+ )
142
+
143
+ self.b.mean.copy_(mean)
144
+ self.b.components.copy_(components)
145
+ self.b.whitener.copy_(whitener)
146
+ self._needs_fit = False
147
+
148
+ def needs_fit(self):
149
+ return self._needs_fit
150
+
151
+ def _temporal_slice(
152
+ self, waveforms: torch.Tensor, time_shifts=None
153
+ ) -> torch.Tensor:
154
+ if self.temporal_slice is None:
155
+ return waveforms
156
+
157
+ if time_shifts is None:
158
+ return waveforms[:, self.temporal_slice]
159
+
160
+ assert self._temporal_ix is not None
161
+ n, t, c = waveforms.shape
162
+ t_ = self._temporal_ix.numel()
163
+ assert t_ <= t
164
+ assert time_shifts.shape == (n,)
165
+ time_ix = self._temporal_ix[None, :, None] + time_shifts[:, None, None]
166
+ waveforms = waveforms.take_along_dim(dim=1, indices=time_ix)
167
+ assert waveforms.shape == (n, t_, c)
168
+ return waveforms
169
+
170
+ def _transform_in_probe(self, waveforms_in_probe):
171
+ n = waveforms_in_probe.shape[0]
172
+ if self.whiten:
173
+ W = self.b.components / self.b.whitener
174
+ else:
175
+ W = self.b.components
176
+ out = waveforms_in_probe.new_empty((n, self.rank))
177
+ dev = self.b.components.device
178
+ same_dev = out.device == dev
179
+ for i0 in range(0, n, self.batch_size):
180
+ i1 = min(n, i0 + self.batch_size)
181
+ x = waveforms_in_probe[i0:i1].to(device=dev)
182
+ if self.centered:
183
+ x = x - self.mean
184
+ if same_dev:
185
+ torch.mm(x, W.T, out=out[i0:i1])
186
+ else:
187
+ out[i0:i1] = (x @ W.T).to(out)
188
+ return out
189
+
190
+ def _inverse_transform_in_probe(self, features):
191
+ if self.whiten:
192
+ W = self.b.components * self.b.whitener
193
+ else:
194
+ W = self.b.components
195
+ assert self.nt is not None
196
+ n = features.shape[0]
197
+ out = features.new_zeros((n, self.nt))
198
+ dev = self.b.components.device
199
+ for i0 in range(0, n, self.batch_size):
200
+ i1 = min(n, i0 + self.batch_size)
201
+ x = features[i0:i1].to(device=dev)
202
+ if out.device == dev:
203
+ if self.centered:
204
+ torch.addmm(self.b.mean, x, W, out=out[i0:i1])
205
+ else:
206
+ torch.mm(x, W, out=out[i0:i1])
207
+ else:
208
+ if self.centered:
209
+ out[i0:i1] = torch.addmm(self.b.mean, x, W).to(out)
210
+ else:
211
+ out[i0:i1] = torch.mm(x, W).to(out)
212
+ return out
213
+
214
+ def _project_in_probe(self, waveforms_in_probe):
215
+ n = waveforms_in_probe.shape[0]
216
+ out = torch.empty_like(waveforms_in_probe)
217
+ proj = self.b.components.T @ self.b.components
218
+ dev = proj.device
219
+ same_dev = out.device == dev
220
+ for i0 in range(0, n, self.batch_size):
221
+ i1 = min(n, i0 + self.batch_size)
222
+ x = waveforms_in_probe[i0:i1].to(device=dev)
223
+ if same_dev:
224
+ if self.centered:
225
+ torch.addmm(self.b.mean, x - self.mean, proj, out=out[i0:i1])
226
+ else:
227
+ torch.mm(waveforms_in_probe, proj, out=out[i0:i1])
228
+ else:
229
+ if self.centered:
230
+ out[i0:i1] = torch.addmm(self.b.mean, x - self.mean, proj).to(out)
231
+ else:
232
+ out[i0:i1] = torch.mm(x, proj).to(out)
233
+ return out
234
+
235
+ def force_reconstruct(self, features):
236
+ ndim = features.ndim
237
+ if ndim == 2:
238
+ features = features.unsqueeze(0)
239
+ n, r, c = features.shape
240
+ waveforms = features.permute(0, 2, 1).reshape(n * c, r)
241
+ waveforms = self._inverse_transform_in_probe(waveforms)
242
+ waveforms = waveforms.reshape(n, c, -1).permute(0, 2, 1)
243
+ if ndim == 2:
244
+ waveforms = waveforms[0]
245
+ return waveforms
246
+
247
+ def force_project(self, features):
248
+ ndim = features.ndim
249
+ if ndim == 2:
250
+ features = features.unsqueeze(0)
251
+ n, t, c = features.shape
252
+ waveforms = features.mT.reshape(n * c, t)
253
+ waveforms = self._project_in_probe(waveforms)
254
+ waveforms = waveforms.reshape(n, c, t).mT
255
+ if ndim == 2:
256
+ waveforms = waveforms[0]
257
+ return waveforms
258
+
259
+ def to_sklearn(self) -> PCA:
260
+ pca = PCA(
261
+ n_components=self.rank,
262
+ random_state=self.random_state, # type: ignore
263
+ whiten=self.whiten,
264
+ )
265
+ pca.mean_ = self.b.mean.numpy(force=True)
266
+ pca.components_ = self.b.components.numpy(force=True)
267
+ pca.explained_variance_ = np.square(self.b.whitener.numpy(force=True))
268
+ pca.temporal_slice = self.temporal_slice # this is not standard # type: ignore
269
+ return pca
270
+
271
+ @classmethod
272
+ def from_sklearn(cls, channel_index, pca: PCA | TruncatedSVD, temporal_slice=None):
273
+ if isinstance(pca, PCA):
274
+ whiten = pca.whiten # type: ignore
275
+ elif isinstance(pca, TruncatedSVD):
276
+ whiten = False
277
+ else:
278
+ assert False
279
+ self = cls(
280
+ channel_index,
281
+ rank=pca.n_components, # type: ignore
282
+ whiten=whiten,
283
+ temporal_slice=temporal_slice,
284
+ )
285
+ self.initialize_from_sklearn(pca)
286
+ return self
287
+
288
+ @classmethod
289
+ def convert(cls, other):
290
+ assert not other._needs_fit # doesn't handle fit_radius right.
291
+ self = cls(
292
+ channel_index=other.channel_index,
293
+ rank=other.rank,
294
+ centered=other.centered,
295
+ whiten=other.whiten,
296
+ temporal_slice=other.temporal_slice,
297
+ )
298
+ self._needs_fit = other._needs_fit
299
+ self.spike_length_samples = other.spike_length_samples
300
+ self.initialize_spike_length_dependent_params()
301
+ self.to(other.channel_index.device)
302
+ self.b.mean.copy_(other.mean)
303
+ self.b.components.copy_(other.components)
304
+ self.b.whitener.copy_(other.whitener)
305
+ return self
306
+
307
+ def initialize_from_sklearn(self, pca):
308
+ if self.temporal_slice is None:
309
+ self.spike_length_samples = pca.components_.shape[1]
310
+ else:
311
+ # not really -- this is a hack.
312
+ self.spike_length_samples = (
313
+ self.temporal_slice.stop - self.temporal_slice.start
314
+ )
315
+ self.initialize_spike_length_dependent_params()
316
+ if hasattr(pca, "mean_"):
317
+ self.b.mean.copy_(torch.from_numpy(pca.mean_))
318
+ self.center = not (self.b.mean == 0.0).all()
319
+ else:
320
+ self.b.mean.zero_()
321
+ self.center = False
322
+ self.b.components.copy_(torch.from_numpy(pca.components_))
323
+ self.b.whitener.copy_(torch.from_numpy(pca.explained_variance_)).sqrt_()
324
+ self._needs_fit = False
325
+
326
+
327
+ class TemporalPCADenoiser(BaseWaveformDenoiser, BaseTemporalPCA):
328
+ default_name = "temporal_pca"
329
+
330
+ def forward(self, waveforms, *, channels, time_shifts=None, **unused):
331
+ waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
332
+ dev = waveforms.device
333
+ channels_in_probe, waveforms_in_probe = get_channels_in_probe(
334
+ waveforms, channels.to(device=dev), self.channel_index.to(device=dev)
335
+ )
336
+ waveforms_in_probe = self._project_in_probe(waveforms_in_probe)
337
+ return set_channels_in_probe(waveforms_in_probe, waveforms, channels_in_probe)
338
+
339
+
340
+ class TemporalPCAFeaturizer(BaseWaveformFeaturizer, BaseTemporalPCA):
341
+ default_name = "tpca_features"
342
+
343
+ def transform(
344
+ self,
345
+ waveforms,
346
+ *,
347
+ channels,
348
+ channel_index=None,
349
+ _return_for_combined=False,
350
+ time_shifts=None,
351
+ **unused,
352
+ ):
353
+ waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
354
+
355
+ if channel_index is None:
356
+ channel_index = self.b.channel_index
357
+ dev = waveforms.device
358
+ channels_in_probe, waveforms_in_probe = get_channels_in_probe(
359
+ waveforms, channels.to(device=dev), channel_index.to(device=dev)
360
+ )
361
+ features_in_probe = self._transform_in_probe(waveforms_in_probe)
362
+ features = features_in_probe.new_full(
363
+ (waveforms.shape[0], self.rank, channel_index.shape[1]), torch.nan
364
+ )
365
+ features = set_channels_in_probe(
366
+ features_in_probe,
367
+ features,
368
+ channels_in_probe,
369
+ )
370
+ if _return_for_combined:
371
+ return (
372
+ waveforms,
373
+ features_in_probe,
374
+ channels_in_probe,
375
+ {self.name: features},
376
+ )
377
+
378
+ return {self.name: features}
379
+
380
+ def inverse_transform(self, features, channels, channel_index=None):
381
+ if channel_index is None:
382
+ channel_index = self.b.channel_index
383
+ channels_in_probe, features_in_probe = get_channels_in_probe(
384
+ features, channels, channel_index
385
+ )
386
+ reconstructions_in_probe = self._inverse_transform_in_probe(features_in_probe)
387
+ recshp = (features.shape[0], self.b.components.shape[1], channel_index.shape[1])
388
+ reconstructions = torch.full(
389
+ recshp,
390
+ torch.nan,
391
+ dtype=reconstructions_in_probe.dtype,
392
+ device=reconstructions_in_probe.device,
393
+ )
394
+ return set_channels_in_probe(
395
+ reconstructions_in_probe, reconstructions, channels_in_probe
396
+ )
397
+
398
+
399
+ class TemporalPCA(BaseWaveformAutoencoder, TemporalPCAFeaturizer):
400
+ default_name = "tpca_features"
401
+
402
+ def forward(self, waveforms, *, channels, time_shifts=None, **unused):
403
+ waveforms, features_in_probe, channels_in_probe, features = self.transform(
404
+ waveforms,
405
+ channels=channels,
406
+ time_shifts=time_shifts,
407
+ _return_for_combined=True,
408
+ )
409
+ reconstructions_in_probe = self._inverse_transform_in_probe(features_in_probe)
410
+ reconstructions = set_channels_in_probe(
411
+ reconstructions_in_probe, waveforms, channels_in_probe
412
+ )
413
+ return reconstructions, features
@@ -0,0 +1,255 @@
1
+ from typing import Any, Iterable
2
+
3
+ import torch
4
+
5
+ from ..util.data_util import SpikeDataset
6
+ from ..util.torch_util import BModule
7
+
8
+
9
+ class BaseWaveformModule(BModule):
10
+ is_denoiser = False
11
+ is_featurizer = False
12
+ default_name = ""
13
+
14
+ def __init__(self, channel_index=None, geom=None, name=None, name_prefix=None):
15
+ super().__init__()
16
+ if name is None:
17
+ name = self.default_name
18
+ if name_prefix:
19
+ name = f"{name_prefix}_{name}"
20
+ self.name = name
21
+ # these buffers below need to be copied, else they share references
22
+ # across all the transformers which seems to cause problems!
23
+ if channel_index is not None:
24
+ channel_index = torch.asarray(channel_index, copy=True)
25
+ self.register_buffer("channel_index", channel_index)
26
+ if geom is not None:
27
+ geom = torch.asarray(geom, dtype=torch.float, copy=True)
28
+ self.register_buffer("geom", geom)
29
+
30
+ self.spike_length_samples = None
31
+ try:
32
+ self._hook = self.register_load_state_dict_pre_hook(
33
+ self.__class__._pre_load_state
34
+ )
35
+ except AttributeError:
36
+ # ...? seems to happen in 2.4.1...
37
+ self._hook = self._register_load_state_dict_pre_hook(
38
+ self.__class__._pre_load_state
39
+ )
40
+
41
+ def __getstate__(self):
42
+ self._hook.remove()
43
+ state = super().__getstate__()
44
+ del state["_hook"]
45
+ return state
46
+
47
+ def __setstate__(self, state):
48
+ super().__setstate__(state)
49
+ try:
50
+ self._hook = self.register_load_state_dict_pre_hook(
51
+ self.__class__._pre_load_state
52
+ )
53
+ except AttributeError:
54
+ self._hook = self._register_load_state_dict_pre_hook(
55
+ self.__class__._pre_load_state
56
+ )
57
+
58
+ def fit(self, recording, waveforms, **fixed_properties) -> Any:
59
+ del recording, fixed_properties
60
+ self.spike_length_samples = waveforms.shape[1]
61
+ self.initialize_spike_length_dependent_params()
62
+
63
+ def get_extra_state(self):
64
+ return dict(
65
+ spike_length_samples=self.spike_length_samples, needs_fit=self.needs_fit()
66
+ )
67
+
68
+ def set_extra_state(self, state):
69
+ self.spike_length_samples = state["spike_length_samples"]
70
+ if hasattr(self, "_needs_fit"):
71
+ self._needs_fit = state["needs_fit"]
72
+ self._init_bgetter()
73
+
74
+ def _pre_load_state(
75
+ self,
76
+ state_dict,
77
+ prefix,
78
+ local_metadata,
79
+ strict,
80
+ missing_keys,
81
+ unexpected_keys,
82
+ error_msgs,
83
+ ):
84
+ # wish torch would strip the prefix for us?
85
+ extra_state_keys = [k for k in state_dict.keys() if k.endswith("_extra_state")]
86
+ assert len(extra_state_keys) == 1
87
+ extra_state = state_dict[extra_state_keys[0]]
88
+
89
+ # some modules want to know the spike length before loading the state dict
90
+ # and unfortunately set_extra_state usually runs after. doesn't hurt to run now.
91
+ self.spike_length_samples = extra_state["spike_length_samples"]
92
+
93
+ # and this is how subclasses use that info
94
+ # if state dict was dumped before fit, then sls was never known and we
95
+ # don't want to call that initializer, so don't if sls is None.
96
+ if self.spike_length_samples is not None:
97
+ self.initialize_spike_length_dependent_params()
98
+
99
+ def initialize_spike_length_dependent_params(self):
100
+ pass
101
+
102
+ def needs_fit(self) -> bool:
103
+ return False
104
+
105
+ def needs_precompute(self) -> bool:
106
+ return False
107
+
108
+ def precompute(self):
109
+ pass
110
+
111
+ def extra_repr(self):
112
+ return f"name={self.name},needs_fit={self.needs_fit()}"
113
+
114
+
115
+ class BaseWaveformDenoiser(BaseWaveformModule):
116
+ is_denoiser = True
117
+
118
+ def forward(self, waveforms, **unused):
119
+ del waveforms, unused
120
+ raise NotImplementedError
121
+
122
+
123
+ class BaseWaveformFeaturizer(BaseWaveformModule):
124
+ is_featurizer = True
125
+ is_multi = False
126
+ # output shape per waveform
127
+ shape: tuple | list[tuple] = ()
128
+ # output dtye
129
+ dtype: torch.dtype | list[torch.dtype] = torch.float
130
+
131
+ def transform(self, waveforms, **unused):
132
+ del waveforms, unused
133
+ # returns dict {key=feat name, value=feature}
134
+ raise NotImplementedError
135
+
136
+ @property
137
+ def spike_datasets(self) -> Iterable[SpikeDataset]:
138
+ if self.is_multi:
139
+ assert isinstance(self.dtype, (list, tuple))
140
+ datasets = [
141
+ SpikeDataset(
142
+ name=n,
143
+ shape_per_spike=s,
144
+ dtype=str(d).split(".")[1],
145
+ )
146
+ for n, s, d in zip(self.name, self.shape, self.dtype)
147
+ ]
148
+ return datasets
149
+ else:
150
+ torch_dtype_as_str = str(self.dtype).split(".")[1]
151
+ dataset = SpikeDataset(
152
+ name=self.name,
153
+ shape_per_spike=self.shape,
154
+ dtype=torch_dtype_as_str,
155
+ )
156
+ return (dataset,)
157
+
158
+
159
+ class BaseWaveformAutoencoder(BaseWaveformDenoiser, BaseWaveformFeaturizer):
160
+ pass
161
+
162
+
163
+ class Passthrough(BaseWaveformDenoiser, BaseWaveformFeaturizer):
164
+ def __init__(
165
+ self, pipeline=None, geom=None, channel_index=None, name=None, name_prefix=None
166
+ ):
167
+ del geom, channel_index
168
+ t = []
169
+ if pipeline is not None:
170
+ t = [t for t in pipeline if t.is_featurizer]
171
+ if not len(t):
172
+ t = pipeline.transformers
173
+ if name is None:
174
+ name = f"passthrough_{t[0].name}"
175
+ super().__init__(name=name, name_prefix=name_prefix)
176
+ self.pipeline = pipeline
177
+
178
+ def needs_precompute(self):
179
+ if self.pipeline is None:
180
+ return False
181
+ return self.pipeline.needs_precompute()
182
+
183
+ def precompute(self):
184
+ if self.pipeline is None:
185
+ return
186
+ return self.pipeline.precompute()
187
+
188
+ def needs_fit(self):
189
+ if self.pipeline is None:
190
+ return False
191
+ return self.pipeline.needs_fit()
192
+
193
+ def fit(self, recording, waveforms, **fixed_properties):
194
+ if self.pipeline is None:
195
+ return
196
+ self.pipeline.fit(recording, waveforms, **fixed_properties)
197
+
198
+ def forward(self, waveforms, **fixed_properties):
199
+ if self.pipeline is None:
200
+ return waveforms, {}
201
+ pipeline_waveforms, pipeline_features = self.pipeline(
202
+ waveforms, **fixed_properties
203
+ )
204
+ del pipeline_waveforms # passthrough!
205
+ return waveforms, pipeline_features
206
+
207
+ @property
208
+ def spike_datasets(self):
209
+ datasets = []
210
+ if self.pipeline is not None:
211
+ for t in self.pipeline.transformers:
212
+ if t.is_featurizer:
213
+ datasets.extend(t.spike_datasets)
214
+ return datasets
215
+
216
+ def transform(self, waveforms, **fixed_properties):
217
+ if self.pipeline is None:
218
+ return {}
219
+ pipeline_waveforms, pipeline_features = self.pipeline(
220
+ waveforms, **fixed_properties
221
+ )
222
+ del pipeline_waveforms
223
+ return pipeline_features
224
+
225
+
226
+ class IdentityWaveformDenoiser(BaseWaveformDenoiser):
227
+ def forward(self, waveforms, **unused):
228
+ del unused
229
+ return waveforms
230
+
231
+
232
+ class Waveform(BaseWaveformFeaturizer):
233
+ default_name = "waveforms"
234
+
235
+ def __init__(
236
+ self,
237
+ channel_index,
238
+ geom=None,
239
+ spike_length_samples=121,
240
+ dtype=torch.float,
241
+ name=None,
242
+ name_prefix=None,
243
+ ):
244
+ super().__init__(
245
+ geom=geom,
246
+ channel_index=channel_index,
247
+ name=name,
248
+ name_prefix=name_prefix,
249
+ )
250
+ self.shape = (spike_length_samples, channel_index.shape[1])
251
+ self.dtype = dtype
252
+
253
+ def transform(self, waveforms, **unused):
254
+ del unused
255
+ return {self.name: waveforms.clone()}
File without changes