DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import torch
|
|
3
|
+
from sklearn.decomposition import PCA, TruncatedSVD
|
|
4
|
+
|
|
5
|
+
from ..util.waveform_util import (
|
|
6
|
+
channel_subset_by_radius,
|
|
7
|
+
get_channels_in_probe,
|
|
8
|
+
set_channels_in_probe,
|
|
9
|
+
)
|
|
10
|
+
from ..util.spiketorch import svd_lowrank_helper
|
|
11
|
+
from .transform_base import (
|
|
12
|
+
BaseWaveformAutoencoder,
|
|
13
|
+
BaseWaveformDenoiser,
|
|
14
|
+
BaseWaveformFeaturizer,
|
|
15
|
+
BaseWaveformModule,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BaseTemporalPCA(BaseWaveformModule):
|
|
20
|
+
default_name = "basis"
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
channel_index,
|
|
25
|
+
geom=None,
|
|
26
|
+
rank=8,
|
|
27
|
+
whiten=False,
|
|
28
|
+
centered=False,
|
|
29
|
+
fit_radius=None,
|
|
30
|
+
random_state=0,
|
|
31
|
+
name=None,
|
|
32
|
+
name_prefix="",
|
|
33
|
+
temporal_slice: slice | None = None,
|
|
34
|
+
n_oversamples=10,
|
|
35
|
+
niter=21,
|
|
36
|
+
max_waveforms=20_000,
|
|
37
|
+
batch_size=4096,
|
|
38
|
+
fit_dtype=torch.double,
|
|
39
|
+
):
|
|
40
|
+
if fit_radius is not None:
|
|
41
|
+
if geom is None or channel_index is None:
|
|
42
|
+
raise ValueError("TemporalPCA with fit_radius!=None requires geom.")
|
|
43
|
+
|
|
44
|
+
super().__init__(
|
|
45
|
+
channel_index=channel_index, geom=geom, name=name, name_prefix=name_prefix
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# behavior
|
|
49
|
+
self.rank = rank
|
|
50
|
+
self.centered = centered
|
|
51
|
+
self.whiten = whiten
|
|
52
|
+
self.temporal_slice = temporal_slice
|
|
53
|
+
self.batch_size = batch_size
|
|
54
|
+
|
|
55
|
+
# fit control
|
|
56
|
+
self.fit_radius = fit_radius
|
|
57
|
+
self.random_state = random_state
|
|
58
|
+
self.n_oversamples = n_oversamples
|
|
59
|
+
self.niter = niter
|
|
60
|
+
self.fit_dtype = fit_dtype
|
|
61
|
+
self.max_waveforms = max_waveforms
|
|
62
|
+
|
|
63
|
+
# gizmo
|
|
64
|
+
self._needs_fit = True
|
|
65
|
+
self.shape = (rank, channel_index.shape[1])
|
|
66
|
+
self.nt = None
|
|
67
|
+
|
|
68
|
+
def initialize_spike_length_dependent_params(self):
|
|
69
|
+
nt = self.spike_length_samples
|
|
70
|
+
assert nt is not None
|
|
71
|
+
if self.temporal_slice is None:
|
|
72
|
+
self._temporal_ix = None
|
|
73
|
+
else:
|
|
74
|
+
self.register_buffer("_temporal_ix", torch.arange(nt)[self.temporal_slice])
|
|
75
|
+
nt = self.b._temporal_ix.numel()
|
|
76
|
+
self.nt = nt
|
|
77
|
+
self.register_buffer("mean", torch.zeros(nt))
|
|
78
|
+
self.register_buffer("components", torch.zeros(self.rank, nt))
|
|
79
|
+
self.register_buffer("whitener", torch.zeros(self.rank))
|
|
80
|
+
self.to(self.b.channel_index.device)
|
|
81
|
+
|
|
82
|
+
def fit(
|
|
83
|
+
self,
|
|
84
|
+
recording,
|
|
85
|
+
waveforms,
|
|
86
|
+
*,
|
|
87
|
+
channels,
|
|
88
|
+
weights=None,
|
|
89
|
+
time_shifts=None,
|
|
90
|
+
**unused,
|
|
91
|
+
):
|
|
92
|
+
super().fit(recording, waveforms, channels=channels, weights=weights)
|
|
93
|
+
if weights is not None and waveforms.shape[0] > self.max_waveforms:
|
|
94
|
+
self.random_state = np.random.default_rng(self.random_state)
|
|
95
|
+
weights = weights.numpy(force=True) if torch.is_tensor(weights) else weights
|
|
96
|
+
weights = weights.astype(np.float64)
|
|
97
|
+
weights = weights / weights.sum()
|
|
98
|
+
choices = self.random_state.choice(
|
|
99
|
+
len(weights), p=weights, size=self.max_waveforms
|
|
100
|
+
)
|
|
101
|
+
choices.sort()
|
|
102
|
+
choices = torch.from_numpy(choices)
|
|
103
|
+
waveforms = waveforms[choices]
|
|
104
|
+
channels = channels[choices]
|
|
105
|
+
waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
|
|
106
|
+
self.dtype = waveforms.dtype
|
|
107
|
+
if self.fit_radius is not None:
|
|
108
|
+
waveforms, train_channel_index = channel_subset_by_radius(
|
|
109
|
+
waveforms,
|
|
110
|
+
channels,
|
|
111
|
+
self.channel_index.to(device=waveforms.device),
|
|
112
|
+
self.geom.to(device=waveforms.device),
|
|
113
|
+
self.fit_radius,
|
|
114
|
+
)
|
|
115
|
+
else:
|
|
116
|
+
train_channel_index = self.b.channel_index.to(waveforms.device)
|
|
117
|
+
_, waveforms_fit = get_channels_in_probe(
|
|
118
|
+
waveforms, channels, train_channel_index
|
|
119
|
+
)
|
|
120
|
+
waveforms_fit = waveforms_fit.to(self.b.channel_index.device)
|
|
121
|
+
|
|
122
|
+
if self.centered:
|
|
123
|
+
mean = waveforms_fit.mean(0)
|
|
124
|
+
else:
|
|
125
|
+
mean = torch.zeros_like(waveforms_fit[0])
|
|
126
|
+
|
|
127
|
+
if self.centered:
|
|
128
|
+
# torch does not seem always to want to broadcast M as advertised?
|
|
129
|
+
M = mean[None].broadcast_to(waveforms_fit.shape)
|
|
130
|
+
else:
|
|
131
|
+
M = None
|
|
132
|
+
|
|
133
|
+
_, components, _, whitener = svd_lowrank_helper(
|
|
134
|
+
x=waveforms_fit,
|
|
135
|
+
rank=self.rank,
|
|
136
|
+
n_oversamples=self.n_oversamples,
|
|
137
|
+
fit_dtype=self.fit_dtype,
|
|
138
|
+
niter=self.niter,
|
|
139
|
+
M=M,
|
|
140
|
+
with_loadings=False,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
self.b.mean.copy_(mean)
|
|
144
|
+
self.b.components.copy_(components)
|
|
145
|
+
self.b.whitener.copy_(whitener)
|
|
146
|
+
self._needs_fit = False
|
|
147
|
+
|
|
148
|
+
def needs_fit(self):
|
|
149
|
+
return self._needs_fit
|
|
150
|
+
|
|
151
|
+
def _temporal_slice(
|
|
152
|
+
self, waveforms: torch.Tensor, time_shifts=None
|
|
153
|
+
) -> torch.Tensor:
|
|
154
|
+
if self.temporal_slice is None:
|
|
155
|
+
return waveforms
|
|
156
|
+
|
|
157
|
+
if time_shifts is None:
|
|
158
|
+
return waveforms[:, self.temporal_slice]
|
|
159
|
+
|
|
160
|
+
assert self._temporal_ix is not None
|
|
161
|
+
n, t, c = waveforms.shape
|
|
162
|
+
t_ = self._temporal_ix.numel()
|
|
163
|
+
assert t_ <= t
|
|
164
|
+
assert time_shifts.shape == (n,)
|
|
165
|
+
time_ix = self._temporal_ix[None, :, None] + time_shifts[:, None, None]
|
|
166
|
+
waveforms = waveforms.take_along_dim(dim=1, indices=time_ix)
|
|
167
|
+
assert waveforms.shape == (n, t_, c)
|
|
168
|
+
return waveforms
|
|
169
|
+
|
|
170
|
+
def _transform_in_probe(self, waveforms_in_probe):
|
|
171
|
+
n = waveforms_in_probe.shape[0]
|
|
172
|
+
if self.whiten:
|
|
173
|
+
W = self.b.components / self.b.whitener
|
|
174
|
+
else:
|
|
175
|
+
W = self.b.components
|
|
176
|
+
out = waveforms_in_probe.new_empty((n, self.rank))
|
|
177
|
+
dev = self.b.components.device
|
|
178
|
+
same_dev = out.device == dev
|
|
179
|
+
for i0 in range(0, n, self.batch_size):
|
|
180
|
+
i1 = min(n, i0 + self.batch_size)
|
|
181
|
+
x = waveforms_in_probe[i0:i1].to(device=dev)
|
|
182
|
+
if self.centered:
|
|
183
|
+
x = x - self.mean
|
|
184
|
+
if same_dev:
|
|
185
|
+
torch.mm(x, W.T, out=out[i0:i1])
|
|
186
|
+
else:
|
|
187
|
+
out[i0:i1] = (x @ W.T).to(out)
|
|
188
|
+
return out
|
|
189
|
+
|
|
190
|
+
def _inverse_transform_in_probe(self, features):
|
|
191
|
+
if self.whiten:
|
|
192
|
+
W = self.b.components * self.b.whitener
|
|
193
|
+
else:
|
|
194
|
+
W = self.b.components
|
|
195
|
+
assert self.nt is not None
|
|
196
|
+
n = features.shape[0]
|
|
197
|
+
out = features.new_zeros((n, self.nt))
|
|
198
|
+
dev = self.b.components.device
|
|
199
|
+
for i0 in range(0, n, self.batch_size):
|
|
200
|
+
i1 = min(n, i0 + self.batch_size)
|
|
201
|
+
x = features[i0:i1].to(device=dev)
|
|
202
|
+
if out.device == dev:
|
|
203
|
+
if self.centered:
|
|
204
|
+
torch.addmm(self.b.mean, x, W, out=out[i0:i1])
|
|
205
|
+
else:
|
|
206
|
+
torch.mm(x, W, out=out[i0:i1])
|
|
207
|
+
else:
|
|
208
|
+
if self.centered:
|
|
209
|
+
out[i0:i1] = torch.addmm(self.b.mean, x, W).to(out)
|
|
210
|
+
else:
|
|
211
|
+
out[i0:i1] = torch.mm(x, W).to(out)
|
|
212
|
+
return out
|
|
213
|
+
|
|
214
|
+
def _project_in_probe(self, waveforms_in_probe):
|
|
215
|
+
n = waveforms_in_probe.shape[0]
|
|
216
|
+
out = torch.empty_like(waveforms_in_probe)
|
|
217
|
+
proj = self.b.components.T @ self.b.components
|
|
218
|
+
dev = proj.device
|
|
219
|
+
same_dev = out.device == dev
|
|
220
|
+
for i0 in range(0, n, self.batch_size):
|
|
221
|
+
i1 = min(n, i0 + self.batch_size)
|
|
222
|
+
x = waveforms_in_probe[i0:i1].to(device=dev)
|
|
223
|
+
if same_dev:
|
|
224
|
+
if self.centered:
|
|
225
|
+
torch.addmm(self.b.mean, x - self.mean, proj, out=out[i0:i1])
|
|
226
|
+
else:
|
|
227
|
+
torch.mm(waveforms_in_probe, proj, out=out[i0:i1])
|
|
228
|
+
else:
|
|
229
|
+
if self.centered:
|
|
230
|
+
out[i0:i1] = torch.addmm(self.b.mean, x - self.mean, proj).to(out)
|
|
231
|
+
else:
|
|
232
|
+
out[i0:i1] = torch.mm(x, proj).to(out)
|
|
233
|
+
return out
|
|
234
|
+
|
|
235
|
+
def force_reconstruct(self, features):
|
|
236
|
+
ndim = features.ndim
|
|
237
|
+
if ndim == 2:
|
|
238
|
+
features = features.unsqueeze(0)
|
|
239
|
+
n, r, c = features.shape
|
|
240
|
+
waveforms = features.permute(0, 2, 1).reshape(n * c, r)
|
|
241
|
+
waveforms = self._inverse_transform_in_probe(waveforms)
|
|
242
|
+
waveforms = waveforms.reshape(n, c, -1).permute(0, 2, 1)
|
|
243
|
+
if ndim == 2:
|
|
244
|
+
waveforms = waveforms[0]
|
|
245
|
+
return waveforms
|
|
246
|
+
|
|
247
|
+
def force_project(self, features):
|
|
248
|
+
ndim = features.ndim
|
|
249
|
+
if ndim == 2:
|
|
250
|
+
features = features.unsqueeze(0)
|
|
251
|
+
n, t, c = features.shape
|
|
252
|
+
waveforms = features.mT.reshape(n * c, t)
|
|
253
|
+
waveforms = self._project_in_probe(waveforms)
|
|
254
|
+
waveforms = waveforms.reshape(n, c, t).mT
|
|
255
|
+
if ndim == 2:
|
|
256
|
+
waveforms = waveforms[0]
|
|
257
|
+
return waveforms
|
|
258
|
+
|
|
259
|
+
def to_sklearn(self) -> PCA:
|
|
260
|
+
pca = PCA(
|
|
261
|
+
n_components=self.rank,
|
|
262
|
+
random_state=self.random_state, # type: ignore
|
|
263
|
+
whiten=self.whiten,
|
|
264
|
+
)
|
|
265
|
+
pca.mean_ = self.b.mean.numpy(force=True)
|
|
266
|
+
pca.components_ = self.b.components.numpy(force=True)
|
|
267
|
+
pca.explained_variance_ = np.square(self.b.whitener.numpy(force=True))
|
|
268
|
+
pca.temporal_slice = self.temporal_slice # this is not standard # type: ignore
|
|
269
|
+
return pca
|
|
270
|
+
|
|
271
|
+
@classmethod
|
|
272
|
+
def from_sklearn(cls, channel_index, pca: PCA | TruncatedSVD, temporal_slice=None):
|
|
273
|
+
if isinstance(pca, PCA):
|
|
274
|
+
whiten = pca.whiten # type: ignore
|
|
275
|
+
elif isinstance(pca, TruncatedSVD):
|
|
276
|
+
whiten = False
|
|
277
|
+
else:
|
|
278
|
+
assert False
|
|
279
|
+
self = cls(
|
|
280
|
+
channel_index,
|
|
281
|
+
rank=pca.n_components, # type: ignore
|
|
282
|
+
whiten=whiten,
|
|
283
|
+
temporal_slice=temporal_slice,
|
|
284
|
+
)
|
|
285
|
+
self.initialize_from_sklearn(pca)
|
|
286
|
+
return self
|
|
287
|
+
|
|
288
|
+
@classmethod
|
|
289
|
+
def convert(cls, other):
|
|
290
|
+
assert not other._needs_fit # doesn't handle fit_radius right.
|
|
291
|
+
self = cls(
|
|
292
|
+
channel_index=other.channel_index,
|
|
293
|
+
rank=other.rank,
|
|
294
|
+
centered=other.centered,
|
|
295
|
+
whiten=other.whiten,
|
|
296
|
+
temporal_slice=other.temporal_slice,
|
|
297
|
+
)
|
|
298
|
+
self._needs_fit = other._needs_fit
|
|
299
|
+
self.spike_length_samples = other.spike_length_samples
|
|
300
|
+
self.initialize_spike_length_dependent_params()
|
|
301
|
+
self.to(other.channel_index.device)
|
|
302
|
+
self.b.mean.copy_(other.mean)
|
|
303
|
+
self.b.components.copy_(other.components)
|
|
304
|
+
self.b.whitener.copy_(other.whitener)
|
|
305
|
+
return self
|
|
306
|
+
|
|
307
|
+
def initialize_from_sklearn(self, pca):
|
|
308
|
+
if self.temporal_slice is None:
|
|
309
|
+
self.spike_length_samples = pca.components_.shape[1]
|
|
310
|
+
else:
|
|
311
|
+
# not really -- this is a hack.
|
|
312
|
+
self.spike_length_samples = (
|
|
313
|
+
self.temporal_slice.stop - self.temporal_slice.start
|
|
314
|
+
)
|
|
315
|
+
self.initialize_spike_length_dependent_params()
|
|
316
|
+
if hasattr(pca, "mean_"):
|
|
317
|
+
self.b.mean.copy_(torch.from_numpy(pca.mean_))
|
|
318
|
+
self.center = not (self.b.mean == 0.0).all()
|
|
319
|
+
else:
|
|
320
|
+
self.b.mean.zero_()
|
|
321
|
+
self.center = False
|
|
322
|
+
self.b.components.copy_(torch.from_numpy(pca.components_))
|
|
323
|
+
self.b.whitener.copy_(torch.from_numpy(pca.explained_variance_)).sqrt_()
|
|
324
|
+
self._needs_fit = False
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
class TemporalPCADenoiser(BaseWaveformDenoiser, BaseTemporalPCA):
|
|
328
|
+
default_name = "temporal_pca"
|
|
329
|
+
|
|
330
|
+
def forward(self, waveforms, *, channels, time_shifts=None, **unused):
|
|
331
|
+
waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
|
|
332
|
+
dev = waveforms.device
|
|
333
|
+
channels_in_probe, waveforms_in_probe = get_channels_in_probe(
|
|
334
|
+
waveforms, channels.to(device=dev), self.channel_index.to(device=dev)
|
|
335
|
+
)
|
|
336
|
+
waveforms_in_probe = self._project_in_probe(waveforms_in_probe)
|
|
337
|
+
return set_channels_in_probe(waveforms_in_probe, waveforms, channels_in_probe)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class TemporalPCAFeaturizer(BaseWaveformFeaturizer, BaseTemporalPCA):
|
|
341
|
+
default_name = "tpca_features"
|
|
342
|
+
|
|
343
|
+
def transform(
|
|
344
|
+
self,
|
|
345
|
+
waveforms,
|
|
346
|
+
*,
|
|
347
|
+
channels,
|
|
348
|
+
channel_index=None,
|
|
349
|
+
_return_for_combined=False,
|
|
350
|
+
time_shifts=None,
|
|
351
|
+
**unused,
|
|
352
|
+
):
|
|
353
|
+
waveforms = self._temporal_slice(waveforms, time_shifts=time_shifts)
|
|
354
|
+
|
|
355
|
+
if channel_index is None:
|
|
356
|
+
channel_index = self.b.channel_index
|
|
357
|
+
dev = waveforms.device
|
|
358
|
+
channels_in_probe, waveforms_in_probe = get_channels_in_probe(
|
|
359
|
+
waveforms, channels.to(device=dev), channel_index.to(device=dev)
|
|
360
|
+
)
|
|
361
|
+
features_in_probe = self._transform_in_probe(waveforms_in_probe)
|
|
362
|
+
features = features_in_probe.new_full(
|
|
363
|
+
(waveforms.shape[0], self.rank, channel_index.shape[1]), torch.nan
|
|
364
|
+
)
|
|
365
|
+
features = set_channels_in_probe(
|
|
366
|
+
features_in_probe,
|
|
367
|
+
features,
|
|
368
|
+
channels_in_probe,
|
|
369
|
+
)
|
|
370
|
+
if _return_for_combined:
|
|
371
|
+
return (
|
|
372
|
+
waveforms,
|
|
373
|
+
features_in_probe,
|
|
374
|
+
channels_in_probe,
|
|
375
|
+
{self.name: features},
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
return {self.name: features}
|
|
379
|
+
|
|
380
|
+
def inverse_transform(self, features, channels, channel_index=None):
|
|
381
|
+
if channel_index is None:
|
|
382
|
+
channel_index = self.b.channel_index
|
|
383
|
+
channels_in_probe, features_in_probe = get_channels_in_probe(
|
|
384
|
+
features, channels, channel_index
|
|
385
|
+
)
|
|
386
|
+
reconstructions_in_probe = self._inverse_transform_in_probe(features_in_probe)
|
|
387
|
+
recshp = (features.shape[0], self.b.components.shape[1], channel_index.shape[1])
|
|
388
|
+
reconstructions = torch.full(
|
|
389
|
+
recshp,
|
|
390
|
+
torch.nan,
|
|
391
|
+
dtype=reconstructions_in_probe.dtype,
|
|
392
|
+
device=reconstructions_in_probe.device,
|
|
393
|
+
)
|
|
394
|
+
return set_channels_in_probe(
|
|
395
|
+
reconstructions_in_probe, reconstructions, channels_in_probe
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
class TemporalPCA(BaseWaveformAutoencoder, TemporalPCAFeaturizer):
|
|
400
|
+
default_name = "tpca_features"
|
|
401
|
+
|
|
402
|
+
def forward(self, waveforms, *, channels, time_shifts=None, **unused):
|
|
403
|
+
waveforms, features_in_probe, channels_in_probe, features = self.transform(
|
|
404
|
+
waveforms,
|
|
405
|
+
channels=channels,
|
|
406
|
+
time_shifts=time_shifts,
|
|
407
|
+
_return_for_combined=True,
|
|
408
|
+
)
|
|
409
|
+
reconstructions_in_probe = self._inverse_transform_in_probe(features_in_probe)
|
|
410
|
+
reconstructions = set_channels_in_probe(
|
|
411
|
+
reconstructions_in_probe, waveforms, channels_in_probe
|
|
412
|
+
)
|
|
413
|
+
return reconstructions, features
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
from typing import Any, Iterable
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from ..util.data_util import SpikeDataset
|
|
6
|
+
from ..util.torch_util import BModule
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class BaseWaveformModule(BModule):
|
|
10
|
+
is_denoiser = False
|
|
11
|
+
is_featurizer = False
|
|
12
|
+
default_name = ""
|
|
13
|
+
|
|
14
|
+
def __init__(self, channel_index=None, geom=None, name=None, name_prefix=None):
|
|
15
|
+
super().__init__()
|
|
16
|
+
if name is None:
|
|
17
|
+
name = self.default_name
|
|
18
|
+
if name_prefix:
|
|
19
|
+
name = f"{name_prefix}_{name}"
|
|
20
|
+
self.name = name
|
|
21
|
+
# these buffers below need to be copied, else they share references
|
|
22
|
+
# across all the transformers which seems to cause problems!
|
|
23
|
+
if channel_index is not None:
|
|
24
|
+
channel_index = torch.asarray(channel_index, copy=True)
|
|
25
|
+
self.register_buffer("channel_index", channel_index)
|
|
26
|
+
if geom is not None:
|
|
27
|
+
geom = torch.asarray(geom, dtype=torch.float, copy=True)
|
|
28
|
+
self.register_buffer("geom", geom)
|
|
29
|
+
|
|
30
|
+
self.spike_length_samples = None
|
|
31
|
+
try:
|
|
32
|
+
self._hook = self.register_load_state_dict_pre_hook(
|
|
33
|
+
self.__class__._pre_load_state
|
|
34
|
+
)
|
|
35
|
+
except AttributeError:
|
|
36
|
+
# ...? seems to happen in 2.4.1...
|
|
37
|
+
self._hook = self._register_load_state_dict_pre_hook(
|
|
38
|
+
self.__class__._pre_load_state
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def __getstate__(self):
|
|
42
|
+
self._hook.remove()
|
|
43
|
+
state = super().__getstate__()
|
|
44
|
+
del state["_hook"]
|
|
45
|
+
return state
|
|
46
|
+
|
|
47
|
+
def __setstate__(self, state):
|
|
48
|
+
super().__setstate__(state)
|
|
49
|
+
try:
|
|
50
|
+
self._hook = self.register_load_state_dict_pre_hook(
|
|
51
|
+
self.__class__._pre_load_state
|
|
52
|
+
)
|
|
53
|
+
except AttributeError:
|
|
54
|
+
self._hook = self._register_load_state_dict_pre_hook(
|
|
55
|
+
self.__class__._pre_load_state
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def fit(self, recording, waveforms, **fixed_properties) -> Any:
|
|
59
|
+
del recording, fixed_properties
|
|
60
|
+
self.spike_length_samples = waveforms.shape[1]
|
|
61
|
+
self.initialize_spike_length_dependent_params()
|
|
62
|
+
|
|
63
|
+
def get_extra_state(self):
|
|
64
|
+
return dict(
|
|
65
|
+
spike_length_samples=self.spike_length_samples, needs_fit=self.needs_fit()
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def set_extra_state(self, state):
|
|
69
|
+
self.spike_length_samples = state["spike_length_samples"]
|
|
70
|
+
if hasattr(self, "_needs_fit"):
|
|
71
|
+
self._needs_fit = state["needs_fit"]
|
|
72
|
+
self._init_bgetter()
|
|
73
|
+
|
|
74
|
+
def _pre_load_state(
|
|
75
|
+
self,
|
|
76
|
+
state_dict,
|
|
77
|
+
prefix,
|
|
78
|
+
local_metadata,
|
|
79
|
+
strict,
|
|
80
|
+
missing_keys,
|
|
81
|
+
unexpected_keys,
|
|
82
|
+
error_msgs,
|
|
83
|
+
):
|
|
84
|
+
# wish torch would strip the prefix for us?
|
|
85
|
+
extra_state_keys = [k for k in state_dict.keys() if k.endswith("_extra_state")]
|
|
86
|
+
assert len(extra_state_keys) == 1
|
|
87
|
+
extra_state = state_dict[extra_state_keys[0]]
|
|
88
|
+
|
|
89
|
+
# some modules want to know the spike length before loading the state dict
|
|
90
|
+
# and unfortunately set_extra_state usually runs after. doesn't hurt to run now.
|
|
91
|
+
self.spike_length_samples = extra_state["spike_length_samples"]
|
|
92
|
+
|
|
93
|
+
# and this is how subclasses use that info
|
|
94
|
+
# if state dict was dumped before fit, then sls was never known and we
|
|
95
|
+
# don't want to call that initializer, so don't if sls is None.
|
|
96
|
+
if self.spike_length_samples is not None:
|
|
97
|
+
self.initialize_spike_length_dependent_params()
|
|
98
|
+
|
|
99
|
+
def initialize_spike_length_dependent_params(self):
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
def needs_fit(self) -> bool:
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
def needs_precompute(self) -> bool:
|
|
106
|
+
return False
|
|
107
|
+
|
|
108
|
+
def precompute(self):
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
def extra_repr(self):
|
|
112
|
+
return f"name={self.name},needs_fit={self.needs_fit()}"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class BaseWaveformDenoiser(BaseWaveformModule):
|
|
116
|
+
is_denoiser = True
|
|
117
|
+
|
|
118
|
+
def forward(self, waveforms, **unused):
|
|
119
|
+
del waveforms, unused
|
|
120
|
+
raise NotImplementedError
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class BaseWaveformFeaturizer(BaseWaveformModule):
|
|
124
|
+
is_featurizer = True
|
|
125
|
+
is_multi = False
|
|
126
|
+
# output shape per waveform
|
|
127
|
+
shape: tuple | list[tuple] = ()
|
|
128
|
+
# output dtye
|
|
129
|
+
dtype: torch.dtype | list[torch.dtype] = torch.float
|
|
130
|
+
|
|
131
|
+
def transform(self, waveforms, **unused):
|
|
132
|
+
del waveforms, unused
|
|
133
|
+
# returns dict {key=feat name, value=feature}
|
|
134
|
+
raise NotImplementedError
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def spike_datasets(self) -> Iterable[SpikeDataset]:
|
|
138
|
+
if self.is_multi:
|
|
139
|
+
assert isinstance(self.dtype, (list, tuple))
|
|
140
|
+
datasets = [
|
|
141
|
+
SpikeDataset(
|
|
142
|
+
name=n,
|
|
143
|
+
shape_per_spike=s,
|
|
144
|
+
dtype=str(d).split(".")[1],
|
|
145
|
+
)
|
|
146
|
+
for n, s, d in zip(self.name, self.shape, self.dtype)
|
|
147
|
+
]
|
|
148
|
+
return datasets
|
|
149
|
+
else:
|
|
150
|
+
torch_dtype_as_str = str(self.dtype).split(".")[1]
|
|
151
|
+
dataset = SpikeDataset(
|
|
152
|
+
name=self.name,
|
|
153
|
+
shape_per_spike=self.shape,
|
|
154
|
+
dtype=torch_dtype_as_str,
|
|
155
|
+
)
|
|
156
|
+
return (dataset,)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class BaseWaveformAutoencoder(BaseWaveformDenoiser, BaseWaveformFeaturizer):
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class Passthrough(BaseWaveformDenoiser, BaseWaveformFeaturizer):
|
|
164
|
+
def __init__(
|
|
165
|
+
self, pipeline=None, geom=None, channel_index=None, name=None, name_prefix=None
|
|
166
|
+
):
|
|
167
|
+
del geom, channel_index
|
|
168
|
+
t = []
|
|
169
|
+
if pipeline is not None:
|
|
170
|
+
t = [t for t in pipeline if t.is_featurizer]
|
|
171
|
+
if not len(t):
|
|
172
|
+
t = pipeline.transformers
|
|
173
|
+
if name is None:
|
|
174
|
+
name = f"passthrough_{t[0].name}"
|
|
175
|
+
super().__init__(name=name, name_prefix=name_prefix)
|
|
176
|
+
self.pipeline = pipeline
|
|
177
|
+
|
|
178
|
+
def needs_precompute(self):
|
|
179
|
+
if self.pipeline is None:
|
|
180
|
+
return False
|
|
181
|
+
return self.pipeline.needs_precompute()
|
|
182
|
+
|
|
183
|
+
def precompute(self):
|
|
184
|
+
if self.pipeline is None:
|
|
185
|
+
return
|
|
186
|
+
return self.pipeline.precompute()
|
|
187
|
+
|
|
188
|
+
def needs_fit(self):
|
|
189
|
+
if self.pipeline is None:
|
|
190
|
+
return False
|
|
191
|
+
return self.pipeline.needs_fit()
|
|
192
|
+
|
|
193
|
+
def fit(self, recording, waveforms, **fixed_properties):
|
|
194
|
+
if self.pipeline is None:
|
|
195
|
+
return
|
|
196
|
+
self.pipeline.fit(recording, waveforms, **fixed_properties)
|
|
197
|
+
|
|
198
|
+
def forward(self, waveforms, **fixed_properties):
|
|
199
|
+
if self.pipeline is None:
|
|
200
|
+
return waveforms, {}
|
|
201
|
+
pipeline_waveforms, pipeline_features = self.pipeline(
|
|
202
|
+
waveforms, **fixed_properties
|
|
203
|
+
)
|
|
204
|
+
del pipeline_waveforms # passthrough!
|
|
205
|
+
return waveforms, pipeline_features
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def spike_datasets(self):
|
|
209
|
+
datasets = []
|
|
210
|
+
if self.pipeline is not None:
|
|
211
|
+
for t in self.pipeline.transformers:
|
|
212
|
+
if t.is_featurizer:
|
|
213
|
+
datasets.extend(t.spike_datasets)
|
|
214
|
+
return datasets
|
|
215
|
+
|
|
216
|
+
def transform(self, waveforms, **fixed_properties):
|
|
217
|
+
if self.pipeline is None:
|
|
218
|
+
return {}
|
|
219
|
+
pipeline_waveforms, pipeline_features = self.pipeline(
|
|
220
|
+
waveforms, **fixed_properties
|
|
221
|
+
)
|
|
222
|
+
del pipeline_waveforms
|
|
223
|
+
return pipeline_features
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class IdentityWaveformDenoiser(BaseWaveformDenoiser):
|
|
227
|
+
def forward(self, waveforms, **unused):
|
|
228
|
+
del unused
|
|
229
|
+
return waveforms
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
class Waveform(BaseWaveformFeaturizer):
|
|
233
|
+
default_name = "waveforms"
|
|
234
|
+
|
|
235
|
+
def __init__(
|
|
236
|
+
self,
|
|
237
|
+
channel_index,
|
|
238
|
+
geom=None,
|
|
239
|
+
spike_length_samples=121,
|
|
240
|
+
dtype=torch.float,
|
|
241
|
+
name=None,
|
|
242
|
+
name_prefix=None,
|
|
243
|
+
):
|
|
244
|
+
super().__init__(
|
|
245
|
+
geom=geom,
|
|
246
|
+
channel_index=channel_index,
|
|
247
|
+
name=name,
|
|
248
|
+
name_prefix=name_prefix,
|
|
249
|
+
)
|
|
250
|
+
self.shape = (spike_length_samples, channel_index.shape[1])
|
|
251
|
+
self.dtype = dtype
|
|
252
|
+
|
|
253
|
+
def transform(self, waveforms, **unused):
|
|
254
|
+
del unused
|
|
255
|
+
return {self.name: waveforms.clone()}
|
|
File without changes
|