DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,503 @@
1
+ from typing import cast
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.nn.functional as F
6
+ from dartsort.util import nn_util
7
+ from dartsort.util.spiketorch import get_relative_index, ptp, reindex, spawn_torch_rg
8
+ from dartsort.util.waveform_util import make_regular_channel_index
9
+ from torch.utils.data import (
10
+ DataLoader,
11
+ TensorDataset,
12
+ RandomSampler,
13
+ BatchSampler,
14
+ WeightedRandomSampler,
15
+ )
16
+ from tqdm.auto import trange
17
+
18
+ from .transform_base import BaseWaveformFeaturizer
19
+
20
+
21
+ class AmortizedLocalization(BaseWaveformFeaturizer):
22
+ """Order of output columns: x, y, z_abs."""
23
+
24
+ default_name = "point_source_localizations"
25
+ shape = (3,)
26
+ dtype = torch.float
27
+
28
+ def __init__(
29
+ self,
30
+ channel_index,
31
+ geom,
32
+ radius=100.0,
33
+ amplitude_kind="peak",
34
+ localization_model="pointsource",
35
+ hidden_dims=(256, 128),
36
+ name=None,
37
+ name_prefix="",
38
+ n_epochs=100,
39
+ learning_rate=3e-3,
40
+ batch_size=32,
41
+ inference_batch_size=2**14,
42
+ norm_kind="layernorm",
43
+ alpha_closed_form=True,
44
+ amplitudes_only=True,
45
+ prior_variance=None,
46
+ convergence_rtol=0.01,
47
+ convergence_atol=1e-4,
48
+ min_epochs=10,
49
+ scale_loss_by_mean=True,
50
+ reference="main_channel",
51
+ softmax_noise_floor=False,
52
+ channelwise_dropout_p=0.00,
53
+ epoch_size=50_000,
54
+ val_split_p=0.3,
55
+ random_seed=0,
56
+ ):
57
+ assert localization_model in ("pointsource", "dipole", "gaussian")
58
+ assert amplitude_kind in ("peak", "ptp")
59
+ assert reference in ("main_channel", "com")
60
+ super().__init__(
61
+ geom=geom, channel_index=channel_index, name=name, name_prefix=name_prefix
62
+ )
63
+
64
+ self.amplitude_kind = amplitude_kind
65
+ self.radius = radius
66
+ self.localization_model = localization_model
67
+ alpha_dim = 1 + 2 * (localization_model == "dipole")
68
+ self.latent_dim = 3 + (not alpha_closed_form) * alpha_dim + softmax_noise_floor
69
+ self.softmax_noise_floor = softmax_noise_floor
70
+ self.n_epochs = n_epochs
71
+ self.learning_rate = learning_rate
72
+ self.batch_size = batch_size
73
+ self.encoder = None
74
+ self.amplitudes_only = amplitudes_only
75
+ self.norm_kind = norm_kind
76
+ self.hidden_dims = hidden_dims
77
+ self.alpha_closed_form = alpha_closed_form
78
+ self.variational = prior_variance is not None
79
+ self.prior_variance = (
80
+ torch.tensor(prior_variance) if prior_variance is not None else None
81
+ )
82
+ self.convergence_atol = convergence_atol
83
+ self.convergence_rtol = convergence_rtol
84
+ self.min_epochs = min_epochs
85
+ self.scale_loss_by_mean = scale_loss_by_mean
86
+ self.channelwise_dropout_p = channelwise_dropout_p
87
+ self.reference = reference
88
+ self.epoch_size = epoch_size
89
+ self.val_split_p = val_split_p
90
+ self.random_seed = random_seed
91
+ self.inference_batch_size = inference_batch_size
92
+ self.nc = self.b.geom.shape[0]
93
+
94
+ if self.nc > 1:
95
+ self.register_buffer(
96
+ "padded_geom", F.pad(self.b.geom.to(torch.float), (0, 0, 0, 1))
97
+ )
98
+ mci = make_regular_channel_index(
99
+ geom=self.b.geom, radius=radius, to_torch=True
100
+ )
101
+ self.register_buffer("model_channel_index", mci)
102
+ ri = get_relative_index(self.channel_index, self.b.model_channel_index)
103
+ self.register_buffer("relative_index", ri)
104
+ self._needs_fit = True
105
+ else:
106
+ self._needs_fit = False
107
+
108
+ def needs_fit(self):
109
+ return self._needs_fit
110
+
111
+ def fit(self, recording, waveforms, *, channels, weights=None, **unused):
112
+ super().fit(recording, waveforms, channels=channels, weights=weights)
113
+ with torch.enable_grad():
114
+ self._fit(waveforms, channels, weights=weights)
115
+ self.eval()
116
+ self._needs_fit = False
117
+
118
+ def initialize_spike_length_dependent_params(self):
119
+ if self.encoder is not None:
120
+ return
121
+ if self.nc == 1:
122
+ return
123
+
124
+ n_latent = self.latent_dim
125
+ if self.variational:
126
+ n_latent *= 2
127
+
128
+ self.encoder = nn_util.get_waveform_mlp(
129
+ 1 if self.amplitudes_only else self.spike_length_samples,
130
+ self.b.model_channel_index.shape[1],
131
+ self.hidden_dims,
132
+ n_latent,
133
+ norm_kind=self.norm_kind,
134
+ channelwise_dropout_p=self.channelwise_dropout_p,
135
+ )
136
+ self.encoder.to(self.b.padded_geom.device)
137
+
138
+ def reparameterize(self, mu, var):
139
+ if var is None:
140
+ return mu
141
+ std = var.sqrt()
142
+ eps = torch.randn_like(std)
143
+ return mu + eps * std
144
+
145
+ def get_reference_points(self, channels, obs_amps=None, neighborhoods=None):
146
+ if self.reference == "main_channel":
147
+ return self.b.padded_geom[channels]
148
+ elif self.reference == "com":
149
+ assert obs_amps is not None
150
+ if neighborhoods is None:
151
+ neighborhoods = self.b.padded_geom[self.b.model_channel_index[channels]]
152
+ w = obs_amps / obs_amps.sum(1, keepdims=True)
153
+ centers = torch.sum(w.unsqueeze(-1) * neighborhoods, dim=1)
154
+ return centers
155
+ else:
156
+ assert False
157
+
158
+ def local_distances(self, z, channels, obs_amps=None):
159
+ """Return distances from each z to its local geom centered at channels."""
160
+ neighbors = self.b.padded_geom[self.b.model_channel_index[channels]]
161
+ centers = self.get_reference_points(
162
+ channels, obs_amps=obs_amps, neighborhoods=neighbors
163
+ )
164
+ local_geom = neighbors - centers.unsqueeze(1)
165
+ dx = z[:, 0, None] - local_geom[:, :, 0]
166
+ dz = z[:, 2, None] - local_geom[:, :, 1]
167
+ y = F.softplus(z[:, 1]).unsqueeze(1)
168
+ dists = torch.sqrt(dx**2 + dz**2 + y**2)
169
+ return dists
170
+
171
+ def get_alphas(self, obs_amps, pred_amps_alpha1, masks, return_pred=False):
172
+ # least squares with no intercept
173
+ alphas = (masks * obs_amps * pred_amps_alpha1).sum(1) / (
174
+ masks * pred_amps_alpha1
175
+ ).square().sum(1)
176
+ if return_pred:
177
+ return alphas, alphas.unsqueeze(1) * pred_amps_alpha1
178
+ return alphas
179
+
180
+ def point_source_model(self, z, obs_amps, masks, channels):
181
+ dists = self.local_distances(z, channels, obs_amps=obs_amps)
182
+ if self.alpha_closed_form:
183
+ if self.localization_model == "gaussian":
184
+ pred_amps_alpha1 = dists.square().mul(-2).exp()
185
+ else:
186
+ pred_amps_alpha1 = 1.0 / dists
187
+ alphas, pred_amps = self.get_alphas(
188
+ obs_amps, pred_amps_alpha1, masks, return_pred=True
189
+ )
190
+ else:
191
+ alphas = F.softplus(z[:, 3])
192
+ pred_amps = alphas.unsqueeze(1) / dists
193
+
194
+ noise_floors = None
195
+ if self.softmax_noise_floor:
196
+ noise_floors = F.softplus(z[:, -1:]).broadcast_to(pred_amps.shape)
197
+ pred_amps_noise = torch.stack([pred_amps, noise_floors], dim=2)
198
+ weights = F.softmax(pred_amps_noise.log(), dim=2)
199
+ pred_amps = pred_amps_noise.mul(weights).sum(dim=2)
200
+
201
+ return alphas, pred_amps
202
+
203
+ def dipole_model(self, z, obs_amps, masks, channels):
204
+ neighbors = self.b.padded_geom[self.b.model_channel_index[channels]]
205
+ centers = self.get_reference_points(
206
+ channels, obs_amps=obs_amps, neighborhoods=neighbors
207
+ )
208
+ local_geom = neighbors - centers.unsqueeze(1)
209
+
210
+ # displacements from probe
211
+ dx = z[:, 0, None] - local_geom[:, :, 0]
212
+ dz = z[:, 2, None] - local_geom[:, :, 1]
213
+ y = F.softplus(z[:, 1]).unsqueeze(1)
214
+ duv = torch.stack((dx, y.broadcast_to(dx.shape), dz), dim=2)
215
+
216
+ # displacment over distance cubed. (n_spikes, n_chans, 3)
217
+ X = duv * duv.square().sum(2, keepdim=True).pow(-1.5)
218
+ if self.alpha_closed_form:
219
+ # beta = torch.linalg.pinv(X.mT @ X) @ (X.mT @ obs_amps.unsqueeze(2))
220
+ # beta = torch.linalg.lstsq(X.mT @ X, X.mT @ obs_amps.unsqueeze(2)).solution
221
+ beta = torch.linalg.lstsq(X, obs_amps.unsqueeze(2)).solution
222
+ pred_amps = (X @ beta)[:, :, 0]
223
+ beta = beta[:, :, 0]
224
+ else:
225
+ beta = z[:, 3:]
226
+ pred_amps = (X @ beta.unsqueeze(2))[:, :, 0]
227
+
228
+ return beta, pred_amps
229
+
230
+ def decode(self, z, channels, obs_amps, masks):
231
+ if self.localization_model in ("pointsource", "monopole", "gaussian"):
232
+ alphas, pred_amps = self.point_source_model(z, obs_amps, masks, channels)
233
+ elif self.localization_model == "dipole":
234
+ alphas, pred_amps = self.dipole_model(z, obs_amps, masks, channels)
235
+ else:
236
+ assert False
237
+ return alphas, pred_amps
238
+
239
+ def forward(self, x, mask, obs_amps, channels):
240
+ x_mask = torch.cat((x, mask.unsqueeze(1)), dim=1)
241
+ assert self.encoder is not None
242
+ mu = self.encoder(x_mask)
243
+ var = None
244
+ if self.variational:
245
+ mu, var = mu.chunk(2, dim=-1)
246
+ var = F.softplus(var)
247
+ z = self.reparameterize(mu, var)
248
+ alphas, pred_amps = self.decode(z, channels, obs_amps, mask)
249
+ return pred_amps, mu, var
250
+
251
+ def loss_function(self, recon_x, x, mask, mu, var):
252
+ recon_x_masked = recon_x * mask
253
+ x_masked = x * mask
254
+ if self.scale_loss_by_mean:
255
+ # 1/(n_chans_retained*mean amplitude)
256
+ rescale = 1.0 / x_masked.sum(1, keepdim=True)
257
+ else:
258
+ rescale = 1.0 / mask.sum(1, keepdim=True)
259
+ x_masked *= rescale
260
+ recon_x_masked *= rescale
261
+ mse = F.mse_loss(recon_x_masked, x_masked, reduction="sum") / self.batch_size
262
+ kld = 0.0
263
+ if self.variational:
264
+ muterm = mu.pow(2) / self.prior_variance - 1
265
+ kld = torch.log(self.prior_variance / var).add(muterm).sum()
266
+ kld = kld.mul(0.5 / self.batch_size)
267
+ return mse, kld
268
+
269
+ def _fit(self, waveforms, channels, weights=None):
270
+ # apply channel reindexing before any fitting...
271
+ amps = None
272
+ wf_dev = waveforms.device
273
+ my_dev = self.b.padded_geom.device
274
+ if waveforms.ndim == 2:
275
+ assert self.amplitudes_only
276
+ waveforms = waveforms.unsqueeze(1)
277
+ waveforms = reindex(
278
+ channels,
279
+ waveforms,
280
+ self.relative_index.to(device=wf_dev),
281
+ pad_value=0.0,
282
+ )
283
+ amps = waveforms[:, 0]
284
+ elif self.amplitudes_only:
285
+ if self.amplitude_kind == "ptp":
286
+ waveforms = ptp(waveforms)
287
+ elif self.amplitude_kind == "peak":
288
+ waveforms = waveforms.abs().max(dim=1).values
289
+ waveforms = waveforms[:, None]
290
+ waveforms = reindex(
291
+ channels,
292
+ waveforms,
293
+ self.relative_index.to(device=wf_dev),
294
+ pad_value=0.0,
295
+ )
296
+ amps = waveforms[:, 0]
297
+ else:
298
+ waveforms = reindex(
299
+ channels,
300
+ waveforms,
301
+ self.relative_index.to(device=wf_dev),
302
+ pad_value=0.0,
303
+ )
304
+ if self.amplitude_kind == "ptp":
305
+ amps = ptp(waveforms)
306
+ elif self.amplitude_kind == "peak":
307
+ amps = waveforms.abs().max(dim=1).values
308
+ assert amps is not None
309
+
310
+ rg = np.random.default_rng(self.random_seed)
311
+
312
+ # make a validation set for early stopping
313
+ if self.val_split_p:
314
+ n_train = int(np.ceil(self.val_split_p * len(waveforms)))
315
+ istrain = np.zeros(len(waveforms), dtype=bool)
316
+ istrain[rg.choice(len(waveforms), size=n_train, replace=False)] = True
317
+ isval = np.logical_not(istrain)
318
+ val_waveforms = waveforms[isval]
319
+ val_amps = amps[isval]
320
+ val_channels = channels[isval]
321
+ waveforms = waveforms[istrain]
322
+ amps = amps[istrain]
323
+ channels = channels[istrain]
324
+ weights = weights[istrain] if weights is not None else None
325
+ else:
326
+ # early stopping will just be done on the train wfs
327
+ val_waveforms = waveforms
328
+ val_amps = amps
329
+ val_channels = channels
330
+
331
+ dataset = TensorDataset(waveforms, amps, channels)
332
+ if weights is None:
333
+ sampler = RandomSampler(dataset, generator=spawn_torch_rg(rg))
334
+ else:
335
+ assert len(weights) == len(dataset)
336
+ sampler = WeightedRandomSampler(
337
+ weights,
338
+ num_samples=len(dataset),
339
+ generator=spawn_torch_rg(rg),
340
+ )
341
+ sampler = BatchSampler(sampler, batch_size=self.batch_size, drop_last=True)
342
+ dataloader = DataLoader(dataset, sampler=sampler)
343
+ optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
344
+
345
+ val_dataset = TensorDataset(val_waveforms, val_amps, val_channels)
346
+ val_loader = DataLoader(val_dataset, batch_size=self.batch_size)
347
+
348
+ self.train()
349
+ mse_history = []
350
+ with trange(self.n_epochs, desc="Train localizer", unit="epoch") as pbar:
351
+ for epoch in pbar:
352
+ total_loss = 0
353
+ total_mse = 0
354
+ total_kld = 0
355
+ n_examples = 0
356
+ for waveform_batch, amps_batch, chans_batch in dataloader:
357
+ # for whatever reason, batch sampler adds an empty dim
358
+ waveform_batch = waveform_batch[0].to(device=my_dev)
359
+ amps_batch = amps_batch[0].to(device=my_dev)
360
+ chans_batch = chans_batch[0].to(device=my_dev)
361
+
362
+ optimizer.zero_grad()
363
+ channels_mask = self.b.model_channel_index[chans_batch] < self.nc
364
+ channels_mask = channels_mask.to(waveform_batch)
365
+ reconstructed_amps, mu, var = self.forward(
366
+ waveform_batch, channels_mask, amps_batch, chans_batch
367
+ )
368
+ mse, kld = self.loss_function(
369
+ reconstructed_amps, amps_batch, channels_mask, mu, var
370
+ )
371
+ loss = mse
372
+ if self.variational:
373
+ loss = loss + kld
374
+ loss.backward()
375
+ optimizer.step()
376
+ total_loss += loss.item()
377
+ total_mse += mse.item()
378
+ if self.variational:
379
+ total_kld += float(kld)
380
+
381
+ n_examples += chans_batch.numel()
382
+ if n_examples >= self.epoch_size:
383
+ break
384
+
385
+ valbatch = 0
386
+ val_loss = 0.0
387
+ with torch.no_grad():
388
+ self.eval()
389
+ for waveform_batch, amps_batch, chans_batch in val_loader:
390
+ waveform_batch = waveform_batch.to(device=my_dev)
391
+ amps_batch = amps_batch.to(device=my_dev)
392
+ chans_batch = chans_batch.to(device=my_dev)
393
+ ci_batch = self.b.model_channel_index[chans_batch]
394
+ channels_mask = ci_batch < len(self.b.geom)
395
+ channels_mask = channels_mask.to(waveform_batch)
396
+ reconstructed_amps, mu, var = self.forward(
397
+ waveform_batch, channels_mask, amps_batch, chans_batch
398
+ )
399
+ mse, kld = self.loss_function(
400
+ reconstructed_amps, amps_batch, channels_mask, mu, var
401
+ )
402
+ loss = mse
403
+ if self.variational:
404
+ loss = loss + kld
405
+ val_loss += loss.item()
406
+ valbatch += 1
407
+ self.train()
408
+
409
+ nbatch = max(1.0, np.ceil(n_examples / self.batch_size))
410
+ loss = total_loss / nbatch
411
+ mse = total_mse / nbatch
412
+ val_loss = val_loss / valbatch
413
+ mse_history.append(val_loss)
414
+ desc = f"[loss={loss:0.4f},val={val_loss:0.4f}"
415
+ if self.variational:
416
+ kld = total_kld / nbatch
417
+ desc += f",mse={mse:0.2f},kld={kld:0.2f}"
418
+ desc += "]"
419
+ pbar.set_description(f"Train localizer {desc}")
420
+
421
+ # check convergence
422
+ if epoch < self.min_epochs:
423
+ continue
424
+
425
+ # positive if cur is smaller than prev
426
+ adiff = min(mse_history[:-1]) - mse
427
+ rdiff = adiff / min(mse_history[:-1])
428
+ if rdiff < self.convergence_rtol or adiff < self.convergence_atol:
429
+ pbar.set_description(f"Localizer converged at epoch={epoch} {desc}")
430
+ break
431
+
432
+ def transform_unbatched(self, waveforms, channels, return_extra=False):
433
+ # handle getting amplitudes, reindexing channels, and amplitudes_only logic
434
+ obs_amps = None
435
+ if waveforms.ndim == 2:
436
+ assert self.amplitudes_only
437
+ waveforms = waveforms.unsqueeze(1)
438
+ elif self.amplitudes_only:
439
+ if self.amplitude_kind == "ptp":
440
+ obs_amps = ptp(waveforms)
441
+ elif self.amplitude_kind == "peak":
442
+ obs_amps = waveforms.abs().max(dim=1).values
443
+ waveforms = obs_amps[:, None]
444
+ else:
445
+ assert False
446
+
447
+ waveforms = reindex(channels, waveforms, self.relative_index, pad_value=0.0)
448
+ if self.amplitudes_only:
449
+ obs_amps = waveforms[:, 0]
450
+ elif return_extra or self.reference == "com":
451
+ if self.amplitude_kind == "ptp":
452
+ obs_amps = ptp(waveforms)
453
+ elif self.amplitude_kind == "peak":
454
+ obs_amps = waveforms.abs().max(dim=1).values
455
+ else:
456
+ # in this condition, we don't need the amp vecs
457
+ obs_amps = None
458
+
459
+ # nn inputs
460
+ mask = self.b.model_channel_index[channels] < self.b.geom.shape[0]
461
+ mask = mask.to(waveforms)
462
+ x_mask = torch.cat((waveforms, mask.unsqueeze(1)), dim=1)
463
+
464
+ # encode
465
+ # this is where we need to batch
466
+ assert self.encoder is not None
467
+ mu = self.encoder(x_mask)
468
+ var = None
469
+ if self.variational:
470
+ mu, var = mu.chunk(2, dim=-1)
471
+ x, y, z = mu[:, :3].T
472
+ y = F.softplus(y)
473
+ mx, mz = self.get_reference_points(channels, obs_amps=obs_amps).T
474
+ x = x + mx
475
+ z = z + mz
476
+ locs = torch.column_stack((x, y, z))
477
+
478
+ if return_extra:
479
+ alphas, pred_amps = self.decode(mu, channels, obs_amps, mask)
480
+ return dict(locs=locs, obs_amps=obs_amps, pred_amps=pred_amps, mx=mx, mz=mz)
481
+
482
+ return locs
483
+
484
+ def transform(self, waveforms, *, channels, show_progress=False, **unused):
485
+ n = len(waveforms)
486
+ if self.nc == 1:
487
+ return {self.name: waveforms.new_zeros((n, 3))}
488
+ with torch.no_grad():
489
+ if n > self.inference_batch_size:
490
+ locs = waveforms.new_empty((n, 3))
491
+ rg = trange if show_progress else range
492
+ my_device = self.b.padded_geom.device
493
+ device_in = locs.device
494
+ for bs in rg(0, n, self.inference_batch_size):
495
+ be = bs + self.inference_batch_size
496
+ batch = waveforms[bs:be].to(my_device)
497
+ batch_chans = channels[bs:be].to(my_device)
498
+ res = self.transform_unbatched(batch, batch_chans)
499
+ locs[bs:be] = cast(torch.Tensor, res).to(device_in)
500
+ del batch, batch_chans, res
501
+ else:
502
+ locs = self.transform_unbatched(waveforms, channels)
503
+ return {self.name: locs}
@@ -0,0 +1,197 @@
1
+ import torch
2
+ from dartsort.util.spiketorch import ptp
3
+
4
+ from .transform_base import BaseWaveformFeaturizer
5
+
6
+
7
+ class AmplitudeFeatures(BaseWaveformFeaturizer):
8
+ is_multi = True
9
+
10
+ def __init__(
11
+ self,
12
+ channel_index,
13
+ geom=None,
14
+ dtype=torch.float,
15
+ name_prefix="",
16
+ ptp_max_amplitude=True,
17
+ peak_amplitude_vectors=True,
18
+ ptp_amplitude_vectors=True,
19
+ log_peak_to_trough=True,
20
+ ):
21
+ self.shape = []
22
+ names = []
23
+ self.dtype = []
24
+
25
+ self.ptp_max_amplitude = ptp_max_amplitude
26
+ if ptp_max_amplitude:
27
+ name = "ptp_amplitudes"
28
+ if name_prefix:
29
+ name = f"{name_prefix}_{name}"
30
+ self.ptp_max_amplitude_name = name
31
+ self.shape.append(())
32
+ names.append(name)
33
+ self.dtype.append(dtype)
34
+
35
+ self.peak_amplitude_vectors = peak_amplitude_vectors
36
+ if peak_amplitude_vectors:
37
+ name = "peak_amplitude_vectors"
38
+ if name_prefix:
39
+ name = f"{name_prefix}_{name}"
40
+ self.peak_amplitude_vectors_name = name
41
+ self.shape.append((channel_index.shape[1],))
42
+ names.append(name)
43
+ self.dtype.append(dtype)
44
+
45
+ self.ptp_amplitude_vectors = ptp_amplitude_vectors
46
+ if ptp_amplitude_vectors:
47
+ name = "ptp_amplitude_vectors"
48
+ if name_prefix:
49
+ name = f"{name_prefix}_{name}"
50
+ self.ptp_amplitude_vectors_name = name
51
+ self.shape.append((channel_index.shape[1],))
52
+ names.append(name)
53
+ self.dtype.append(dtype)
54
+
55
+ self.log_peak_to_trough = log_peak_to_trough
56
+ if log_peak_to_trough:
57
+ name = "logpeaktotrough"
58
+ if name_prefix:
59
+ name = f"{name_prefix}_{name}"
60
+ self.log_peak_to_trough_name = name
61
+ self.shape.append(())
62
+ names.append(name)
63
+ self.dtype.append(dtype)
64
+
65
+ self.compute_minmax_vectors = (
66
+ ptp_max_amplitude or ptp_amplitude_vectors or log_peak_to_trough
67
+ )
68
+ self.compute_maxchan = ptp_max_amplitude or log_peak_to_trough
69
+
70
+ super().__init__(name=names)
71
+
72
+ def transform(self, waveforms, **unused):
73
+ features = {}
74
+ if self.peak_amplitude_vectors:
75
+ features[self.peak_amplitude_vectors_name] = waveforms.abs().amax(dim=1)
76
+
77
+ max_vectors = min_vectors = ptp_vectors = None
78
+ maxs = mins = ptps = None
79
+ if self.compute_minmax_vectors:
80
+ max_vectors = waveforms.amax(dim=1)
81
+ min_vectors = waveforms.amin(dim=1)
82
+ ptp_vectors = max_vectors - min_vectors
83
+ if self.compute_maxchan:
84
+ assert ptp_vectors is not None
85
+ maxchans = torch.argmax(ptp_vectors.nan_to_num(), dim=1, keepdim=True)
86
+ if self.log_peak_to_trough:
87
+ assert max_vectors is not None
88
+ assert min_vectors is not None
89
+ maxs = torch.take_along_dim(max_vectors, maxchans, dim=1)[:, 0]
90
+ mins = torch.take_along_dim(min_vectors, maxchans, dim=1)[:, 0]
91
+ if self.ptp_max_amplitude:
92
+ ptps = maxs - mins
93
+ elif self.ptp_max_amplitude:
94
+ ptps = torch.take_along_dim(ptp_vectors, maxchans, dim=1)[:, 0]
95
+
96
+ if self.ptp_amplitude_vectors:
97
+ features[self.ptp_amplitude_vectors_name] = ptp_vectors
98
+ if self.ptp_max_amplitude:
99
+ assert ptps is not None
100
+ features[self.ptp_max_amplitude_name] = ptps
101
+ if self.log_peak_to_trough:
102
+ assert maxs is not None
103
+ assert mins is not None
104
+ logptt = torch.log(maxs) - torch.log(mins.neg())
105
+ features[self.log_peak_to_trough_name] = logptt
106
+
107
+ return features
108
+
109
+
110
+ class AmplitudeVector(BaseWaveformFeaturizer):
111
+ default_name = "amplitude_vectors"
112
+
113
+ def __init__(
114
+ self,
115
+ channel_index,
116
+ geom=None,
117
+ kind="peak",
118
+ dtype=torch.float,
119
+ name=None,
120
+ name_prefix="",
121
+ ):
122
+ assert kind in ("peak", "ptp")
123
+ if name is None:
124
+ name = f"{kind}_{self.default_name}"
125
+ if name_prefix:
126
+ name = f"{name_prefix}_{name}"
127
+ super().__init__(name=name, name_prefix=name_prefix)
128
+ self.kind = kind
129
+ self.shape = (channel_index.shape[1],)
130
+ self.dtype = dtype
131
+
132
+ def transform(self, waveforms, **unused):
133
+ if self.kind == "peak":
134
+ return {self.name: waveforms.abs().amax(dim=1)}
135
+ elif self.kind == "ptp":
136
+ return {self.name: ptp(waveforms, dim=1)}
137
+
138
+
139
+ class MaxAmplitude(BaseWaveformFeaturizer):
140
+ default_name = "amplitudes"
141
+ shape = ()
142
+
143
+ def __init__(
144
+ self,
145
+ channel_index=None,
146
+ geom=None,
147
+ kind="ptp",
148
+ dtype=torch.float,
149
+ name=None,
150
+ name_prefix="",
151
+ ):
152
+ assert kind in ("peak", "ptp")
153
+ if name is None:
154
+ name = f"{kind}_{self.default_name}"
155
+ if name_prefix:
156
+ name = f"{name_prefix}_{name}"
157
+ super().__init__(name=name, name_prefix=name_prefix)
158
+ self.kind = kind
159
+ self.dtype = dtype
160
+
161
+ def transform(self, waveforms, **unused):
162
+ if self.kind == "peak":
163
+ return {self.name: torch.nan_to_num(waveforms.abs()).amax(dim=(1, 2))}
164
+ elif self.kind == "ptp":
165
+ return {self.name: torch.nan_to_num(ptp(waveforms, dim=1)).amax(dim=1)}
166
+
167
+
168
+ class Voltage(BaseWaveformFeaturizer):
169
+ default_name = "voltages"
170
+ shape = ()
171
+
172
+ def __init__(
173
+ self,
174
+ channel_index,
175
+ geom=None,
176
+ dtype=torch.float,
177
+ name=None,
178
+ name_prefix="",
179
+ trough_offset_samples: int = 42,
180
+ ):
181
+ if name is None:
182
+ name = self.default_name
183
+ if name_prefix:
184
+ name = f"{name_prefix}_{name}"
185
+ super().__init__(name=name, name_prefix=name_prefix)
186
+ self.dtype = dtype
187
+ self.trough_offset_samples = trough_offset_samples
188
+ chans_arange = torch.arange(len(channel_index)).to(channel_index)
189
+ cix, rel_inds = (channel_index == chans_arange[:, None]).nonzero(as_tuple=True)
190
+ assert torch.equal(chans_arange, cix)
191
+ self.register_buffer("main_channel_relative_inds", rel_inds)
192
+
193
+ def transform(self, waveforms, *, channels, **unused):
194
+ waveforms_at_time = waveforms[:, self.trough_offset_samples]
195
+ rel_inds = self.b.main_channel_relative_inds[channels]
196
+ v = waveforms_at_time.take_along_dim(rel_inds[:, None], dim=1)[:, 0]
197
+ return {self.name: v}