DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,486 @@
1
+ import math
2
+
3
+ import h5py
4
+ import torch
5
+ import torch.nn.functional as F
6
+ from torch.utils.data import DataLoader, TensorDataset, random_split
7
+ from tqdm.auto import trange
8
+
9
+
10
+ from ._multichan_denoiser_kit import (
11
+ BaseMultichannelDenoiser,
12
+ RefreshableDataset,
13
+ RefreshableDataLoader,
14
+ AsyncBatchDataset,
15
+ )
16
+ from ..util import waveform_util
17
+
18
+
19
+ class SupervisedDenoiser(BaseMultichannelDenoiser):
20
+ default_name = "superviseddenoiser"
21
+
22
+ def initialize_spike_length_dependent_params(self):
23
+ self.initialize_shapes()
24
+ self.exy = self.get_mlp(res_type=self.res_type)
25
+ self.to(self.device)
26
+
27
+ def forward_unbatched(self, waveforms, channels, to_orig_channels=True):
28
+ """Called only at inference time."""
29
+ waveforms, masks = self.to_nn_channels(waveforms, channels)
30
+ net_input = waveforms, masks.unsqueeze(1)
31
+ pred = self.exy(net_input)
32
+ if to_orig_channels:
33
+ pred = self.to_orig_channels(pred, channels)
34
+ return pred
35
+
36
+ def fit(self, recording, waveforms, *, gt_waveforms, channels, **unused):
37
+ super().fit(recording, waveforms, channels=channels)
38
+ train_loader, val_loader = self._waveforms_to_loaders(
39
+ waveforms, gt_waveforms, channels
40
+ )
41
+ with torch.enable_grad():
42
+ res = self.fit_with_loaders(train_loader, val_loader)
43
+ self._needs_fit = False
44
+ return res
45
+
46
+ def loss(self, mask, gt_waveforms, pred):
47
+ mask = mask.unsqueeze(1)
48
+ loss_dict = dict(mse=F.mse_loss(mask * gt_waveforms, mask * pred))
49
+ return loss_dict
50
+
51
+ def fit_with_loaders(
52
+ self,
53
+ train_loader: RefreshableDataLoader,
54
+ val_loader: DataLoader | None = None,
55
+ ):
56
+ _wf, *_ = next(iter(train_loader))
57
+
58
+ optimizer = self.get_optimizer()
59
+ scheduler = self.get_scheduler(optimizer)
60
+
61
+ train_losses_per_epoch = []
62
+ val_losses_per_epoch = []
63
+ last_val_loss = None
64
+
65
+ with trange(self.n_epochs, desc="Epochs", unit="epoch") as pbar:
66
+ for epoch in pbar:
67
+ self.train()
68
+ train_loss_sum = 0.0
69
+ train_loader.refresh()
70
+ for waveform_batch, gt_waveform_batch, channels_batch in train_loader:
71
+ optimizer.zero_grad()
72
+
73
+ waveform_batch = waveform_batch.to(self.device)
74
+ gt_waveform_batch = gt_waveform_batch.to(self.device)
75
+ channels_batch = channels_batch.to(self.device)
76
+
77
+ gt_waveform_batch, mask = self.to_nn_channels(
78
+ gt_waveform_batch, channels_batch
79
+ )
80
+ pred = self.forward_unbatched(
81
+ waveform_batch, channels_batch, to_orig_channels=False
82
+ )
83
+
84
+ loss_dict = self.loss(mask, gt_waveform_batch, pred)
85
+ loss = sum(loss_dict.values())
86
+ loss.backward()
87
+ optimizer.step()
88
+
89
+ train_loss_sum += loss.item()
90
+
91
+ avg_train_loss = train_loss_sum / len(train_loader)
92
+ train_losses_per_epoch.append(avg_train_loss)
93
+ train_loader.cleanup()
94
+
95
+ avg_val_loss = None
96
+ if val_loader is not None:
97
+ self.eval()
98
+ val_loss_sum = 0.0
99
+ with torch.no_grad():
100
+ for (
101
+ waveform_batch,
102
+ gt_waveform_batch,
103
+ channels_batch,
104
+ ) in val_loader:
105
+ waveform_batch = waveform_batch.to(self.device)
106
+ gt_waveform_batch = gt_waveform_batch.to(self.device)
107
+ channels_batch = channels_batch.to(self.device)
108
+
109
+ gt_waveform_batch, mask = self.to_nn_channels(
110
+ gt_waveform_batch, channels_batch
111
+ )
112
+ pred = self.forward_unbatched(
113
+ waveform_batch, channels_batch, to_orig_channels=False
114
+ )
115
+
116
+ loss_dict = self.loss(mask, gt_waveform_batch, pred)
117
+ loss = sum(loss_dict.values())
118
+ val_loss_sum += loss.item()
119
+
120
+ avg_val_loss = val_loss_sum / len(val_loader)
121
+ val_losses_per_epoch.append(avg_val_loss)
122
+
123
+ if (
124
+ self.earlystop_eps is not None
125
+ and last_val_loss is not None
126
+ and avg_val_loss - last_val_loss > self.earlystop_eps
127
+ ):
128
+ if epoch >= self.min_epochs:
129
+ print(f"Early stopping after {epoch} epochs.")
130
+ break
131
+ last_val_loss = avg_val_loss
132
+
133
+ if self.step_callback is not None:
134
+ self.step_callback(self, epoch, avg_val_loss)
135
+
136
+ loss_str = f"Train loss: {avg_train_loss:3g}"
137
+ if val_loader is not None:
138
+ loss_str += f" | Val loss: {avg_val_loss:3g}"
139
+ pbar.set_description(f"Epochs [{loss_str}]")
140
+
141
+ self.step_scheduler(scheduler, avg_train_loss, avg_val_loss)
142
+
143
+ return dict(
144
+ train_losses=train_losses_per_epoch,
145
+ val_losses=val_losses_per_epoch,
146
+ )
147
+
148
+ def _waveforms_to_loaders(self, waveforms, gt_waveforms, channels):
149
+ num_samples = len(waveforms)
150
+ train_size = math.ceil(num_samples * (1 - self.val_split_p))
151
+ val_size = num_samples - train_size
152
+ full_dataset = TensorDataset(waveforms, gt_waveforms, channels)
153
+ if val_size:
154
+ train_dataset, val_dataset = random_split(
155
+ full_dataset, [train_size, val_size]
156
+ )
157
+ val_loader = DataLoader(
158
+ val_dataset, batch_size=self.batch_size, shuffle=False
159
+ )
160
+ else:
161
+ train_dataset = full_dataset
162
+ val_loader = None
163
+
164
+ train_loader = RefreshableDataLoader(
165
+ train_dataset, batch_size=self.batch_size, shuffle=True
166
+ )
167
+ return train_loader, val_loader
168
+
169
+
170
+ class PairedHDF5WaveformsDataset(AsyncBatchDataset):
171
+ def __init__(
172
+ self,
173
+ hdf5_path,
174
+ channel_index,
175
+ waveforms_dataset_name="waveforms",
176
+ gt_waveforms_dataset_name="gt_waveforms",
177
+ channels_dataset_name="channels",
178
+ gt_waveform_index_dataset_name=None,
179
+ geom=None,
180
+ trough_offset_samples=42,
181
+ spike_length_samples=121,
182
+ time_jitter_samples=3,
183
+ spatial_jitter_um=35.0,
184
+ generator=None,
185
+ chunk_size=2048,
186
+ queue_chunks=8,
187
+ ):
188
+ """This is meant to be used with the AOTIndicesRefreshableDataLoader
189
+
190
+ Class balancing etc can be achieved with its weights argument. They would
191
+ 'hit' the waveforms here (not gt_waveforms, if those are indexed via
192
+ labels in gt_waveform_index_dataset_name).
193
+ """
194
+ self.hdf5_path = hdf5_path
195
+ self._h5 = h5py.File(hdf5_path, "r", locking=False)
196
+ self._waveforms_dataset: h5py.Dataset = self._h5[waveforms_dataset_name]
197
+ self._gt_waveforms_dataset: h5py.Dataset = self._h5[gt_waveforms_dataset_name]
198
+ self._channels = torch.from_numpy(self._h5[channels_dataset_name][:])
199
+ if gt_waveform_index_dataset_name:
200
+ self._gt_indices = self._h5[gt_waveform_index_dataset_name][:]
201
+ else:
202
+ self._gt_indices = None
203
+
204
+ super().__init__(
205
+ n_examples=len(self._channels),
206
+ spike_length_samples=spike_length_samples,
207
+ generator=generator,
208
+ chunk_size=chunk_size,
209
+ queue_chunks=queue_chunks,
210
+ )
211
+ self.trough_offset_samples = trough_offset_samples
212
+ self.time_jitter_samples = time_jitter_samples
213
+ self.spatial_jitter_um = spatial_jitter_um
214
+
215
+ # spatial information
216
+ self.channel_index = channel_index
217
+ self.n_channels = len(channel_index)
218
+ if spatial_jitter_um:
219
+ if geom is None:
220
+ geom = self._h5["geom"][:]
221
+ self.spatial_jitter_index = waveform_util.make_channel_index(
222
+ geom, spatial_jitter_um
223
+ )
224
+ else:
225
+ self.spatial_jitter_index = None
226
+
227
+ # temporal info
228
+ min_length = spike_length_samples + 2 * time_jitter_samples
229
+ self._wf_length = self._waveforms_dataset.shape[1]
230
+ assert self._wf_length >= min_length
231
+ offset = (self._wf_length - spike_length_samples) // 2
232
+ assert self._wf_length == 2 * offset + spike_length_samples
233
+ self._wf_offset = offset
234
+ self._tix_rel = torch.arange(spike_length_samples)
235
+
236
+ def __del__(self):
237
+ del self._gt_waveforms_dataset
238
+ del self._waveforms_dataset
239
+ self._h5.close()
240
+
241
+ def load_batch(self, index):
242
+ n = index.numel()
243
+ waveforms = self._waveforms_dataset[index.numpy()]
244
+ gt_index = index.numpy()
245
+ if self._gt_indices is not None:
246
+ gt_index = self._gt_indices[gt_index]
247
+ gt_waveforms = self._gt_waveforms_dataset[gt_index]
248
+ channels = self._channels[index.numpy()]
249
+
250
+ # jitter the channels
251
+ if self.spatial_jitter_index:
252
+ choices = self.spatial_jitter_index[channels]
253
+ n_valid = (choices < self.n_channels).sum(1)
254
+ # wish they had tensor bounds.
255
+ which = torch.randint(2**63 - 1, size=n, generator=self.generator) % n_valid
256
+ channels = choices[torch.arange(n), which]
257
+
258
+ # extract the resulting channel subsets
259
+ chan_inds = self.channel_index[channels]
260
+ waveforms = F.pad(waveforms, (0, 1), value=torch.nan)
261
+ waveforms = waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
262
+ gt_waveforms = F.pad(gt_waveforms, (0, 1), value=torch.nan)
263
+ gt_waveforms = gt_waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
264
+
265
+ # jitter time
266
+ time_ixs = torch.full((n,), self._wf_offset)
267
+ if self.time_jitter_samples:
268
+ time_ixs += torch.randint(
269
+ -self.time_jitter_samples,
270
+ self.time_jitter_samples + 1,
271
+ generator=self.generator,
272
+ size=n,
273
+ )
274
+ time_ixs = time_ixs[:, None] + self._tix_rel
275
+ waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
276
+ gt_waveforms = gt_waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
277
+
278
+ return waveforms, gt_waveforms, channels
279
+
280
+
281
+ class NoisedTemplatesDataset(AsyncBatchDataset):
282
+ def __init__(
283
+ self,
284
+ channel_index,
285
+ templates,
286
+ noise,
287
+ geom=None,
288
+ collision_prob=0.5,
289
+ max_collisions=18,
290
+ collision_sample_weights=None,
291
+ trough_offset_samples=42,
292
+ spike_length_samples=121,
293
+ time_jitter_samples=3,
294
+ spatial_jitter_um=35.0,
295
+ amplitude_jitter_std=0.1,
296
+ amplitude_jitter_max=1.5,
297
+ temporal_upsampling_factor=8,
298
+ generator=None,
299
+ chunk_size=2048,
300
+ queue_chunks=8,
301
+ dtype=torch.float,
302
+ ):
303
+ """This is meant to be used with the AOTIndicesRefreshableDataLoader
304
+
305
+ Class balancing etc can be achieved with its weights argument. This
306
+ would assign a weight to each template.
307
+
308
+ NOTE: collisions are on the full channel set. So, many collisions will
309
+ do nothing! Figure that the probe is 4000um and that a "significant"
310
+ collision occurs when units are spaced by <100um. If the unit positions
311
+ are random, a collision has ~200/4000=0.05 chance of overlapping significantly.
312
+ Twice that for a minor overlap?
313
+
314
+ Collisions are added binomially (# ~ Bin(max,p)), up to max_collisions,
315
+ so the mean # is like max_collisions*collision_prob.
316
+ So, the real mean is something like 0.05*max_collisions*p but censored.
317
+ For p=0.5 max=4, that would be like .5*4/20=0.1ish? But then the completely
318
+ random shifting reduces the probability by another factor of 3!
319
+
320
+ So the defaults are set to 18 max collisions for this reason:
321
+ 18 max * 0.5 p * 1/3 time * 1/20 space = 0.15.
322
+
323
+
324
+ `noise` should probably be a noise_util.FactorizedNoise object, or
325
+ one of the other obects there which have simulate() methods.
326
+ """
327
+ super().__init__(
328
+ n_examples=len(templates),
329
+ spike_length_samples=spike_length_samples,
330
+ generator=generator,
331
+ chunk_size=chunk_size,
332
+ queue_chunks=queue_chunks,
333
+ )
334
+ self.trough_offset_samples = trough_offset_samples
335
+ self.time_jitter_samples = time_jitter_samples
336
+ self.spatial_jitter_um = spatial_jitter_um
337
+ self.amplitude_jitter_std = amplitude_jitter_std
338
+ self.amplitude_jitter_max = amplitude_jitter_max
339
+ self.amplitude_jitter_min = 1.0 / amplitude_jitter_max
340
+ self.temporal_upsampling_factor = temporal_upsampling_factor
341
+
342
+ self.noise = noise
343
+
344
+ # collision control
345
+ self.collision_prob = collision_prob
346
+ self.max_collisions = max_collisions
347
+ self.collision_sample_weights = collision_sample_weights
348
+
349
+ # upsample and store templates as n_examples, time_up, time, chans
350
+ if torch.is_tensor(templates):
351
+ templates = templates.numpy(force=True)
352
+ n, t, c = templates.shape
353
+ templates = templates.transpose(0, 2, 1).reshape(n * c, t)
354
+ templates_up = waveform_util.upsample_singlechan(
355
+ templates, temporal_jitter=temporal_upsampling_factor
356
+ )
357
+ templates_up = templates_up.reshape(n, c, temporal_upsampling_factor, t)
358
+ templates_up = templates_up.transpose(0, 2, 3, 1)
359
+ self.templates_up = torch.asarray(templates_up, dtype=dtype).contiguous()
360
+ self.channels_up = self.templates_up.abs().amax(dim=2).argmax(dim=2)
361
+
362
+ # spatial information
363
+ self.channel_index = channel_index
364
+ self.n_channels = len(channel_index)
365
+ if spatial_jitter_um:
366
+ assert geom is not None
367
+ self.spatial_jitter_index = waveform_util.make_channel_index(
368
+ geom, spatial_jitter_um
369
+ )
370
+ else:
371
+ self.spatial_jitter_index = None
372
+
373
+ # temporal info
374
+ min_length = spike_length_samples + 2 * time_jitter_samples
375
+ self._wf_length = t
376
+ assert self._wf_length >= min_length, "Waveforms need to be padded."
377
+ offset = (self._wf_length - spike_length_samples) // 2
378
+ assert self._wf_length == 2 * offset + spike_length_samples
379
+ self._wf_offset = offset
380
+ self._tix_rel = torch.arange(spike_length_samples)
381
+
382
+ def load_batch(self, index):
383
+ n = index.numel()
384
+
385
+ # get gt waveforms
386
+ up_ix = torch.randint(
387
+ 0, self.temporal_upsampling_factor, size=n, generator=self.generator
388
+ )
389
+ gt_waveforms = self.templates_up[index, up_ix]
390
+ channels = self.channels_up[index, up_ix]
391
+
392
+ # jitter amplitude
393
+ if self.amplitude_jitter_std:
394
+ scaling = torch.normal(
395
+ 1.0, self.amplitude_jitter_std, size=n, generator=self.generator
396
+ )
397
+ scaling.clamp_(self.amplitude_jitter_min, self.amplitude_jitter_max)
398
+ gt_waveforms = gt_waveforms * scaling[..., None, None]
399
+
400
+ # get noise waveforms
401
+ noise = self.noise.simulate(size=n, generator=self.generator)
402
+
403
+ # add in the collisions
404
+ p = torch.full((self.max_collisions, n), self.collision_prob)
405
+ mask = torch.bernoulli(p)
406
+ collisions = self.get_batch_of_collisions(mask.sum())
407
+ s = 0
408
+ for m in mask:
409
+ (m,) = m.nonzero(as_tuple=True)
410
+ noise[m] += collisions[s : s + m.numel()]
411
+ s += m.numel()
412
+
413
+ # noisy waveforms...
414
+ waveforms = gt_waveforms + noise
415
+
416
+ # jitter the channels
417
+ if self.spatial_jitter_index:
418
+ choices = self.spatial_jitter_index[channels]
419
+ n_valid = (choices < self.n_channels).sum(1)
420
+ # wish they had tensor bounds.
421
+ which = torch.randint(2**63 - 1, size=n, generator=self.generator) % n_valid
422
+ channels = choices[torch.arange(n), which]
423
+
424
+ # extract the resulting channel subsets
425
+ chan_inds = self.channel_index[channels]
426
+ waveforms = F.pad(waveforms, (0, 1), value=torch.nan)
427
+ waveforms = waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
428
+ gt_waveforms = F.pad(gt_waveforms, (0, 1), value=torch.nan)
429
+ gt_waveforms = gt_waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
430
+
431
+ # jitter time
432
+ time_ixs = torch.full((n,), self._wf_offset)
433
+ if self.time_jitter_samples:
434
+ time_ixs += torch.randint(
435
+ -self.time_jitter_samples,
436
+ self.time_jitter_samples + 1,
437
+ generator=self.generator,
438
+ size=n,
439
+ )
440
+ time_ixs = time_ixs[:, None] + self._tix_rel
441
+ waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
442
+ gt_waveforms = gt_waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
443
+
444
+ return waveforms, gt_waveforms, channels
445
+
446
+ def get_batch_of_collisions(self, n):
447
+ # draw weighted samples
448
+ if self.collision_sample_weights is None:
449
+ index = torch.randint(
450
+ high=self.n_examples,
451
+ size=n,
452
+ dtype=torch.int64,
453
+ generator=self.generator,
454
+ )
455
+ else:
456
+ index = torch.multinomial(
457
+ self.collision_sample_weights,
458
+ n,
459
+ replacement=True,
460
+ generator=self.generator,
461
+ )
462
+ up_ix = torch.randint(
463
+ 0, self.temporal_upsampling_factor, size=n, generator=self.generator
464
+ )
465
+ waveforms = self.templates_up[index, up_ix]
466
+
467
+ # shift to a completely random offset, in the valid range
468
+ # TODO should this be some kind of taper rather than 0pad?
469
+ full_length = 3 * self.spike_length_samples
470
+ pad = (full_length - waveforms.shape[1]) // 2
471
+ waveforms = F.pad(waveforms, (0, 0, pad, pad))
472
+ time_ixs = torch.randint(
473
+ 0, 2 * self.spike_length_samples, generator=self.generator, size=n
474
+ )
475
+ time_ixs = time_ixs[:, None] + self._tix_rel
476
+ waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
477
+
478
+ # jitter amplitude
479
+ if self.amplitude_jitter_std:
480
+ scaling = torch.normal(
481
+ 1.0, self.amplitude_jitter_std, size=n, generator=self.generator
482
+ )
483
+ scaling.clamp_(self.amplitude_jitter_min, self.amplitude_jitter_max)
484
+ waveforms.mul_(scaling[..., None, None])
485
+
486
+ return waveforms