DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
import h5py
|
|
4
|
+
import torch
|
|
5
|
+
import torch.nn.functional as F
|
|
6
|
+
from torch.utils.data import DataLoader, TensorDataset, random_split
|
|
7
|
+
from tqdm.auto import trange
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
from ._multichan_denoiser_kit import (
|
|
11
|
+
BaseMultichannelDenoiser,
|
|
12
|
+
RefreshableDataset,
|
|
13
|
+
RefreshableDataLoader,
|
|
14
|
+
AsyncBatchDataset,
|
|
15
|
+
)
|
|
16
|
+
from ..util import waveform_util
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SupervisedDenoiser(BaseMultichannelDenoiser):
|
|
20
|
+
default_name = "superviseddenoiser"
|
|
21
|
+
|
|
22
|
+
def initialize_spike_length_dependent_params(self):
|
|
23
|
+
self.initialize_shapes()
|
|
24
|
+
self.exy = self.get_mlp(res_type=self.res_type)
|
|
25
|
+
self.to(self.device)
|
|
26
|
+
|
|
27
|
+
def forward_unbatched(self, waveforms, channels, to_orig_channels=True):
|
|
28
|
+
"""Called only at inference time."""
|
|
29
|
+
waveforms, masks = self.to_nn_channels(waveforms, channels)
|
|
30
|
+
net_input = waveforms, masks.unsqueeze(1)
|
|
31
|
+
pred = self.exy(net_input)
|
|
32
|
+
if to_orig_channels:
|
|
33
|
+
pred = self.to_orig_channels(pred, channels)
|
|
34
|
+
return pred
|
|
35
|
+
|
|
36
|
+
def fit(self, recording, waveforms, *, gt_waveforms, channels, **unused):
|
|
37
|
+
super().fit(recording, waveforms, channels=channels)
|
|
38
|
+
train_loader, val_loader = self._waveforms_to_loaders(
|
|
39
|
+
waveforms, gt_waveforms, channels
|
|
40
|
+
)
|
|
41
|
+
with torch.enable_grad():
|
|
42
|
+
res = self.fit_with_loaders(train_loader, val_loader)
|
|
43
|
+
self._needs_fit = False
|
|
44
|
+
return res
|
|
45
|
+
|
|
46
|
+
def loss(self, mask, gt_waveforms, pred):
|
|
47
|
+
mask = mask.unsqueeze(1)
|
|
48
|
+
loss_dict = dict(mse=F.mse_loss(mask * gt_waveforms, mask * pred))
|
|
49
|
+
return loss_dict
|
|
50
|
+
|
|
51
|
+
def fit_with_loaders(
|
|
52
|
+
self,
|
|
53
|
+
train_loader: RefreshableDataLoader,
|
|
54
|
+
val_loader: DataLoader | None = None,
|
|
55
|
+
):
|
|
56
|
+
_wf, *_ = next(iter(train_loader))
|
|
57
|
+
|
|
58
|
+
optimizer = self.get_optimizer()
|
|
59
|
+
scheduler = self.get_scheduler(optimizer)
|
|
60
|
+
|
|
61
|
+
train_losses_per_epoch = []
|
|
62
|
+
val_losses_per_epoch = []
|
|
63
|
+
last_val_loss = None
|
|
64
|
+
|
|
65
|
+
with trange(self.n_epochs, desc="Epochs", unit="epoch") as pbar:
|
|
66
|
+
for epoch in pbar:
|
|
67
|
+
self.train()
|
|
68
|
+
train_loss_sum = 0.0
|
|
69
|
+
train_loader.refresh()
|
|
70
|
+
for waveform_batch, gt_waveform_batch, channels_batch in train_loader:
|
|
71
|
+
optimizer.zero_grad()
|
|
72
|
+
|
|
73
|
+
waveform_batch = waveform_batch.to(self.device)
|
|
74
|
+
gt_waveform_batch = gt_waveform_batch.to(self.device)
|
|
75
|
+
channels_batch = channels_batch.to(self.device)
|
|
76
|
+
|
|
77
|
+
gt_waveform_batch, mask = self.to_nn_channels(
|
|
78
|
+
gt_waveform_batch, channels_batch
|
|
79
|
+
)
|
|
80
|
+
pred = self.forward_unbatched(
|
|
81
|
+
waveform_batch, channels_batch, to_orig_channels=False
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
loss_dict = self.loss(mask, gt_waveform_batch, pred)
|
|
85
|
+
loss = sum(loss_dict.values())
|
|
86
|
+
loss.backward()
|
|
87
|
+
optimizer.step()
|
|
88
|
+
|
|
89
|
+
train_loss_sum += loss.item()
|
|
90
|
+
|
|
91
|
+
avg_train_loss = train_loss_sum / len(train_loader)
|
|
92
|
+
train_losses_per_epoch.append(avg_train_loss)
|
|
93
|
+
train_loader.cleanup()
|
|
94
|
+
|
|
95
|
+
avg_val_loss = None
|
|
96
|
+
if val_loader is not None:
|
|
97
|
+
self.eval()
|
|
98
|
+
val_loss_sum = 0.0
|
|
99
|
+
with torch.no_grad():
|
|
100
|
+
for (
|
|
101
|
+
waveform_batch,
|
|
102
|
+
gt_waveform_batch,
|
|
103
|
+
channels_batch,
|
|
104
|
+
) in val_loader:
|
|
105
|
+
waveform_batch = waveform_batch.to(self.device)
|
|
106
|
+
gt_waveform_batch = gt_waveform_batch.to(self.device)
|
|
107
|
+
channels_batch = channels_batch.to(self.device)
|
|
108
|
+
|
|
109
|
+
gt_waveform_batch, mask = self.to_nn_channels(
|
|
110
|
+
gt_waveform_batch, channels_batch
|
|
111
|
+
)
|
|
112
|
+
pred = self.forward_unbatched(
|
|
113
|
+
waveform_batch, channels_batch, to_orig_channels=False
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
loss_dict = self.loss(mask, gt_waveform_batch, pred)
|
|
117
|
+
loss = sum(loss_dict.values())
|
|
118
|
+
val_loss_sum += loss.item()
|
|
119
|
+
|
|
120
|
+
avg_val_loss = val_loss_sum / len(val_loader)
|
|
121
|
+
val_losses_per_epoch.append(avg_val_loss)
|
|
122
|
+
|
|
123
|
+
if (
|
|
124
|
+
self.earlystop_eps is not None
|
|
125
|
+
and last_val_loss is not None
|
|
126
|
+
and avg_val_loss - last_val_loss > self.earlystop_eps
|
|
127
|
+
):
|
|
128
|
+
if epoch >= self.min_epochs:
|
|
129
|
+
print(f"Early stopping after {epoch} epochs.")
|
|
130
|
+
break
|
|
131
|
+
last_val_loss = avg_val_loss
|
|
132
|
+
|
|
133
|
+
if self.step_callback is not None:
|
|
134
|
+
self.step_callback(self, epoch, avg_val_loss)
|
|
135
|
+
|
|
136
|
+
loss_str = f"Train loss: {avg_train_loss:3g}"
|
|
137
|
+
if val_loader is not None:
|
|
138
|
+
loss_str += f" | Val loss: {avg_val_loss:3g}"
|
|
139
|
+
pbar.set_description(f"Epochs [{loss_str}]")
|
|
140
|
+
|
|
141
|
+
self.step_scheduler(scheduler, avg_train_loss, avg_val_loss)
|
|
142
|
+
|
|
143
|
+
return dict(
|
|
144
|
+
train_losses=train_losses_per_epoch,
|
|
145
|
+
val_losses=val_losses_per_epoch,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
def _waveforms_to_loaders(self, waveforms, gt_waveforms, channels):
|
|
149
|
+
num_samples = len(waveforms)
|
|
150
|
+
train_size = math.ceil(num_samples * (1 - self.val_split_p))
|
|
151
|
+
val_size = num_samples - train_size
|
|
152
|
+
full_dataset = TensorDataset(waveforms, gt_waveforms, channels)
|
|
153
|
+
if val_size:
|
|
154
|
+
train_dataset, val_dataset = random_split(
|
|
155
|
+
full_dataset, [train_size, val_size]
|
|
156
|
+
)
|
|
157
|
+
val_loader = DataLoader(
|
|
158
|
+
val_dataset, batch_size=self.batch_size, shuffle=False
|
|
159
|
+
)
|
|
160
|
+
else:
|
|
161
|
+
train_dataset = full_dataset
|
|
162
|
+
val_loader = None
|
|
163
|
+
|
|
164
|
+
train_loader = RefreshableDataLoader(
|
|
165
|
+
train_dataset, batch_size=self.batch_size, shuffle=True
|
|
166
|
+
)
|
|
167
|
+
return train_loader, val_loader
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class PairedHDF5WaveformsDataset(AsyncBatchDataset):
|
|
171
|
+
def __init__(
|
|
172
|
+
self,
|
|
173
|
+
hdf5_path,
|
|
174
|
+
channel_index,
|
|
175
|
+
waveforms_dataset_name="waveforms",
|
|
176
|
+
gt_waveforms_dataset_name="gt_waveforms",
|
|
177
|
+
channels_dataset_name="channels",
|
|
178
|
+
gt_waveform_index_dataset_name=None,
|
|
179
|
+
geom=None,
|
|
180
|
+
trough_offset_samples=42,
|
|
181
|
+
spike_length_samples=121,
|
|
182
|
+
time_jitter_samples=3,
|
|
183
|
+
spatial_jitter_um=35.0,
|
|
184
|
+
generator=None,
|
|
185
|
+
chunk_size=2048,
|
|
186
|
+
queue_chunks=8,
|
|
187
|
+
):
|
|
188
|
+
"""This is meant to be used with the AOTIndicesRefreshableDataLoader
|
|
189
|
+
|
|
190
|
+
Class balancing etc can be achieved with its weights argument. They would
|
|
191
|
+
'hit' the waveforms here (not gt_waveforms, if those are indexed via
|
|
192
|
+
labels in gt_waveform_index_dataset_name).
|
|
193
|
+
"""
|
|
194
|
+
self.hdf5_path = hdf5_path
|
|
195
|
+
self._h5 = h5py.File(hdf5_path, "r", locking=False)
|
|
196
|
+
self._waveforms_dataset: h5py.Dataset = self._h5[waveforms_dataset_name]
|
|
197
|
+
self._gt_waveforms_dataset: h5py.Dataset = self._h5[gt_waveforms_dataset_name]
|
|
198
|
+
self._channels = torch.from_numpy(self._h5[channels_dataset_name][:])
|
|
199
|
+
if gt_waveform_index_dataset_name:
|
|
200
|
+
self._gt_indices = self._h5[gt_waveform_index_dataset_name][:]
|
|
201
|
+
else:
|
|
202
|
+
self._gt_indices = None
|
|
203
|
+
|
|
204
|
+
super().__init__(
|
|
205
|
+
n_examples=len(self._channels),
|
|
206
|
+
spike_length_samples=spike_length_samples,
|
|
207
|
+
generator=generator,
|
|
208
|
+
chunk_size=chunk_size,
|
|
209
|
+
queue_chunks=queue_chunks,
|
|
210
|
+
)
|
|
211
|
+
self.trough_offset_samples = trough_offset_samples
|
|
212
|
+
self.time_jitter_samples = time_jitter_samples
|
|
213
|
+
self.spatial_jitter_um = spatial_jitter_um
|
|
214
|
+
|
|
215
|
+
# spatial information
|
|
216
|
+
self.channel_index = channel_index
|
|
217
|
+
self.n_channels = len(channel_index)
|
|
218
|
+
if spatial_jitter_um:
|
|
219
|
+
if geom is None:
|
|
220
|
+
geom = self._h5["geom"][:]
|
|
221
|
+
self.spatial_jitter_index = waveform_util.make_channel_index(
|
|
222
|
+
geom, spatial_jitter_um
|
|
223
|
+
)
|
|
224
|
+
else:
|
|
225
|
+
self.spatial_jitter_index = None
|
|
226
|
+
|
|
227
|
+
# temporal info
|
|
228
|
+
min_length = spike_length_samples + 2 * time_jitter_samples
|
|
229
|
+
self._wf_length = self._waveforms_dataset.shape[1]
|
|
230
|
+
assert self._wf_length >= min_length
|
|
231
|
+
offset = (self._wf_length - spike_length_samples) // 2
|
|
232
|
+
assert self._wf_length == 2 * offset + spike_length_samples
|
|
233
|
+
self._wf_offset = offset
|
|
234
|
+
self._tix_rel = torch.arange(spike_length_samples)
|
|
235
|
+
|
|
236
|
+
def __del__(self):
|
|
237
|
+
del self._gt_waveforms_dataset
|
|
238
|
+
del self._waveforms_dataset
|
|
239
|
+
self._h5.close()
|
|
240
|
+
|
|
241
|
+
def load_batch(self, index):
|
|
242
|
+
n = index.numel()
|
|
243
|
+
waveforms = self._waveforms_dataset[index.numpy()]
|
|
244
|
+
gt_index = index.numpy()
|
|
245
|
+
if self._gt_indices is not None:
|
|
246
|
+
gt_index = self._gt_indices[gt_index]
|
|
247
|
+
gt_waveforms = self._gt_waveforms_dataset[gt_index]
|
|
248
|
+
channels = self._channels[index.numpy()]
|
|
249
|
+
|
|
250
|
+
# jitter the channels
|
|
251
|
+
if self.spatial_jitter_index:
|
|
252
|
+
choices = self.spatial_jitter_index[channels]
|
|
253
|
+
n_valid = (choices < self.n_channels).sum(1)
|
|
254
|
+
# wish they had tensor bounds.
|
|
255
|
+
which = torch.randint(2**63 - 1, size=n, generator=self.generator) % n_valid
|
|
256
|
+
channels = choices[torch.arange(n), which]
|
|
257
|
+
|
|
258
|
+
# extract the resulting channel subsets
|
|
259
|
+
chan_inds = self.channel_index[channels]
|
|
260
|
+
waveforms = F.pad(waveforms, (0, 1), value=torch.nan)
|
|
261
|
+
waveforms = waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
|
|
262
|
+
gt_waveforms = F.pad(gt_waveforms, (0, 1), value=torch.nan)
|
|
263
|
+
gt_waveforms = gt_waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
|
|
264
|
+
|
|
265
|
+
# jitter time
|
|
266
|
+
time_ixs = torch.full((n,), self._wf_offset)
|
|
267
|
+
if self.time_jitter_samples:
|
|
268
|
+
time_ixs += torch.randint(
|
|
269
|
+
-self.time_jitter_samples,
|
|
270
|
+
self.time_jitter_samples + 1,
|
|
271
|
+
generator=self.generator,
|
|
272
|
+
size=n,
|
|
273
|
+
)
|
|
274
|
+
time_ixs = time_ixs[:, None] + self._tix_rel
|
|
275
|
+
waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
|
|
276
|
+
gt_waveforms = gt_waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
|
|
277
|
+
|
|
278
|
+
return waveforms, gt_waveforms, channels
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class NoisedTemplatesDataset(AsyncBatchDataset):
|
|
282
|
+
def __init__(
|
|
283
|
+
self,
|
|
284
|
+
channel_index,
|
|
285
|
+
templates,
|
|
286
|
+
noise,
|
|
287
|
+
geom=None,
|
|
288
|
+
collision_prob=0.5,
|
|
289
|
+
max_collisions=18,
|
|
290
|
+
collision_sample_weights=None,
|
|
291
|
+
trough_offset_samples=42,
|
|
292
|
+
spike_length_samples=121,
|
|
293
|
+
time_jitter_samples=3,
|
|
294
|
+
spatial_jitter_um=35.0,
|
|
295
|
+
amplitude_jitter_std=0.1,
|
|
296
|
+
amplitude_jitter_max=1.5,
|
|
297
|
+
temporal_upsampling_factor=8,
|
|
298
|
+
generator=None,
|
|
299
|
+
chunk_size=2048,
|
|
300
|
+
queue_chunks=8,
|
|
301
|
+
dtype=torch.float,
|
|
302
|
+
):
|
|
303
|
+
"""This is meant to be used with the AOTIndicesRefreshableDataLoader
|
|
304
|
+
|
|
305
|
+
Class balancing etc can be achieved with its weights argument. This
|
|
306
|
+
would assign a weight to each template.
|
|
307
|
+
|
|
308
|
+
NOTE: collisions are on the full channel set. So, many collisions will
|
|
309
|
+
do nothing! Figure that the probe is 4000um and that a "significant"
|
|
310
|
+
collision occurs when units are spaced by <100um. If the unit positions
|
|
311
|
+
are random, a collision has ~200/4000=0.05 chance of overlapping significantly.
|
|
312
|
+
Twice that for a minor overlap?
|
|
313
|
+
|
|
314
|
+
Collisions are added binomially (# ~ Bin(max,p)), up to max_collisions,
|
|
315
|
+
so the mean # is like max_collisions*collision_prob.
|
|
316
|
+
So, the real mean is something like 0.05*max_collisions*p but censored.
|
|
317
|
+
For p=0.5 max=4, that would be like .5*4/20=0.1ish? But then the completely
|
|
318
|
+
random shifting reduces the probability by another factor of 3!
|
|
319
|
+
|
|
320
|
+
So the defaults are set to 18 max collisions for this reason:
|
|
321
|
+
18 max * 0.5 p * 1/3 time * 1/20 space = 0.15.
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
`noise` should probably be a noise_util.FactorizedNoise object, or
|
|
325
|
+
one of the other obects there which have simulate() methods.
|
|
326
|
+
"""
|
|
327
|
+
super().__init__(
|
|
328
|
+
n_examples=len(templates),
|
|
329
|
+
spike_length_samples=spike_length_samples,
|
|
330
|
+
generator=generator,
|
|
331
|
+
chunk_size=chunk_size,
|
|
332
|
+
queue_chunks=queue_chunks,
|
|
333
|
+
)
|
|
334
|
+
self.trough_offset_samples = trough_offset_samples
|
|
335
|
+
self.time_jitter_samples = time_jitter_samples
|
|
336
|
+
self.spatial_jitter_um = spatial_jitter_um
|
|
337
|
+
self.amplitude_jitter_std = amplitude_jitter_std
|
|
338
|
+
self.amplitude_jitter_max = amplitude_jitter_max
|
|
339
|
+
self.amplitude_jitter_min = 1.0 / amplitude_jitter_max
|
|
340
|
+
self.temporal_upsampling_factor = temporal_upsampling_factor
|
|
341
|
+
|
|
342
|
+
self.noise = noise
|
|
343
|
+
|
|
344
|
+
# collision control
|
|
345
|
+
self.collision_prob = collision_prob
|
|
346
|
+
self.max_collisions = max_collisions
|
|
347
|
+
self.collision_sample_weights = collision_sample_weights
|
|
348
|
+
|
|
349
|
+
# upsample and store templates as n_examples, time_up, time, chans
|
|
350
|
+
if torch.is_tensor(templates):
|
|
351
|
+
templates = templates.numpy(force=True)
|
|
352
|
+
n, t, c = templates.shape
|
|
353
|
+
templates = templates.transpose(0, 2, 1).reshape(n * c, t)
|
|
354
|
+
templates_up = waveform_util.upsample_singlechan(
|
|
355
|
+
templates, temporal_jitter=temporal_upsampling_factor
|
|
356
|
+
)
|
|
357
|
+
templates_up = templates_up.reshape(n, c, temporal_upsampling_factor, t)
|
|
358
|
+
templates_up = templates_up.transpose(0, 2, 3, 1)
|
|
359
|
+
self.templates_up = torch.asarray(templates_up, dtype=dtype).contiguous()
|
|
360
|
+
self.channels_up = self.templates_up.abs().amax(dim=2).argmax(dim=2)
|
|
361
|
+
|
|
362
|
+
# spatial information
|
|
363
|
+
self.channel_index = channel_index
|
|
364
|
+
self.n_channels = len(channel_index)
|
|
365
|
+
if spatial_jitter_um:
|
|
366
|
+
assert geom is not None
|
|
367
|
+
self.spatial_jitter_index = waveform_util.make_channel_index(
|
|
368
|
+
geom, spatial_jitter_um
|
|
369
|
+
)
|
|
370
|
+
else:
|
|
371
|
+
self.spatial_jitter_index = None
|
|
372
|
+
|
|
373
|
+
# temporal info
|
|
374
|
+
min_length = spike_length_samples + 2 * time_jitter_samples
|
|
375
|
+
self._wf_length = t
|
|
376
|
+
assert self._wf_length >= min_length, "Waveforms need to be padded."
|
|
377
|
+
offset = (self._wf_length - spike_length_samples) // 2
|
|
378
|
+
assert self._wf_length == 2 * offset + spike_length_samples
|
|
379
|
+
self._wf_offset = offset
|
|
380
|
+
self._tix_rel = torch.arange(spike_length_samples)
|
|
381
|
+
|
|
382
|
+
def load_batch(self, index):
|
|
383
|
+
n = index.numel()
|
|
384
|
+
|
|
385
|
+
# get gt waveforms
|
|
386
|
+
up_ix = torch.randint(
|
|
387
|
+
0, self.temporal_upsampling_factor, size=n, generator=self.generator
|
|
388
|
+
)
|
|
389
|
+
gt_waveforms = self.templates_up[index, up_ix]
|
|
390
|
+
channels = self.channels_up[index, up_ix]
|
|
391
|
+
|
|
392
|
+
# jitter amplitude
|
|
393
|
+
if self.amplitude_jitter_std:
|
|
394
|
+
scaling = torch.normal(
|
|
395
|
+
1.0, self.amplitude_jitter_std, size=n, generator=self.generator
|
|
396
|
+
)
|
|
397
|
+
scaling.clamp_(self.amplitude_jitter_min, self.amplitude_jitter_max)
|
|
398
|
+
gt_waveforms = gt_waveforms * scaling[..., None, None]
|
|
399
|
+
|
|
400
|
+
# get noise waveforms
|
|
401
|
+
noise = self.noise.simulate(size=n, generator=self.generator)
|
|
402
|
+
|
|
403
|
+
# add in the collisions
|
|
404
|
+
p = torch.full((self.max_collisions, n), self.collision_prob)
|
|
405
|
+
mask = torch.bernoulli(p)
|
|
406
|
+
collisions = self.get_batch_of_collisions(mask.sum())
|
|
407
|
+
s = 0
|
|
408
|
+
for m in mask:
|
|
409
|
+
(m,) = m.nonzero(as_tuple=True)
|
|
410
|
+
noise[m] += collisions[s : s + m.numel()]
|
|
411
|
+
s += m.numel()
|
|
412
|
+
|
|
413
|
+
# noisy waveforms...
|
|
414
|
+
waveforms = gt_waveforms + noise
|
|
415
|
+
|
|
416
|
+
# jitter the channels
|
|
417
|
+
if self.spatial_jitter_index:
|
|
418
|
+
choices = self.spatial_jitter_index[channels]
|
|
419
|
+
n_valid = (choices < self.n_channels).sum(1)
|
|
420
|
+
# wish they had tensor bounds.
|
|
421
|
+
which = torch.randint(2**63 - 1, size=n, generator=self.generator) % n_valid
|
|
422
|
+
channels = choices[torch.arange(n), which]
|
|
423
|
+
|
|
424
|
+
# extract the resulting channel subsets
|
|
425
|
+
chan_inds = self.channel_index[channels]
|
|
426
|
+
waveforms = F.pad(waveforms, (0, 1), value=torch.nan)
|
|
427
|
+
waveforms = waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
|
|
428
|
+
gt_waveforms = F.pad(gt_waveforms, (0, 1), value=torch.nan)
|
|
429
|
+
gt_waveforms = gt_waveforms.take_along_dim(chan_inds[:, None, :], dim=2)
|
|
430
|
+
|
|
431
|
+
# jitter time
|
|
432
|
+
time_ixs = torch.full((n,), self._wf_offset)
|
|
433
|
+
if self.time_jitter_samples:
|
|
434
|
+
time_ixs += torch.randint(
|
|
435
|
+
-self.time_jitter_samples,
|
|
436
|
+
self.time_jitter_samples + 1,
|
|
437
|
+
generator=self.generator,
|
|
438
|
+
size=n,
|
|
439
|
+
)
|
|
440
|
+
time_ixs = time_ixs[:, None] + self._tix_rel
|
|
441
|
+
waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
|
|
442
|
+
gt_waveforms = gt_waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
|
|
443
|
+
|
|
444
|
+
return waveforms, gt_waveforms, channels
|
|
445
|
+
|
|
446
|
+
def get_batch_of_collisions(self, n):
|
|
447
|
+
# draw weighted samples
|
|
448
|
+
if self.collision_sample_weights is None:
|
|
449
|
+
index = torch.randint(
|
|
450
|
+
high=self.n_examples,
|
|
451
|
+
size=n,
|
|
452
|
+
dtype=torch.int64,
|
|
453
|
+
generator=self.generator,
|
|
454
|
+
)
|
|
455
|
+
else:
|
|
456
|
+
index = torch.multinomial(
|
|
457
|
+
self.collision_sample_weights,
|
|
458
|
+
n,
|
|
459
|
+
replacement=True,
|
|
460
|
+
generator=self.generator,
|
|
461
|
+
)
|
|
462
|
+
up_ix = torch.randint(
|
|
463
|
+
0, self.temporal_upsampling_factor, size=n, generator=self.generator
|
|
464
|
+
)
|
|
465
|
+
waveforms = self.templates_up[index, up_ix]
|
|
466
|
+
|
|
467
|
+
# shift to a completely random offset, in the valid range
|
|
468
|
+
# TODO should this be some kind of taper rather than 0pad?
|
|
469
|
+
full_length = 3 * self.spike_length_samples
|
|
470
|
+
pad = (full_length - waveforms.shape[1]) // 2
|
|
471
|
+
waveforms = F.pad(waveforms, (0, 0, pad, pad))
|
|
472
|
+
time_ixs = torch.randint(
|
|
473
|
+
0, 2 * self.spike_length_samples, generator=self.generator, size=n
|
|
474
|
+
)
|
|
475
|
+
time_ixs = time_ixs[:, None] + self._tix_rel
|
|
476
|
+
waveforms = waveforms.take_along_dim(time_ixs[:, :, None], dim=1)
|
|
477
|
+
|
|
478
|
+
# jitter amplitude
|
|
479
|
+
if self.amplitude_jitter_std:
|
|
480
|
+
scaling = torch.normal(
|
|
481
|
+
1.0, self.amplitude_jitter_std, size=n, generator=self.generator
|
|
482
|
+
)
|
|
483
|
+
scaling.clamp_(self.amplitude_jitter_min, self.amplitude_jitter_max)
|
|
484
|
+
waveforms.mul_(scaling[..., None, None])
|
|
485
|
+
|
|
486
|
+
return waveforms
|