DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
from threading import Thread
|
|
2
|
+
from queue import Queue
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as F
|
|
7
|
+
from torch.utils.data import Dataset, Sampler, DataLoader
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from .transform_base import BaseWaveformDenoiser
|
|
11
|
+
from ..util.waveform_util import regularize_channel_index
|
|
12
|
+
from ..util.spiketorch import get_relative_index, reindex, spawn_torch_rg
|
|
13
|
+
from ..util.logging_util import get_logger
|
|
14
|
+
from ..util import nn_util
|
|
15
|
+
from ..util import spikeio
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
logger = get_logger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BaseMultichannelDenoiser(BaseWaveformDenoiser):
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
channel_index,
|
|
25
|
+
geom,
|
|
26
|
+
hidden_dims=(1024, 1024),
|
|
27
|
+
norm_kind="none",
|
|
28
|
+
name=None,
|
|
29
|
+
name_prefix="",
|
|
30
|
+
batch_size=256,
|
|
31
|
+
learning_rate=3e-4,
|
|
32
|
+
weight_decay=0.0,
|
|
33
|
+
n_epochs=75,
|
|
34
|
+
pad_depth_only=True,
|
|
35
|
+
channelwise_dropout_p=0.0,
|
|
36
|
+
with_conv_fullheight=False,
|
|
37
|
+
val_split_p=0.0,
|
|
38
|
+
min_epochs=10,
|
|
39
|
+
earlystop_eps=None,
|
|
40
|
+
random_seed=0,
|
|
41
|
+
res_type="none",
|
|
42
|
+
lr_schedule="CosineAnnealingLR",
|
|
43
|
+
lr_schedule_kwargs=None,
|
|
44
|
+
inference_batch_size=1024,
|
|
45
|
+
optimizer="Adam",
|
|
46
|
+
optimizer_kwargs=None,
|
|
47
|
+
nonlinearity="PReLU",
|
|
48
|
+
scaling="max",
|
|
49
|
+
signal_gates=True,
|
|
50
|
+
step_callback=None,
|
|
51
|
+
):
|
|
52
|
+
super().__init__(
|
|
53
|
+
geom=geom, channel_index=channel_index, name=name, name_prefix=name_prefix
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
self.norm_kind = norm_kind
|
|
57
|
+
self.hidden_dims = hidden_dims
|
|
58
|
+
self.n_channels = len(geom)
|
|
59
|
+
self.batch_size = batch_size
|
|
60
|
+
self.learning_rate = learning_rate
|
|
61
|
+
self.n_epochs = n_epochs
|
|
62
|
+
self.channelwise_dropout_p = channelwise_dropout_p
|
|
63
|
+
self.with_conv_fullheight = with_conv_fullheight
|
|
64
|
+
self.lr_schedule = lr_schedule
|
|
65
|
+
self.lr_schedule_kwargs = lr_schedule_kwargs
|
|
66
|
+
self.weight_decay = weight_decay
|
|
67
|
+
self.optimizer = optimizer
|
|
68
|
+
self.optimizer_kwargs = optimizer_kwargs
|
|
69
|
+
self.nonlinearity = nonlinearity
|
|
70
|
+
self.scaling = scaling
|
|
71
|
+
self.signal_gates = signal_gates
|
|
72
|
+
self.step_callback = step_callback
|
|
73
|
+
self.val_split_p = val_split_p
|
|
74
|
+
self.min_epochs = min_epochs
|
|
75
|
+
self.earlystop_eps = earlystop_eps
|
|
76
|
+
self.res_type = res_type
|
|
77
|
+
self.inference_batch_size = inference_batch_size
|
|
78
|
+
|
|
79
|
+
model_channel_index = regularize_channel_index(
|
|
80
|
+
geom=self.geom, channel_index=channel_index, depth_only=pad_depth_only
|
|
81
|
+
)
|
|
82
|
+
self.register_buffer(
|
|
83
|
+
"model_channel_index", torch.from_numpy(model_channel_index)
|
|
84
|
+
)
|
|
85
|
+
self.register_buffer(
|
|
86
|
+
"relative_index",
|
|
87
|
+
get_relative_index(self.channel_index, self.model_channel_index),
|
|
88
|
+
)
|
|
89
|
+
# suburban lawns -- janitor
|
|
90
|
+
self.register_buffer(
|
|
91
|
+
"irrelative_index",
|
|
92
|
+
get_relative_index(self.model_channel_index, self.channel_index),
|
|
93
|
+
)
|
|
94
|
+
self._needs_fit = True
|
|
95
|
+
self.random_seed = random_seed
|
|
96
|
+
|
|
97
|
+
def forward_unbatched(self, waveforms, channels):
|
|
98
|
+
raise NotImplementedError
|
|
99
|
+
|
|
100
|
+
def forward(self, waveforms, *, channels, **unused):
|
|
101
|
+
out = torch.empty_like(waveforms)
|
|
102
|
+
odev = waveforms.device
|
|
103
|
+
idev = self.device
|
|
104
|
+
|
|
105
|
+
for bs in range(0, len(waveforms), self.inference_batch_size):
|
|
106
|
+
be = min(bs + self.inference_batch_size, len(waveforms))
|
|
107
|
+
pred = self.forward_unbatched(
|
|
108
|
+
waveforms[bs:be].to(idev), channels[bs:be].to(idev)
|
|
109
|
+
)
|
|
110
|
+
out[bs:be] = pred.to(odev)
|
|
111
|
+
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
def initialize_shapes(self):
|
|
115
|
+
logger.dartsortdebug(
|
|
116
|
+
f"Initialize {self.__class__.__name__} with {self.spike_length_samples=}."
|
|
117
|
+
)
|
|
118
|
+
# we don't know these dimensions til we see a spike
|
|
119
|
+
self.wf_dim = self.spike_length_samples * self.model_channel_index.shape[1]
|
|
120
|
+
self.output_dim = self.wf_dim
|
|
121
|
+
|
|
122
|
+
def get_optimizer(self):
|
|
123
|
+
opt = self.optimizer
|
|
124
|
+
okw = self.optimizer_kwargs
|
|
125
|
+
if isinstance(opt, str):
|
|
126
|
+
opt = getattr(torch.optim, opt)
|
|
127
|
+
if opt is None:
|
|
128
|
+
opt = torch.optim.AdamW
|
|
129
|
+
assert issubclass(opt, torch.optim.Optimizer)
|
|
130
|
+
if okw is None:
|
|
131
|
+
okw = {}
|
|
132
|
+
return opt(
|
|
133
|
+
self.parameters(),
|
|
134
|
+
lr=self.learning_rate,
|
|
135
|
+
weight_decay=self.weight_decay,
|
|
136
|
+
**okw,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def get_scheduler(self, optimizer):
|
|
140
|
+
if self.lr_schedule in (None, "none"):
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
lr_schedule = self.lr_schedule
|
|
144
|
+
if isinstance(lr_schedule, str):
|
|
145
|
+
lr_schedule = getattr(torch.optim.lr_scheduler, lr_schedule)
|
|
146
|
+
|
|
147
|
+
sched_kw = self.lr_schedule_kwargs or dict(T_max=self.n_epochs)
|
|
148
|
+
assert issubclass(lr_schedule, torch.optim.lr_scheduler.LRScheduler)
|
|
149
|
+
return lr_schedule(optimizer, **sched_kw)
|
|
150
|
+
|
|
151
|
+
def step_scheduler(self, scheduler, loss, val_loss):
|
|
152
|
+
if scheduler is None:
|
|
153
|
+
return
|
|
154
|
+
sc_args = ()
|
|
155
|
+
if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):
|
|
156
|
+
sc_args = (val_loss,) if val_loss is not None else loss
|
|
157
|
+
scheduler.step(*sc_args)
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def device(self) -> torch.device:
|
|
161
|
+
return self.b.channel_index.device
|
|
162
|
+
|
|
163
|
+
def needs_fit(self):
|
|
164
|
+
return self._needs_fit
|
|
165
|
+
|
|
166
|
+
def get_mlp(
|
|
167
|
+
self,
|
|
168
|
+
res_type="none",
|
|
169
|
+
hidden_dims=None,
|
|
170
|
+
output_layer=None,
|
|
171
|
+
log_transform=False,
|
|
172
|
+
):
|
|
173
|
+
if hidden_dims is None:
|
|
174
|
+
hidden_dims = self.hidden_dims
|
|
175
|
+
if output_layer is None:
|
|
176
|
+
output_layer = "gated_linear" if self.signal_gates else "linear"
|
|
177
|
+
return nn_util.get_waveform_mlp(
|
|
178
|
+
self.spike_length_samples,
|
|
179
|
+
self.model_channel_index.shape[1],
|
|
180
|
+
self.hidden_dims,
|
|
181
|
+
self.output_dim,
|
|
182
|
+
norm_kind=self.norm_kind,
|
|
183
|
+
channelwise_dropout_p=self.channelwise_dropout_p,
|
|
184
|
+
separated_mask_input=True,
|
|
185
|
+
return_initial_shape=True,
|
|
186
|
+
initial_conv_fullheight=self.with_conv_fullheight,
|
|
187
|
+
final_conv_fullheight=self.with_conv_fullheight,
|
|
188
|
+
output_layer=output_layer,
|
|
189
|
+
res_type=res_type,
|
|
190
|
+
nonlinearity=self.nonlinearity,
|
|
191
|
+
log_transform=log_transform,
|
|
192
|
+
scaling=self.scaling,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def to_nn_channels(self, waveforms, channels):
|
|
196
|
+
waveforms = reindex(channels, waveforms, self.relative_index, pad_value=0.0)
|
|
197
|
+
masks = self.get_masks(channels).to(waveforms)
|
|
198
|
+
return waveforms, masks
|
|
199
|
+
|
|
200
|
+
def to_orig_channels(self, waveforms, channels):
|
|
201
|
+
return reindex(channels, waveforms, self.irrelative_index)
|
|
202
|
+
|
|
203
|
+
def get_masks(self, channels):
|
|
204
|
+
return self.model_channel_index[channels] < self.n_channels
|
|
205
|
+
|
|
206
|
+
# -- these two below are used for storing pretrained net weights
|
|
207
|
+
|
|
208
|
+
@classmethod
|
|
209
|
+
def load_from_pt(cls, pretrained_path, **kwargs):
|
|
210
|
+
net = cls(**kwargs)
|
|
211
|
+
state_dict = torch.load(pretrained_path, map_location="cpu")
|
|
212
|
+
net.spike_length_samples = state_dict.pop("spike_length_samples")
|
|
213
|
+
net.initialize_spike_length_dependent_params()
|
|
214
|
+
net.load_state_dict(state_dict)
|
|
215
|
+
net._needs_fit = False
|
|
216
|
+
return net
|
|
217
|
+
|
|
218
|
+
def save_to_pt(self, pretrained_path):
|
|
219
|
+
state = self.state_dict()
|
|
220
|
+
state["spike_length_samples"] = self.spike_length_samples
|
|
221
|
+
torch.save(state, pretrained_path)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# -- torch data / noise utilities
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def get_noise(
|
|
228
|
+
recording,
|
|
229
|
+
channels,
|
|
230
|
+
channel_index,
|
|
231
|
+
spike_length_samples=121,
|
|
232
|
+
rg: int | None | np.random.Generator = 0,
|
|
233
|
+
generator: torch.Generator | None = None,
|
|
234
|
+
):
|
|
235
|
+
if rg is not None:
|
|
236
|
+
rg = np.random.default_rng(rg)
|
|
237
|
+
# pick random times
|
|
238
|
+
times_samples = rg.integers(
|
|
239
|
+
recording.get_num_samples() - spike_length_samples,
|
|
240
|
+
size=len(channels),
|
|
241
|
+
)
|
|
242
|
+
else:
|
|
243
|
+
times_samples = torch.randint(
|
|
244
|
+
low=0,
|
|
245
|
+
high=recording.get_num_samples() - spike_length_samples,
|
|
246
|
+
size=(len(channels),),
|
|
247
|
+
device="cpu",
|
|
248
|
+
generator=generator,
|
|
249
|
+
).numpy()
|
|
250
|
+
|
|
251
|
+
order = np.argsort(times_samples)
|
|
252
|
+
inv_order = np.argsort(order)
|
|
253
|
+
|
|
254
|
+
# load waveforms on the same channels and channel neighborhoods
|
|
255
|
+
noise_waveforms = spikeio.read_waveforms_channel_index(
|
|
256
|
+
recording,
|
|
257
|
+
times_samples[order],
|
|
258
|
+
channel_index,
|
|
259
|
+
channels,
|
|
260
|
+
trough_offset_samples=0,
|
|
261
|
+
spike_length_samples=spike_length_samples,
|
|
262
|
+
fill_value=0.0,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
# back to random order
|
|
266
|
+
noise_waveforms = noise_waveforms[inv_order]
|
|
267
|
+
|
|
268
|
+
return torch.from_numpy(noise_waveforms)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class NoneDataset(Dataset):
|
|
272
|
+
def __init__(self, n):
|
|
273
|
+
super().__init__()
|
|
274
|
+
self.n = n
|
|
275
|
+
|
|
276
|
+
def __len__(self):
|
|
277
|
+
return self.n
|
|
278
|
+
|
|
279
|
+
def __getitem__(self, index):
|
|
280
|
+
return None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class SameChannelNoiseDataset(Dataset):
|
|
284
|
+
def __init__(
|
|
285
|
+
self,
|
|
286
|
+
recording,
|
|
287
|
+
channels,
|
|
288
|
+
channel_index,
|
|
289
|
+
spike_length_samples=121,
|
|
290
|
+
generator=None,
|
|
291
|
+
):
|
|
292
|
+
super().__init__()
|
|
293
|
+
self.recording = recording
|
|
294
|
+
self.channels = channels
|
|
295
|
+
self.spike_length_samples = spike_length_samples
|
|
296
|
+
self.channel_index = channel_index
|
|
297
|
+
self.generator = generator
|
|
298
|
+
|
|
299
|
+
def __len__(self):
|
|
300
|
+
return len(self.channels)
|
|
301
|
+
|
|
302
|
+
def __getitem__(self, index):
|
|
303
|
+
noise = get_noise(
|
|
304
|
+
self.recording,
|
|
305
|
+
self.channels[index],
|
|
306
|
+
self.channel_index,
|
|
307
|
+
spike_length_samples=self.spike_length_samples,
|
|
308
|
+
rg=None,
|
|
309
|
+
generator=self.generator,
|
|
310
|
+
)
|
|
311
|
+
return noise
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
class RefreshableSampler(Sampler):
|
|
315
|
+
def refresh(self):
|
|
316
|
+
pass
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class RefreshableDataLoader(DataLoader):
|
|
320
|
+
def refresh(self):
|
|
321
|
+
pass
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class RefreshableDataset(Dataset):
|
|
325
|
+
def refresh(self, indices):
|
|
326
|
+
pass
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class AOTIndicesInOrderBatchSampler(RefreshableSampler):
|
|
330
|
+
def __init__(self, n_examples, batch_size=None):
|
|
331
|
+
super().__init__()
|
|
332
|
+
self.n_examples = n_examples
|
|
333
|
+
self.indices = torch.arange(n_examples)
|
|
334
|
+
self.batch_size = batch_size
|
|
335
|
+
|
|
336
|
+
def __iter__(self):
|
|
337
|
+
if self.batch_size is None:
|
|
338
|
+
yield from self.indices
|
|
339
|
+
for bs in range(0, self.n_examples, self.batch_size):
|
|
340
|
+
be = min(self.n_examples, self.batch_size)
|
|
341
|
+
yield self.indices[bs:be]
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class AOTIndicesWeightedRandomBatchSampler(RefreshableSampler):
|
|
345
|
+
def __init__(
|
|
346
|
+
self,
|
|
347
|
+
n_examples=None,
|
|
348
|
+
weights=None,
|
|
349
|
+
replacement=True,
|
|
350
|
+
batch_size=None,
|
|
351
|
+
generator=None,
|
|
352
|
+
epoch_size=None,
|
|
353
|
+
):
|
|
354
|
+
super().__init__()
|
|
355
|
+
|
|
356
|
+
if weights is not None:
|
|
357
|
+
weights = torch.as_tensor(weights, dtype=torch.double)
|
|
358
|
+
if n_examples is None:
|
|
359
|
+
n_examples = len(weights)
|
|
360
|
+
|
|
361
|
+
self.n_examples = n_examples
|
|
362
|
+
self.weights = weights
|
|
363
|
+
self.replacement = replacement
|
|
364
|
+
self.generator = generator
|
|
365
|
+
self.epoch_size = epoch_size
|
|
366
|
+
|
|
367
|
+
self.indices = None
|
|
368
|
+
self.batch_size = batch_size
|
|
369
|
+
|
|
370
|
+
def __len__(self):
|
|
371
|
+
n = self.epoch_size or self.n_examples
|
|
372
|
+
if self.batch_size is None:
|
|
373
|
+
return n
|
|
374
|
+
return (n + self.batch_size - 1) // self.batch_size
|
|
375
|
+
|
|
376
|
+
def __iter__(self):
|
|
377
|
+
if self.batch_size is None:
|
|
378
|
+
yield from self.indices
|
|
379
|
+
else:
|
|
380
|
+
n = self.epoch_size or self.n_examples
|
|
381
|
+
for bs in range(0, n, self.batch_size):
|
|
382
|
+
be = min(n, bs + self.batch_size)
|
|
383
|
+
yield self.indices[bs:be]
|
|
384
|
+
|
|
385
|
+
def refresh(self):
|
|
386
|
+
if self.generator is None:
|
|
387
|
+
seed = int(torch.empty((), dtype=torch.int64).random_().item())
|
|
388
|
+
generator = torch.Generator()
|
|
389
|
+
generator.manual_seed(seed)
|
|
390
|
+
else:
|
|
391
|
+
generator = self.generator
|
|
392
|
+
|
|
393
|
+
n_draws = self.epoch_size or self.n_examples
|
|
394
|
+
|
|
395
|
+
if self.weights is None:
|
|
396
|
+
if self.replacement:
|
|
397
|
+
self.indices = torch.randint(
|
|
398
|
+
high=self.n_examples,
|
|
399
|
+
size=(n_draws,),
|
|
400
|
+
dtype=torch.int64,
|
|
401
|
+
generator=generator,
|
|
402
|
+
)
|
|
403
|
+
else:
|
|
404
|
+
self.indices = torch.randperm(self.n_examples, generator=generator)[
|
|
405
|
+
:n_draws
|
|
406
|
+
]
|
|
407
|
+
else:
|
|
408
|
+
self.indices = torch.multinomial(
|
|
409
|
+
self.weights.to(generator.device),
|
|
410
|
+
n_draws,
|
|
411
|
+
self.replacement,
|
|
412
|
+
generator=generator,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
class AOTIndicesRefreshableDataLoader(RefreshableDataLoader):
|
|
417
|
+
def __init__(
|
|
418
|
+
self,
|
|
419
|
+
dataset,
|
|
420
|
+
in_order=False,
|
|
421
|
+
weights=None,
|
|
422
|
+
replacement=True,
|
|
423
|
+
batch_size=None,
|
|
424
|
+
generator=None,
|
|
425
|
+
epoch_size=None,
|
|
426
|
+
):
|
|
427
|
+
if in_order:
|
|
428
|
+
self.sampler = AOTIndicesWeightedRandomBatchSampler(
|
|
429
|
+
len(dataset),
|
|
430
|
+
weights=weights,
|
|
431
|
+
replacement=replacement,
|
|
432
|
+
batch_size=batch_size,
|
|
433
|
+
generator=generator,
|
|
434
|
+
epoch_size=epoch_size,
|
|
435
|
+
)
|
|
436
|
+
else:
|
|
437
|
+
self.sampler = AOTIndicesInOrderBatchSampler(
|
|
438
|
+
len(dataset), batch_size=batch_size
|
|
439
|
+
)
|
|
440
|
+
super().__init__(dataset, sampler=self.sampler)
|
|
441
|
+
|
|
442
|
+
def refresh(self):
|
|
443
|
+
self.sampler.refresh()
|
|
444
|
+
if hasattr(self.dataset, "refresh"):
|
|
445
|
+
self.dataset.refresh(indices)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
class AsyncBatchDataset(RefreshableDataset):
|
|
449
|
+
def __init__(
|
|
450
|
+
self,
|
|
451
|
+
n_examples,
|
|
452
|
+
spike_length_samples=121,
|
|
453
|
+
generator=None,
|
|
454
|
+
chunk_size=2048,
|
|
455
|
+
queue_chunks=8,
|
|
456
|
+
):
|
|
457
|
+
super().__init__()
|
|
458
|
+
self.spike_length_samples = spike_length_samples
|
|
459
|
+
# note: although there's threading here, this is only used in one
|
|
460
|
+
# thread, so that's okay.
|
|
461
|
+
self.generator = generator
|
|
462
|
+
self.n_examples = n_examples
|
|
463
|
+
|
|
464
|
+
self.chunk_size = chunk_size
|
|
465
|
+
self._queue = Queue(maxsize=queue_chunks)
|
|
466
|
+
self._thread = None
|
|
467
|
+
self._indices = None
|
|
468
|
+
self._cur_data_ix = None
|
|
469
|
+
self._cur_chunk = None
|
|
470
|
+
self._cur_chunk_ix = None
|
|
471
|
+
|
|
472
|
+
def __len__(self):
|
|
473
|
+
return self.n_examples
|
|
474
|
+
|
|
475
|
+
def cleanup(self):
|
|
476
|
+
if self._thread is not None:
|
|
477
|
+
self._thread.join()
|
|
478
|
+
assert not self._thread.is_alive()
|
|
479
|
+
self._thread = None
|
|
480
|
+
|
|
481
|
+
def refresh(self, indices):
|
|
482
|
+
assert self._thread is None, "Run cleanup()!"
|
|
483
|
+
self._cur_data_ix = 0
|
|
484
|
+
self._indices = indices
|
|
485
|
+
self._run_thread()
|
|
486
|
+
|
|
487
|
+
def __getitem__(self, index):
|
|
488
|
+
assert self._indices is not None
|
|
489
|
+
assert self._cur_data_ix is not None
|
|
490
|
+
|
|
491
|
+
bs = len(index)
|
|
492
|
+
my_indices = self._indices[self._cur_data_ix : self._cur_data_ix + bs]
|
|
493
|
+
assert torch.equal(torch.asarray(index), my_indices)
|
|
494
|
+
|
|
495
|
+
# need to batch up the chunks...
|
|
496
|
+
if self._cur_chunk is None:
|
|
497
|
+
self._cur_chunk = self._queue.get()
|
|
498
|
+
self._cur_chunk_ix = 0
|
|
499
|
+
assert self._cur_chunk_ix is not None
|
|
500
|
+
|
|
501
|
+
chunk_end_ix = min(self._cur_chunk_ix + bs, len(self._cur_chunk))
|
|
502
|
+
noise_batch = self._cur_chunk[self._cur_chunk_ix : chunk_end_ix]
|
|
503
|
+
|
|
504
|
+
# check if we are done with this chunk
|
|
505
|
+
if chunk_end_ix == len(self._cur_chunk):
|
|
506
|
+
self._cur_chunk = None
|
|
507
|
+
self._cur_chunk_ix = None
|
|
508
|
+
else:
|
|
509
|
+
self._cur_chunk_ix = chunk_end_ix
|
|
510
|
+
|
|
511
|
+
self._cur_data_ix += bs
|
|
512
|
+
return noise_batch
|
|
513
|
+
|
|
514
|
+
def _run_thread(self):
|
|
515
|
+
self._thread = Thread(target=self._thread_main, daemon=True)
|
|
516
|
+
self._thread.start()
|
|
517
|
+
|
|
518
|
+
def _thread_main(self):
|
|
519
|
+
assert self._indices is not None
|
|
520
|
+
for chunk_start in range(0, len(self._indices), self.chunk_size):
|
|
521
|
+
chunk_end = min(len(self._indices), chunk_start + self.chunk_size)
|
|
522
|
+
|
|
523
|
+
index = self._indices[chunk_start:chunk_end]
|
|
524
|
+
noise = self.load_batch(index)
|
|
525
|
+
|
|
526
|
+
self._queue.put(noise)
|
|
527
|
+
|
|
528
|
+
def load_batch(self, index):
|
|
529
|
+
# subclasses must implement
|
|
530
|
+
raise NotImplementedError
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
class AsyncSameChannelNoiseDataset(AsyncBatchDataset):
|
|
534
|
+
def __init__(
|
|
535
|
+
self,
|
|
536
|
+
channel_index,
|
|
537
|
+
n_examples=None,
|
|
538
|
+
channels=None,
|
|
539
|
+
spike_length_samples=121,
|
|
540
|
+
generator=None,
|
|
541
|
+
chunk_size=2048,
|
|
542
|
+
queue_chunks=8,
|
|
543
|
+
):
|
|
544
|
+
super().__init__(
|
|
545
|
+
n_examples=len(channels) if channels is not None else torch.inf,
|
|
546
|
+
spike_length_samples=spike_length_samples,
|
|
547
|
+
generator=generator,
|
|
548
|
+
chunk_size=chunk_size,
|
|
549
|
+
queue_chunks=queue_chunks,
|
|
550
|
+
)
|
|
551
|
+
self.channels = channels
|
|
552
|
+
self.channel_index = channel_index
|
|
553
|
+
|
|
554
|
+
def __len__(self):
|
|
555
|
+
if self.channels is None:
|
|
556
|
+
return torch.inf
|
|
557
|
+
return len(self.channels)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
|
|
561
|
+
def __init__(
|
|
562
|
+
self,
|
|
563
|
+
recording,
|
|
564
|
+
channels,
|
|
565
|
+
channel_index,
|
|
566
|
+
spike_length_samples=121,
|
|
567
|
+
generator=None,
|
|
568
|
+
chunk_size=2048,
|
|
569
|
+
queue_chunks=8,
|
|
570
|
+
):
|
|
571
|
+
super().__init__(
|
|
572
|
+
channels=channels,
|
|
573
|
+
channel_index=channel_index,
|
|
574
|
+
spike_length_samples=spike_length_samples,
|
|
575
|
+
generator=generator,
|
|
576
|
+
chunk_size=chunk_size,
|
|
577
|
+
queue_chunks=queue_chunks,
|
|
578
|
+
)
|
|
579
|
+
self.recording = recording
|
|
580
|
+
|
|
581
|
+
def load_batch(self, index):
|
|
582
|
+
return get_noise(
|
|
583
|
+
self.recording,
|
|
584
|
+
self.channels[index],
|
|
585
|
+
self.channel_index,
|
|
586
|
+
spike_length_samples=self.spike_length_samples,
|
|
587
|
+
rg=None,
|
|
588
|
+
generator=self.generator,
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
|
|
593
|
+
def __init__(
|
|
594
|
+
self,
|
|
595
|
+
hdf5_path,
|
|
596
|
+
channels,
|
|
597
|
+
channel_index,
|
|
598
|
+
noise_dataset_name="noise",
|
|
599
|
+
spike_length_samples=121,
|
|
600
|
+
generator=None,
|
|
601
|
+
chunk_size=2048,
|
|
602
|
+
queue_chunks=8,
|
|
603
|
+
):
|
|
604
|
+
super().__init__(
|
|
605
|
+
channels=channels,
|
|
606
|
+
channel_index=channel_index,
|
|
607
|
+
spike_length_samples=spike_length_samples,
|
|
608
|
+
generator=generator,
|
|
609
|
+
chunk_size=chunk_size,
|
|
610
|
+
queue_chunks=queue_chunks,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
self.hdf5_path = hdf5_path
|
|
614
|
+
self._h5 = h5py.File(hdf5_path, "r", locking=False)
|
|
615
|
+
self._dataset: h5py.Dataset = self._h5[noise_dataset_name]
|
|
616
|
+
assert self._dataset.ndim == 3
|
|
617
|
+
assert self._dataset.shape[2] == len(channel_index)
|
|
618
|
+
self.noise_snip_len = self._dataset.shape[1]
|
|
619
|
+
self.nsnips = len(self._datasset)
|
|
620
|
+
self._tix_rel = torch.arange(spike_length_samples)
|
|
621
|
+
self._tix_max = self.noise_snip_len - spike_length_samples
|
|
622
|
+
|
|
623
|
+
def __del__(self):
|
|
624
|
+
del self._dataset
|
|
625
|
+
self._h5.close()
|
|
626
|
+
|
|
627
|
+
def load_batch(self, index):
|
|
628
|
+
nix = index.numel()
|
|
629
|
+
dixs = torch.randint(
|
|
630
|
+
low=0, high=self.nsnips, generator=self.generator, size=nix
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
data = self._dataset[dixs.numpy()]
|
|
634
|
+
data = torch.from_numpy(data)
|
|
635
|
+
|
|
636
|
+
chans = self.channels[index]
|
|
637
|
+
chan_inds = self.channel_index[chans]
|
|
638
|
+
data = F.pad(data, (0, 1), value=torch.nan)
|
|
639
|
+
data = data.take_along_dim(chan_inds[:, None, :], dim=2)
|
|
640
|
+
|
|
641
|
+
if self._tix_max:
|
|
642
|
+
tixs = torch.randint(
|
|
643
|
+
low=0, high=self._tix_max, generator=self.generator, size=nix
|
|
644
|
+
)
|
|
645
|
+
time_inds = tixs[:, None] + self._tix_rel
|
|
646
|
+
data = data.take_along_dim(time_inds[:, :, None], dim=1)
|
|
647
|
+
else:
|
|
648
|
+
assert data.shape[1] == self.spike_length_samples
|
|
649
|
+
|
|
650
|
+
return data
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
from .amortized_localization import AmortizedLocalization
|
|
4
|
+
from .amplitudes import AmplitudeFeatures, AmplitudeVector, MaxAmplitude, Voltage
|
|
5
|
+
from .decollider import Decollider
|
|
6
|
+
from .enforce_decrease import EnforceDecrease
|
|
7
|
+
from .localize import Localization, PointSourceLocalization
|
|
8
|
+
from .matching_denoiser import DebugMatchingPursuitDenoiser
|
|
9
|
+
from .pipeline import WaveformPipeline
|
|
10
|
+
from .single_channel_denoiser import (
|
|
11
|
+
SingleChannelDenoiser,
|
|
12
|
+
SingleChannelWaveformDenoiser,
|
|
13
|
+
)
|
|
14
|
+
from .single_channel_templates import SingleChannelTemplates
|
|
15
|
+
from .supervised_denoiser import SupervisedDenoiser
|
|
16
|
+
from .temporal_pca import (
|
|
17
|
+
BaseTemporalPCA,
|
|
18
|
+
TemporalPCA,
|
|
19
|
+
TemporalPCADenoiser,
|
|
20
|
+
TemporalPCAFeaturizer,
|
|
21
|
+
)
|
|
22
|
+
from .transform_base import Passthrough, Waveform
|
|
23
|
+
|
|
24
|
+
all_transformers = [
|
|
25
|
+
Waveform,
|
|
26
|
+
AmplitudeVector,
|
|
27
|
+
MaxAmplitude,
|
|
28
|
+
EnforceDecrease,
|
|
29
|
+
SingleChannelWaveformDenoiser,
|
|
30
|
+
BaseTemporalPCA,
|
|
31
|
+
TemporalPCADenoiser,
|
|
32
|
+
TemporalPCAFeaturizer,
|
|
33
|
+
TemporalPCA,
|
|
34
|
+
Localization,
|
|
35
|
+
PointSourceLocalization,
|
|
36
|
+
AmortizedLocalization,
|
|
37
|
+
AmplitudeFeatures,
|
|
38
|
+
Voltage,
|
|
39
|
+
Decollider,
|
|
40
|
+
SupervisedDenoiser,
|
|
41
|
+
Passthrough,
|
|
42
|
+
SingleChannelTemplates,
|
|
43
|
+
DebugMatchingPursuitDenoiser,
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
transformers_by_class_name = {cls.__name__: cls for cls in all_transformers}
|
|
47
|
+
|
|
48
|
+
# serialization
|
|
49
|
+
if hasattr(torch.serialization, "add_safe_globals"):
|
|
50
|
+
others = [
|
|
51
|
+
set,
|
|
52
|
+
slice,
|
|
53
|
+
torch.nn.ModuleList,
|
|
54
|
+
torch.nn.Sequential,
|
|
55
|
+
WaveformPipeline,
|
|
56
|
+
SingleChannelDenoiser,
|
|
57
|
+
]
|
|
58
|
+
torch.serialization.add_safe_globals(all_transformers + others)
|