DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,650 @@
1
+ from threading import Thread
2
+ from queue import Queue
3
+
4
+ import h5py
5
+ import torch
6
+ import torch.nn.functional as F
7
+ from torch.utils.data import Dataset, Sampler, DataLoader
8
+ import numpy as np
9
+
10
+ from .transform_base import BaseWaveformDenoiser
11
+ from ..util.waveform_util import regularize_channel_index
12
+ from ..util.spiketorch import get_relative_index, reindex, spawn_torch_rg
13
+ from ..util.logging_util import get_logger
14
+ from ..util import nn_util
15
+ from ..util import spikeio
16
+
17
+
18
+ logger = get_logger(__name__)
19
+
20
+
21
+ class BaseMultichannelDenoiser(BaseWaveformDenoiser):
22
+ def __init__(
23
+ self,
24
+ channel_index,
25
+ geom,
26
+ hidden_dims=(1024, 1024),
27
+ norm_kind="none",
28
+ name=None,
29
+ name_prefix="",
30
+ batch_size=256,
31
+ learning_rate=3e-4,
32
+ weight_decay=0.0,
33
+ n_epochs=75,
34
+ pad_depth_only=True,
35
+ channelwise_dropout_p=0.0,
36
+ with_conv_fullheight=False,
37
+ val_split_p=0.0,
38
+ min_epochs=10,
39
+ earlystop_eps=None,
40
+ random_seed=0,
41
+ res_type="none",
42
+ lr_schedule="CosineAnnealingLR",
43
+ lr_schedule_kwargs=None,
44
+ inference_batch_size=1024,
45
+ optimizer="Adam",
46
+ optimizer_kwargs=None,
47
+ nonlinearity="PReLU",
48
+ scaling="max",
49
+ signal_gates=True,
50
+ step_callback=None,
51
+ ):
52
+ super().__init__(
53
+ geom=geom, channel_index=channel_index, name=name, name_prefix=name_prefix
54
+ )
55
+
56
+ self.norm_kind = norm_kind
57
+ self.hidden_dims = hidden_dims
58
+ self.n_channels = len(geom)
59
+ self.batch_size = batch_size
60
+ self.learning_rate = learning_rate
61
+ self.n_epochs = n_epochs
62
+ self.channelwise_dropout_p = channelwise_dropout_p
63
+ self.with_conv_fullheight = with_conv_fullheight
64
+ self.lr_schedule = lr_schedule
65
+ self.lr_schedule_kwargs = lr_schedule_kwargs
66
+ self.weight_decay = weight_decay
67
+ self.optimizer = optimizer
68
+ self.optimizer_kwargs = optimizer_kwargs
69
+ self.nonlinearity = nonlinearity
70
+ self.scaling = scaling
71
+ self.signal_gates = signal_gates
72
+ self.step_callback = step_callback
73
+ self.val_split_p = val_split_p
74
+ self.min_epochs = min_epochs
75
+ self.earlystop_eps = earlystop_eps
76
+ self.res_type = res_type
77
+ self.inference_batch_size = inference_batch_size
78
+
79
+ model_channel_index = regularize_channel_index(
80
+ geom=self.geom, channel_index=channel_index, depth_only=pad_depth_only
81
+ )
82
+ self.register_buffer(
83
+ "model_channel_index", torch.from_numpy(model_channel_index)
84
+ )
85
+ self.register_buffer(
86
+ "relative_index",
87
+ get_relative_index(self.channel_index, self.model_channel_index),
88
+ )
89
+ # suburban lawns -- janitor
90
+ self.register_buffer(
91
+ "irrelative_index",
92
+ get_relative_index(self.model_channel_index, self.channel_index),
93
+ )
94
+ self._needs_fit = True
95
+ self.random_seed = random_seed
96
+
97
+ def forward_unbatched(self, waveforms, channels):
98
+ raise NotImplementedError
99
+
100
+ def forward(self, waveforms, *, channels, **unused):
101
+ out = torch.empty_like(waveforms)
102
+ odev = waveforms.device
103
+ idev = self.device
104
+
105
+ for bs in range(0, len(waveforms), self.inference_batch_size):
106
+ be = min(bs + self.inference_batch_size, len(waveforms))
107
+ pred = self.forward_unbatched(
108
+ waveforms[bs:be].to(idev), channels[bs:be].to(idev)
109
+ )
110
+ out[bs:be] = pred.to(odev)
111
+
112
+ return out
113
+
114
+ def initialize_shapes(self):
115
+ logger.dartsortdebug(
116
+ f"Initialize {self.__class__.__name__} with {self.spike_length_samples=}."
117
+ )
118
+ # we don't know these dimensions til we see a spike
119
+ self.wf_dim = self.spike_length_samples * self.model_channel_index.shape[1]
120
+ self.output_dim = self.wf_dim
121
+
122
+ def get_optimizer(self):
123
+ opt = self.optimizer
124
+ okw = self.optimizer_kwargs
125
+ if isinstance(opt, str):
126
+ opt = getattr(torch.optim, opt)
127
+ if opt is None:
128
+ opt = torch.optim.AdamW
129
+ assert issubclass(opt, torch.optim.Optimizer)
130
+ if okw is None:
131
+ okw = {}
132
+ return opt(
133
+ self.parameters(),
134
+ lr=self.learning_rate,
135
+ weight_decay=self.weight_decay,
136
+ **okw,
137
+ )
138
+
139
+ def get_scheduler(self, optimizer):
140
+ if self.lr_schedule in (None, "none"):
141
+ return None
142
+
143
+ lr_schedule = self.lr_schedule
144
+ if isinstance(lr_schedule, str):
145
+ lr_schedule = getattr(torch.optim.lr_scheduler, lr_schedule)
146
+
147
+ sched_kw = self.lr_schedule_kwargs or dict(T_max=self.n_epochs)
148
+ assert issubclass(lr_schedule, torch.optim.lr_scheduler.LRScheduler)
149
+ return lr_schedule(optimizer, **sched_kw)
150
+
151
+ def step_scheduler(self, scheduler, loss, val_loss):
152
+ if scheduler is None:
153
+ return
154
+ sc_args = ()
155
+ if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):
156
+ sc_args = (val_loss,) if val_loss is not None else loss
157
+ scheduler.step(*sc_args)
158
+
159
+ @property
160
+ def device(self) -> torch.device:
161
+ return self.b.channel_index.device
162
+
163
+ def needs_fit(self):
164
+ return self._needs_fit
165
+
166
+ def get_mlp(
167
+ self,
168
+ res_type="none",
169
+ hidden_dims=None,
170
+ output_layer=None,
171
+ log_transform=False,
172
+ ):
173
+ if hidden_dims is None:
174
+ hidden_dims = self.hidden_dims
175
+ if output_layer is None:
176
+ output_layer = "gated_linear" if self.signal_gates else "linear"
177
+ return nn_util.get_waveform_mlp(
178
+ self.spike_length_samples,
179
+ self.model_channel_index.shape[1],
180
+ self.hidden_dims,
181
+ self.output_dim,
182
+ norm_kind=self.norm_kind,
183
+ channelwise_dropout_p=self.channelwise_dropout_p,
184
+ separated_mask_input=True,
185
+ return_initial_shape=True,
186
+ initial_conv_fullheight=self.with_conv_fullheight,
187
+ final_conv_fullheight=self.with_conv_fullheight,
188
+ output_layer=output_layer,
189
+ res_type=res_type,
190
+ nonlinearity=self.nonlinearity,
191
+ log_transform=log_transform,
192
+ scaling=self.scaling,
193
+ )
194
+
195
+ def to_nn_channels(self, waveforms, channels):
196
+ waveforms = reindex(channels, waveforms, self.relative_index, pad_value=0.0)
197
+ masks = self.get_masks(channels).to(waveforms)
198
+ return waveforms, masks
199
+
200
+ def to_orig_channels(self, waveforms, channels):
201
+ return reindex(channels, waveforms, self.irrelative_index)
202
+
203
+ def get_masks(self, channels):
204
+ return self.model_channel_index[channels] < self.n_channels
205
+
206
+ # -- these two below are used for storing pretrained net weights
207
+
208
+ @classmethod
209
+ def load_from_pt(cls, pretrained_path, **kwargs):
210
+ net = cls(**kwargs)
211
+ state_dict = torch.load(pretrained_path, map_location="cpu")
212
+ net.spike_length_samples = state_dict.pop("spike_length_samples")
213
+ net.initialize_spike_length_dependent_params()
214
+ net.load_state_dict(state_dict)
215
+ net._needs_fit = False
216
+ return net
217
+
218
+ def save_to_pt(self, pretrained_path):
219
+ state = self.state_dict()
220
+ state["spike_length_samples"] = self.spike_length_samples
221
+ torch.save(state, pretrained_path)
222
+
223
+
224
+ # -- torch data / noise utilities
225
+
226
+
227
+ def get_noise(
228
+ recording,
229
+ channels,
230
+ channel_index,
231
+ spike_length_samples=121,
232
+ rg: int | None | np.random.Generator = 0,
233
+ generator: torch.Generator | None = None,
234
+ ):
235
+ if rg is not None:
236
+ rg = np.random.default_rng(rg)
237
+ # pick random times
238
+ times_samples = rg.integers(
239
+ recording.get_num_samples() - spike_length_samples,
240
+ size=len(channels),
241
+ )
242
+ else:
243
+ times_samples = torch.randint(
244
+ low=0,
245
+ high=recording.get_num_samples() - spike_length_samples,
246
+ size=(len(channels),),
247
+ device="cpu",
248
+ generator=generator,
249
+ ).numpy()
250
+
251
+ order = np.argsort(times_samples)
252
+ inv_order = np.argsort(order)
253
+
254
+ # load waveforms on the same channels and channel neighborhoods
255
+ noise_waveforms = spikeio.read_waveforms_channel_index(
256
+ recording,
257
+ times_samples[order],
258
+ channel_index,
259
+ channels,
260
+ trough_offset_samples=0,
261
+ spike_length_samples=spike_length_samples,
262
+ fill_value=0.0,
263
+ )
264
+
265
+ # back to random order
266
+ noise_waveforms = noise_waveforms[inv_order]
267
+
268
+ return torch.from_numpy(noise_waveforms)
269
+
270
+
271
+ class NoneDataset(Dataset):
272
+ def __init__(self, n):
273
+ super().__init__()
274
+ self.n = n
275
+
276
+ def __len__(self):
277
+ return self.n
278
+
279
+ def __getitem__(self, index):
280
+ return None
281
+
282
+
283
+ class SameChannelNoiseDataset(Dataset):
284
+ def __init__(
285
+ self,
286
+ recording,
287
+ channels,
288
+ channel_index,
289
+ spike_length_samples=121,
290
+ generator=None,
291
+ ):
292
+ super().__init__()
293
+ self.recording = recording
294
+ self.channels = channels
295
+ self.spike_length_samples = spike_length_samples
296
+ self.channel_index = channel_index
297
+ self.generator = generator
298
+
299
+ def __len__(self):
300
+ return len(self.channels)
301
+
302
+ def __getitem__(self, index):
303
+ noise = get_noise(
304
+ self.recording,
305
+ self.channels[index],
306
+ self.channel_index,
307
+ spike_length_samples=self.spike_length_samples,
308
+ rg=None,
309
+ generator=self.generator,
310
+ )
311
+ return noise
312
+
313
+
314
+ class RefreshableSampler(Sampler):
315
+ def refresh(self):
316
+ pass
317
+
318
+
319
+ class RefreshableDataLoader(DataLoader):
320
+ def refresh(self):
321
+ pass
322
+
323
+
324
+ class RefreshableDataset(Dataset):
325
+ def refresh(self, indices):
326
+ pass
327
+
328
+
329
+ class AOTIndicesInOrderBatchSampler(RefreshableSampler):
330
+ def __init__(self, n_examples, batch_size=None):
331
+ super().__init__()
332
+ self.n_examples = n_examples
333
+ self.indices = torch.arange(n_examples)
334
+ self.batch_size = batch_size
335
+
336
+ def __iter__(self):
337
+ if self.batch_size is None:
338
+ yield from self.indices
339
+ for bs in range(0, self.n_examples, self.batch_size):
340
+ be = min(self.n_examples, self.batch_size)
341
+ yield self.indices[bs:be]
342
+
343
+
344
+ class AOTIndicesWeightedRandomBatchSampler(RefreshableSampler):
345
+ def __init__(
346
+ self,
347
+ n_examples=None,
348
+ weights=None,
349
+ replacement=True,
350
+ batch_size=None,
351
+ generator=None,
352
+ epoch_size=None,
353
+ ):
354
+ super().__init__()
355
+
356
+ if weights is not None:
357
+ weights = torch.as_tensor(weights, dtype=torch.double)
358
+ if n_examples is None:
359
+ n_examples = len(weights)
360
+
361
+ self.n_examples = n_examples
362
+ self.weights = weights
363
+ self.replacement = replacement
364
+ self.generator = generator
365
+ self.epoch_size = epoch_size
366
+
367
+ self.indices = None
368
+ self.batch_size = batch_size
369
+
370
+ def __len__(self):
371
+ n = self.epoch_size or self.n_examples
372
+ if self.batch_size is None:
373
+ return n
374
+ return (n + self.batch_size - 1) // self.batch_size
375
+
376
+ def __iter__(self):
377
+ if self.batch_size is None:
378
+ yield from self.indices
379
+ else:
380
+ n = self.epoch_size or self.n_examples
381
+ for bs in range(0, n, self.batch_size):
382
+ be = min(n, bs + self.batch_size)
383
+ yield self.indices[bs:be]
384
+
385
+ def refresh(self):
386
+ if self.generator is None:
387
+ seed = int(torch.empty((), dtype=torch.int64).random_().item())
388
+ generator = torch.Generator()
389
+ generator.manual_seed(seed)
390
+ else:
391
+ generator = self.generator
392
+
393
+ n_draws = self.epoch_size or self.n_examples
394
+
395
+ if self.weights is None:
396
+ if self.replacement:
397
+ self.indices = torch.randint(
398
+ high=self.n_examples,
399
+ size=(n_draws,),
400
+ dtype=torch.int64,
401
+ generator=generator,
402
+ )
403
+ else:
404
+ self.indices = torch.randperm(self.n_examples, generator=generator)[
405
+ :n_draws
406
+ ]
407
+ else:
408
+ self.indices = torch.multinomial(
409
+ self.weights.to(generator.device),
410
+ n_draws,
411
+ self.replacement,
412
+ generator=generator,
413
+ )
414
+
415
+
416
+ class AOTIndicesRefreshableDataLoader(RefreshableDataLoader):
417
+ def __init__(
418
+ self,
419
+ dataset,
420
+ in_order=False,
421
+ weights=None,
422
+ replacement=True,
423
+ batch_size=None,
424
+ generator=None,
425
+ epoch_size=None,
426
+ ):
427
+ if in_order:
428
+ self.sampler = AOTIndicesWeightedRandomBatchSampler(
429
+ len(dataset),
430
+ weights=weights,
431
+ replacement=replacement,
432
+ batch_size=batch_size,
433
+ generator=generator,
434
+ epoch_size=epoch_size,
435
+ )
436
+ else:
437
+ self.sampler = AOTIndicesInOrderBatchSampler(
438
+ len(dataset), batch_size=batch_size
439
+ )
440
+ super().__init__(dataset, sampler=self.sampler)
441
+
442
+ def refresh(self):
443
+ self.sampler.refresh()
444
+ if hasattr(self.dataset, "refresh"):
445
+ self.dataset.refresh(indices)
446
+
447
+
448
+ class AsyncBatchDataset(RefreshableDataset):
449
+ def __init__(
450
+ self,
451
+ n_examples,
452
+ spike_length_samples=121,
453
+ generator=None,
454
+ chunk_size=2048,
455
+ queue_chunks=8,
456
+ ):
457
+ super().__init__()
458
+ self.spike_length_samples = spike_length_samples
459
+ # note: although there's threading here, this is only used in one
460
+ # thread, so that's okay.
461
+ self.generator = generator
462
+ self.n_examples = n_examples
463
+
464
+ self.chunk_size = chunk_size
465
+ self._queue = Queue(maxsize=queue_chunks)
466
+ self._thread = None
467
+ self._indices = None
468
+ self._cur_data_ix = None
469
+ self._cur_chunk = None
470
+ self._cur_chunk_ix = None
471
+
472
+ def __len__(self):
473
+ return self.n_examples
474
+
475
+ def cleanup(self):
476
+ if self._thread is not None:
477
+ self._thread.join()
478
+ assert not self._thread.is_alive()
479
+ self._thread = None
480
+
481
+ def refresh(self, indices):
482
+ assert self._thread is None, "Run cleanup()!"
483
+ self._cur_data_ix = 0
484
+ self._indices = indices
485
+ self._run_thread()
486
+
487
+ def __getitem__(self, index):
488
+ assert self._indices is not None
489
+ assert self._cur_data_ix is not None
490
+
491
+ bs = len(index)
492
+ my_indices = self._indices[self._cur_data_ix : self._cur_data_ix + bs]
493
+ assert torch.equal(torch.asarray(index), my_indices)
494
+
495
+ # need to batch up the chunks...
496
+ if self._cur_chunk is None:
497
+ self._cur_chunk = self._queue.get()
498
+ self._cur_chunk_ix = 0
499
+ assert self._cur_chunk_ix is not None
500
+
501
+ chunk_end_ix = min(self._cur_chunk_ix + bs, len(self._cur_chunk))
502
+ noise_batch = self._cur_chunk[self._cur_chunk_ix : chunk_end_ix]
503
+
504
+ # check if we are done with this chunk
505
+ if chunk_end_ix == len(self._cur_chunk):
506
+ self._cur_chunk = None
507
+ self._cur_chunk_ix = None
508
+ else:
509
+ self._cur_chunk_ix = chunk_end_ix
510
+
511
+ self._cur_data_ix += bs
512
+ return noise_batch
513
+
514
+ def _run_thread(self):
515
+ self._thread = Thread(target=self._thread_main, daemon=True)
516
+ self._thread.start()
517
+
518
+ def _thread_main(self):
519
+ assert self._indices is not None
520
+ for chunk_start in range(0, len(self._indices), self.chunk_size):
521
+ chunk_end = min(len(self._indices), chunk_start + self.chunk_size)
522
+
523
+ index = self._indices[chunk_start:chunk_end]
524
+ noise = self.load_batch(index)
525
+
526
+ self._queue.put(noise)
527
+
528
+ def load_batch(self, index):
529
+ # subclasses must implement
530
+ raise NotImplementedError
531
+
532
+
533
+ class AsyncSameChannelNoiseDataset(AsyncBatchDataset):
534
+ def __init__(
535
+ self,
536
+ channel_index,
537
+ n_examples=None,
538
+ channels=None,
539
+ spike_length_samples=121,
540
+ generator=None,
541
+ chunk_size=2048,
542
+ queue_chunks=8,
543
+ ):
544
+ super().__init__(
545
+ n_examples=len(channels) if channels is not None else torch.inf,
546
+ spike_length_samples=spike_length_samples,
547
+ generator=generator,
548
+ chunk_size=chunk_size,
549
+ queue_chunks=queue_chunks,
550
+ )
551
+ self.channels = channels
552
+ self.channel_index = channel_index
553
+
554
+ def __len__(self):
555
+ if self.channels is None:
556
+ return torch.inf
557
+ return len(self.channels)
558
+
559
+
560
+ class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
561
+ def __init__(
562
+ self,
563
+ recording,
564
+ channels,
565
+ channel_index,
566
+ spike_length_samples=121,
567
+ generator=None,
568
+ chunk_size=2048,
569
+ queue_chunks=8,
570
+ ):
571
+ super().__init__(
572
+ channels=channels,
573
+ channel_index=channel_index,
574
+ spike_length_samples=spike_length_samples,
575
+ generator=generator,
576
+ chunk_size=chunk_size,
577
+ queue_chunks=queue_chunks,
578
+ )
579
+ self.recording = recording
580
+
581
+ def load_batch(self, index):
582
+ return get_noise(
583
+ self.recording,
584
+ self.channels[index],
585
+ self.channel_index,
586
+ spike_length_samples=self.spike_length_samples,
587
+ rg=None,
588
+ generator=self.generator,
589
+ )
590
+
591
+
592
+ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
593
+ def __init__(
594
+ self,
595
+ hdf5_path,
596
+ channels,
597
+ channel_index,
598
+ noise_dataset_name="noise",
599
+ spike_length_samples=121,
600
+ generator=None,
601
+ chunk_size=2048,
602
+ queue_chunks=8,
603
+ ):
604
+ super().__init__(
605
+ channels=channels,
606
+ channel_index=channel_index,
607
+ spike_length_samples=spike_length_samples,
608
+ generator=generator,
609
+ chunk_size=chunk_size,
610
+ queue_chunks=queue_chunks,
611
+ )
612
+
613
+ self.hdf5_path = hdf5_path
614
+ self._h5 = h5py.File(hdf5_path, "r", locking=False)
615
+ self._dataset: h5py.Dataset = self._h5[noise_dataset_name]
616
+ assert self._dataset.ndim == 3
617
+ assert self._dataset.shape[2] == len(channel_index)
618
+ self.noise_snip_len = self._dataset.shape[1]
619
+ self.nsnips = len(self._datasset)
620
+ self._tix_rel = torch.arange(spike_length_samples)
621
+ self._tix_max = self.noise_snip_len - spike_length_samples
622
+
623
+ def __del__(self):
624
+ del self._dataset
625
+ self._h5.close()
626
+
627
+ def load_batch(self, index):
628
+ nix = index.numel()
629
+ dixs = torch.randint(
630
+ low=0, high=self.nsnips, generator=self.generator, size=nix
631
+ )
632
+
633
+ data = self._dataset[dixs.numpy()]
634
+ data = torch.from_numpy(data)
635
+
636
+ chans = self.channels[index]
637
+ chan_inds = self.channel_index[chans]
638
+ data = F.pad(data, (0, 1), value=torch.nan)
639
+ data = data.take_along_dim(chan_inds[:, None, :], dim=2)
640
+
641
+ if self._tix_max:
642
+ tixs = torch.randint(
643
+ low=0, high=self._tix_max, generator=self.generator, size=nix
644
+ )
645
+ time_inds = tixs[:, None] + self._tix_rel
646
+ data = data.take_along_dim(time_inds[:, :, None], dim=1)
647
+ else:
648
+ assert data.shape[1] == self.spike_length_samples
649
+
650
+ return data
@@ -0,0 +1,58 @@
1
+ import torch
2
+
3
+ from .amortized_localization import AmortizedLocalization
4
+ from .amplitudes import AmplitudeFeatures, AmplitudeVector, MaxAmplitude, Voltage
5
+ from .decollider import Decollider
6
+ from .enforce_decrease import EnforceDecrease
7
+ from .localize import Localization, PointSourceLocalization
8
+ from .matching_denoiser import DebugMatchingPursuitDenoiser
9
+ from .pipeline import WaveformPipeline
10
+ from .single_channel_denoiser import (
11
+ SingleChannelDenoiser,
12
+ SingleChannelWaveformDenoiser,
13
+ )
14
+ from .single_channel_templates import SingleChannelTemplates
15
+ from .supervised_denoiser import SupervisedDenoiser
16
+ from .temporal_pca import (
17
+ BaseTemporalPCA,
18
+ TemporalPCA,
19
+ TemporalPCADenoiser,
20
+ TemporalPCAFeaturizer,
21
+ )
22
+ from .transform_base import Passthrough, Waveform
23
+
24
+ all_transformers = [
25
+ Waveform,
26
+ AmplitudeVector,
27
+ MaxAmplitude,
28
+ EnforceDecrease,
29
+ SingleChannelWaveformDenoiser,
30
+ BaseTemporalPCA,
31
+ TemporalPCADenoiser,
32
+ TemporalPCAFeaturizer,
33
+ TemporalPCA,
34
+ Localization,
35
+ PointSourceLocalization,
36
+ AmortizedLocalization,
37
+ AmplitudeFeatures,
38
+ Voltage,
39
+ Decollider,
40
+ SupervisedDenoiser,
41
+ Passthrough,
42
+ SingleChannelTemplates,
43
+ DebugMatchingPursuitDenoiser,
44
+ ]
45
+
46
+ transformers_by_class_name = {cls.__name__: cls for cls in all_transformers}
47
+
48
+ # serialization
49
+ if hasattr(torch.serialization, "add_safe_globals"):
50
+ others = [
51
+ set,
52
+ slice,
53
+ torch.nn.ModuleList,
54
+ torch.nn.Sequential,
55
+ WaveformPipeline,
56
+ SingleChannelDenoiser,
57
+ ]
58
+ torch.serialization.add_safe_globals(all_transformers + others)