DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,635 @@
1
+ import dataclasses
2
+ from typing import cast
3
+
4
+ import numpy as np
5
+ import pandas as pd
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from torch.utils.data import (
9
+ DataLoader,
10
+ Dataset,
11
+ Sampler,
12
+ StackDataset,
13
+ TensorDataset,
14
+ )
15
+ from tqdm.auto import trange
16
+
17
+ from ..util.spiketorch import reindex, spawn_torch_rg
18
+ from ..util.logging_util import get_logger
19
+ from ._multichan_denoiser_kit import (
20
+ BaseMultichannelDenoiser,
21
+ get_noise,
22
+ AsyncSameChannelRecordingNoiseDataset,
23
+ AOTIndicesWeightedRandomBatchSampler,
24
+ NoneDataset,
25
+ )
26
+
27
+
28
+ logger = get_logger(__name__)
29
+
30
+
31
+ class Decollider(BaseMultichannelDenoiser):
32
+ default_name = "decollider"
33
+
34
+ def __init__(
35
+ self,
36
+ channel_index,
37
+ geom,
38
+ hidden_dims=(1024, 1024),
39
+ norm_kind="none",
40
+ name=None,
41
+ name_prefix="",
42
+ batch_size=256,
43
+ learning_rate=3e-4,
44
+ weight_decay=0.0,
45
+ n_epochs=75,
46
+ pad_depth_only=True,
47
+ channelwise_dropout_p=0.0,
48
+ with_conv_fullheight=False,
49
+ val_split_p=0.0,
50
+ min_epochs=10,
51
+ earlystop_eps=None,
52
+ random_seed=0,
53
+ res_type="none",
54
+ lr_schedule="CosineAnnealingLR",
55
+ lr_schedule_kwargs=None,
56
+ inference_batch_size=1024,
57
+ optimizer="Adam",
58
+ optimizer_kwargs=None,
59
+ nonlinearity="PReLU",
60
+ scaling="max",
61
+ signal_gates=True,
62
+ step_callback=None,
63
+ # my args. todo: port over common ones.
64
+ epoch_size=200 * 256,
65
+ inference_z_samples=10,
66
+ detach_amortizer=True,
67
+ exz_estimator="n3n",
68
+ inference_kind="amortized",
69
+ eyz_res_type="none",
70
+ e_exz_y_res_type="none",
71
+ emz_res_type="none",
72
+ l4_alpha=0,
73
+ l1_alpha=0,
74
+ output_l1_alpha=1e-5,
75
+ cycle_loss_alpha=1.0,
76
+ separate_cycle_net=False,
77
+ detach_cycle_loss=False,
78
+ val_noise_random_seed=0,
79
+ inf_net_hidden_dims=None,
80
+ eyz_net_hidden_dims=None,
81
+ queue_chunks=32,
82
+ ):
83
+ assert exz_estimator in ("n2n", "2n2", "n3n", "3n3")
84
+ assert inference_kind in ("raw", "exz", "exz_fromz", "amortized", "exy_fake")
85
+
86
+ super().__init__(
87
+ geom=geom,
88
+ channel_index=channel_index,
89
+ name=name,
90
+ name_prefix=name_prefix,
91
+ hidden_dims=hidden_dims,
92
+ norm_kind=norm_kind,
93
+ batch_size=batch_size,
94
+ learning_rate=learning_rate,
95
+ weight_decay=weight_decay,
96
+ n_epochs=n_epochs,
97
+ pad_depth_only=pad_depth_only,
98
+ channelwise_dropout_p=channelwise_dropout_p,
99
+ with_conv_fullheight=with_conv_fullheight,
100
+ val_split_p=val_split_p,
101
+ min_epochs=min_epochs,
102
+ earlystop_eps=earlystop_eps,
103
+ random_seed=random_seed,
104
+ res_type=res_type,
105
+ lr_schedule=lr_schedule,
106
+ lr_schedule_kwargs=lr_schedule_kwargs,
107
+ inference_batch_size=inference_batch_size,
108
+ optimizer=optimizer,
109
+ optimizer_kwargs=optimizer_kwargs,
110
+ nonlinearity=nonlinearity,
111
+ scaling=scaling,
112
+ signal_gates=signal_gates,
113
+ step_callback=step_callback,
114
+ )
115
+
116
+ self.epoch_size = epoch_size
117
+ self.queue_chunks = queue_chunks
118
+
119
+ self.inference_z_samples = inference_z_samples
120
+ self.detach_amortizer = detach_amortizer
121
+ self.exz_estimator = exz_estimator
122
+ self.inference_kind = inference_kind
123
+ self.eyz_res_type = eyz_res_type
124
+ self.e_exz_y_res_type = e_exz_y_res_type
125
+ self.emz_res_type = emz_res_type
126
+ self.val_noise_random_seed = val_noise_random_seed
127
+ self.inf_net_hidden_dims = inf_net_hidden_dims
128
+ self.eyz_net_hidden_dims = eyz_net_hidden_dims
129
+ self.l1_alpha = l1_alpha
130
+ self.l4_alpha = l4_alpha
131
+ self.output_l1_alpha = output_l1_alpha
132
+ self.cycle_loss_alpha = cycle_loss_alpha
133
+ self.separate_cycle_net = separate_cycle_net
134
+ self.detach_cycle_loss = detach_cycle_loss
135
+
136
+ if separate_cycle_net:
137
+ assert cycle_loss_alpha > 0
138
+
139
+ def initialize_spike_length_dependent_params(self):
140
+ if hasattr(self, "inf_net"):
141
+ logger.dartsortdebug("Already initialized.")
142
+ return
143
+ self.initialize_shapes()
144
+ if self.exz_estimator in ("n2n", "n3n"):
145
+ self.eyz = self.get_mlp(
146
+ res_type=self.eyz_res_type, hidden_dims=self.eyz_net_hidden_dims
147
+ )
148
+ if self.exz_estimator in ("n3n", "2n2", "3n3"):
149
+ self.emz = self.get_mlp(res_type=self.emz_res_type, output_layer="linear")
150
+ if self.inference_kind == "amortized":
151
+ self.inf_net = self.get_mlp(
152
+ res_type=self.e_exz_y_res_type, hidden_dims=self.inf_net_hidden_dims
153
+ )
154
+ if self.separate_cycle_net:
155
+ self.den_net = self.get_mlp(
156
+ res_type=self.e_exz_y_res_type, hidden_dims=self.inf_net_hidden_dims
157
+ )
158
+ else:
159
+ self.den_net = self.inf_net
160
+ self.to(self.device)
161
+
162
+ def fit(self, recording, waveforms, *, channels, weights=None, **unused):
163
+ super().fit(recording, waveforms, channels=channels, weights=weights)
164
+ train_data, val_data = self._construct_datasets_from_waveforms(
165
+ waveforms, channels, recording, weights
166
+ )
167
+ with torch.enable_grad():
168
+ res = self._fit(train_data, val_data)
169
+ self._needs_fit = False
170
+ return res
171
+
172
+ def forward_unbatched(self, waveforms, channels):
173
+ """Called only at inference time."""
174
+ # TODO: batch all of this.
175
+ waveforms, masks = self.to_nn_channels(waveforms, channels)
176
+ net_input = waveforms, masks.unsqueeze(1)
177
+
178
+ if self.inference_kind == "amortized":
179
+ pred = self.den_net(net_input)
180
+ elif self.inference_kind == "raw":
181
+ if hasattr(self, "emz"):
182
+ emz = self.emz(net_input)
183
+ pred = waveforms - emz
184
+ elif hasattr(self, "eyz"):
185
+ pred = self.eyz(net_input)
186
+ else:
187
+ assert False
188
+ elif self.inference_kind == "exz_fromz":
189
+ pred = torch.zeros_like(waveforms)
190
+ for j in range(self.inference_z_samples):
191
+ m = get_noise(
192
+ self.recording,
193
+ channels.numpy(force=True),
194
+ self.b.model_channel_index.numpy(force=True),
195
+ spike_length_samples=cast(int, self.spike_length_samples),
196
+ rg=None,
197
+ )
198
+ m = m.to(waveforms)
199
+ z = waveforms + m
200
+ net_input = z, masks.unsqueeze(1)
201
+ if self.exz_estimator == "n2n":
202
+ eyz = self.eyz(net_input)
203
+ pred += 2 * eyz - z
204
+ elif self.exz_estimator == "2n2":
205
+ emz = self.emz(net_input)
206
+ pred += z - 2 * emz
207
+ elif self.exz_estimator == "n3n":
208
+ eyz = self.eyz(net_input)
209
+ emz = self.emz(net_input)
210
+ pred += eyz - emz
211
+ elif self.exz_estimator == "3n3":
212
+ emz = self.emz(net_input)
213
+ pred += z - emz
214
+ else:
215
+ assert False
216
+ pred /= self.inference_z_samples
217
+ elif self.inference_kind == "exz":
218
+ if self.exz_estimator == "n2n":
219
+ eyz = self.eyz(net_input)
220
+ pred = 2 * eyz - waveforms
221
+ elif self.exz_estimator == "2n2":
222
+ emz = self.emz(net_input)
223
+ pred = waveforms - 2 * emz
224
+ elif self.exz_estimator == "n3n":
225
+ eyz = self.eyz(net_input)
226
+ emz = self.emz(net_input)
227
+ pred = eyz - emz
228
+ elif self.exz_estimator == "3n3":
229
+ emz = self.emz(net_input)
230
+ pred = waveforms - emz
231
+ else:
232
+ assert False
233
+ else:
234
+ assert False
235
+
236
+ pred = self.to_orig_channels(pred, channels)
237
+
238
+ return pred
239
+
240
+ def train_forward(self, y, m, ell, mask):
241
+ z = y + m
242
+
243
+ # predictions given z
244
+ # TODO: variance given z and put it in the loss
245
+ exz = eyz = emz = e_exz_y = cycle_output = None
246
+ net_input = z, mask.unsqueeze(1)
247
+ if self.exz_estimator == "n2n":
248
+ eyz = self.eyz(net_input)
249
+ exz = 2 * eyz - z
250
+ elif self.exz_estimator == "2n2":
251
+ emz = self.emz(net_input)
252
+ exz = z - 2 * emz
253
+ elif self.exz_estimator == "n3n":
254
+ eyz = self.eyz(net_input)
255
+ emz = self.emz(net_input)
256
+ exz = eyz - emz
257
+ elif self.exz_estimator == "3n3":
258
+ emz = self.emz(net_input)
259
+ exz = y - emz
260
+ else:
261
+ assert False
262
+
263
+ # predictions given y, if relevant
264
+ if self.inference_kind == "amortized":
265
+ e_exz_y = self.inf_net((y, mask.unsqueeze(1)))
266
+
267
+ if self.cycle_loss_alpha:
268
+ assert e_exz_y is not None
269
+ cycle_targ = e_exz_y.detach() if self.detach_cycle_loss else e_exz_y
270
+ cycle_input = cycle_targ + ell
271
+ cycle_output = self.den_net((cycle_input, mask.unsqueeze(1)))
272
+
273
+ return dict(
274
+ exz=exz,
275
+ eyz=eyz,
276
+ emz=emz,
277
+ e_exz_y=e_exz_y,
278
+ cycle_output=cycle_output,
279
+ )
280
+
281
+ def loss(
282
+ self,
283
+ mask,
284
+ waveforms,
285
+ m,
286
+ net_outputs,
287
+ l1_alpha=None,
288
+ l4_alpha=None,
289
+ output_l1_alpha=None,
290
+ ):
291
+ loss_dict = {}
292
+ mask = mask.unsqueeze(1)
293
+
294
+ exz = net_outputs["exz"]
295
+ eyz = net_outputs["eyz"]
296
+ emz = net_outputs["emz"]
297
+ e_exz_y = net_outputs["e_exz_y"]
298
+ cycle_output = net_outputs["cycle_output"]
299
+
300
+ if eyz is not None:
301
+ eyz_mask = mask * eyz
302
+ loss_dict["eyz"] = F.mse_loss(eyz_mask, mask * waveforms)
303
+ if l1_alpha:
304
+ loss_dict["eyz_l1"] = (
305
+ l4_alpha * (eyz - waveforms).mul_(mask).abs_().mean()
306
+ )
307
+ if l4_alpha:
308
+ loss_dict["eyz_l4"] = (
309
+ l4_alpha * ((eyz - waveforms).mul_(mask) ** 4).mean()
310
+ )
311
+ if output_l1_alpha:
312
+ loss_dict["eyz_ol1"] = output_l1_alpha * eyz_mask.abs().mean()
313
+ if emz is not None:
314
+ loss_dict["emz"] = F.mse_loss(mask * emz, mask * m)
315
+ if l1_alpha:
316
+ loss_dict["emz_l1"] = l4_alpha * (emz - m).mul_(mask).abs_().mean()
317
+ if l4_alpha:
318
+ loss_dict["emz_l4"] = l4_alpha * ((emz - m).mul_(mask) ** 4).mean()
319
+ if e_exz_y is not None:
320
+ to_amortize = exz
321
+ if self.detach_amortizer:
322
+ # should amortize-ability affect the learning of eyz, emz?
323
+ to_amortize = to_amortize.detach()
324
+ am_mask = mask * to_amortize
325
+ loss_dict["e_exz_y"] = F.mse_loss(am_mask, mask * e_exz_y)
326
+ if l1_alpha:
327
+ loss_dict["e_exz_y_l1"] = (
328
+ l4_alpha * (to_amortize - e_exz_y).mul_(mask).abs_().mean()
329
+ )
330
+ if l4_alpha:
331
+ loss_dict["e_exz_y_l4"] = (
332
+ l4_alpha * ((to_amortize - e_exz_y).mul_(mask) ** 4).mean()
333
+ )
334
+ if output_l1_alpha:
335
+ loss_dict["e_exz_y_ol1"] = output_l1_alpha * am_mask.abs().mean()
336
+ if cycle_output is not None:
337
+ coef = 1 if self.separate_cycle_net else self.cycle_loss_alpha
338
+ cycle_targ = e_exz_y.detach() if self.detach_cycle_loss else e_exz_y
339
+ loss_dict["cycle"] = coef * F.mse_loss(
340
+ mask * cycle_targ, mask * cycle_output
341
+ )
342
+ if l1_alpha:
343
+ loss_dict["cycle_l1"] = (coef * l1_alpha) * (
344
+ (cycle_targ - cycle_output).mul_(mask).abs_().mean()
345
+ )
346
+ return loss_dict
347
+
348
+ def _fit(
349
+ self,
350
+ train_data: "DecolliderDataLoader",
351
+ val_data: "DecolliderDataLoader | None",
352
+ ):
353
+ optimizer = self.get_optimizer()
354
+ scheduler = self.get_scheduler(optimizer)
355
+
356
+ last_val_loss = None
357
+ train_records = []
358
+
359
+ with trange(self.n_epochs, desc="Epochs", unit="epoch") as pbar:
360
+ for epoch in pbar:
361
+ # deal with random indices...
362
+ train_data.refresh()
363
+
364
+ # Training phase
365
+ self.train()
366
+ train_losses = {}
367
+ for (
368
+ waveform_batch,
369
+ channels_batch,
370
+ noise_batch,
371
+ cnoise_batch,
372
+ ) in train_data:
373
+ waveform_batch = waveform_batch.to(self.device)
374
+ channels_batch = channels_batch.to(self.device)
375
+ waveform_batch = reindex(
376
+ channels_batch,
377
+ waveform_batch,
378
+ self.relative_index,
379
+ pad_value=0.0,
380
+ )
381
+
382
+ optimizer.zero_grad()
383
+ m = noise_batch.to(waveform_batch)
384
+ ell = None
385
+ if cnoise_batch is not None:
386
+ ell = cnoise_batch.to(waveform_batch)
387
+ mask = self.get_masks(channels_batch).to(waveform_batch)
388
+ fres = self.train_forward(waveform_batch, m, ell, mask)
389
+ loss_dict = self.loss(
390
+ mask,
391
+ waveform_batch,
392
+ m,
393
+ fres,
394
+ l1_alpha=self.l1_alpha,
395
+ l4_alpha=self.l4_alpha,
396
+ output_l1_alpha=self.output_l1_alpha,
397
+ )
398
+ loss = sum(loss_dict.values())
399
+ loss.backward() # type: ignore
400
+ optimizer.step()
401
+
402
+ for k, v in loss_dict.items():
403
+ train_losses[k] = v + train_losses.get(k, 0.0)
404
+ # // epoch loop
405
+ train_data.cleanup()
406
+ train_losses = {
407
+ k: v.item() / len(train_data) for k, v in train_losses.items()
408
+ }
409
+ train_records.append({**train_losses})
410
+
411
+ # Validation phase (only if val_loader is not None)
412
+ val_losses = {}
413
+ val_loss = None
414
+ if val_data is not None:
415
+ self.eval()
416
+ val_losses = {}
417
+ with torch.no_grad():
418
+ for (
419
+ waveform_batch,
420
+ channels_batch,
421
+ noise_batch,
422
+ ell_batch,
423
+ ) in val_data:
424
+ waveform_batch = waveform_batch.to(self.device)
425
+ channels_batch = channels_batch.to(self.device)
426
+ noise_batch = noise_batch.to(self.device)
427
+ if ell_batch is not None:
428
+ ell_batch = ell_batch.to(self.device)
429
+
430
+ waveform_batch, mask = self.to_nn_channels(
431
+ waveform_batch, channels_batch
432
+ )
433
+ m = noise_batch.to(waveform_batch)
434
+ fres = self.train_forward(
435
+ waveform_batch, m, ell_batch, mask
436
+ )
437
+ loss_dict = self.loss(
438
+ mask,
439
+ waveform_batch,
440
+ m,
441
+ fres,
442
+ l1_alpha=self.l1_alpha,
443
+ l4_alpha=self.l4_alpha,
444
+ output_l1_alpha=self.output_l1_alpha,
445
+ )
446
+ for k, v in loss_dict.items():
447
+ val_losses[k] = v + val_losses.get(k, 0.0)
448
+
449
+ val_losses = {
450
+ k: v.item() / len(val_data) for k, v in val_losses.items()
451
+ }
452
+ val_loss = sum(val_losses.values())
453
+ train_records[-1]["val_loss"] = val_loss
454
+
455
+ if (
456
+ self.earlystop_eps is not None
457
+ and last_val_loss is not None
458
+ and val_loss - last_val_loss > self.earlystop_eps
459
+ ):
460
+ if epoch >= self.min_epochs:
461
+ print(f"Early stopping after {epoch} epochs.")
462
+ break
463
+ last_val_loss = val_loss
464
+
465
+ if self.step_callback is not None:
466
+ self.step_callback(self, epoch, val_loss)
467
+
468
+ # Print loss summary
469
+ loss_str = f"Train {loss:.4f} " + "|".join( # type: ignore
470
+ f"{k}: {v:.3f}" for k, v in train_losses.items()
471
+ )
472
+ if val_data is not None:
473
+ loss_str += f" Val {val_loss:.4f}" + "|".join(
474
+ f"{k}: {v:.3f}" for k, v in val_losses.items()
475
+ )
476
+ pbar.set_description(f"Epochs [{loss_str}]")
477
+
478
+ self.step_scheduler(scheduler, loss, val_loss) # type: ignore
479
+
480
+ train_df = pd.DataFrame.from_records(train_records)
481
+ return train_df
482
+
483
+ def _construct_datasets_from_waveforms(
484
+ self, waveforms, channels, recording, weights=None
485
+ ):
486
+ rg = np.random.default_rng(self.random_seed)
487
+
488
+ val_size = 0
489
+ train_indices = slice(None)
490
+ val_indices = None
491
+ if self.val_split_p:
492
+ num_samples = len(waveforms)
493
+ val_size = int(self.val_split_p * num_samples)
494
+ train_size = num_samples - val_size
495
+ train_indices = rg.choice(num_samples, size=train_size, replace=False)
496
+ val_indices = np.setdiff1d(np.arange(num_samples), train_indices)
497
+
498
+ spike_length_samples = waveforms.shape[1]
499
+
500
+ # training dataset
501
+ train_waveforms = waveforms[train_indices]
502
+ train_channels = channels[train_indices]
503
+ train_dataset = TensorDataset(train_waveforms, train_channels)
504
+ train_noise_dataset = AsyncSameChannelRecordingNoiseDataset(
505
+ recording,
506
+ train_channels.numpy(force=True),
507
+ self.b.model_channel_index.numpy(force=True),
508
+ spike_length_samples=spike_length_samples,
509
+ generator=spawn_torch_rg(rg),
510
+ queue_chunks=self.queue_chunks,
511
+ )
512
+ if self.cycle_loss_alpha:
513
+ train_cycle_noise_dataset = AsyncSameChannelRecordingNoiseDataset(
514
+ recording,
515
+ train_channels.numpy(force=True),
516
+ self.b.model_channel_index.numpy(force=True),
517
+ spike_length_samples=spike_length_samples,
518
+ generator=spawn_torch_rg(rg),
519
+ queue_chunks=self.queue_chunks,
520
+ )
521
+ else:
522
+ train_cycle_noise_dataset = NoneDataset(len(train_channels))
523
+
524
+ train_stack_dataset = StackDataset(
525
+ train_dataset, train_noise_dataset, train_cycle_noise_dataset
526
+ )
527
+ train_weights = None if weights is None else weights[train_indices]
528
+ train_sampler = AOTIndicesWeightedRandomBatchSampler(
529
+ n_examples=len(train_channels),
530
+ weights=train_weights,
531
+ replacement=train_weights is not None,
532
+ batch_size=self.batch_size,
533
+ generator=spawn_torch_rg(rg),
534
+ epoch_size=self.epoch_size,
535
+ )
536
+ train_loader = DataLoader(
537
+ train_stack_dataset,
538
+ sampler=train_sampler,
539
+ num_workers=0,
540
+ batch_size=None,
541
+ )
542
+ train_data = DecolliderDataLoader(
543
+ loader=train_loader,
544
+ sampler=train_sampler,
545
+ noise_dataset=train_noise_dataset,
546
+ cycle_noise_dataset=train_cycle_noise_dataset,
547
+ spike_length_samples=spike_length_samples,
548
+ )
549
+
550
+ # initialize validation datasets only if val_split_p > 0
551
+ if val_size > 0:
552
+ val_waveforms = waveforms[val_indices]
553
+ val_channels = channels[val_indices]
554
+ val_noise = get_noise(
555
+ recording,
556
+ val_channels.numpy(force=True),
557
+ self.b.model_channel_index.numpy(force=True),
558
+ spike_length_samples=spike_length_samples,
559
+ rg=rg,
560
+ )
561
+ if self.cycle_loss_alpha:
562
+ cycle_val_noise = get_noise(
563
+ recording,
564
+ val_channels.numpy(force=True),
565
+ self.b.model_channel_index.numpy(force=True),
566
+ spike_length_samples=spike_length_samples,
567
+ rg=rg,
568
+ )
569
+ cycle_val_noise = TensorDataset(cycle_val_noise)
570
+ else:
571
+ cycle_val_noise = NoneDataset(len(train_channels))
572
+ val_dataset = TensorDataset(val_waveforms, val_channels)
573
+ val_noise_dataset = TensorDataset(val_noise)
574
+
575
+ # val set does not need shuffling
576
+ val_loader = DataLoader(
577
+ val_dataset,
578
+ num_workers=self.n_data_workers, # type: ignore
579
+ persistent_workers=bool(self.n_data_workers),
580
+ batch_size=self.batch_size,
581
+ )
582
+ val_data = DecolliderDataLoader(
583
+ loader=val_loader,
584
+ sampler=None,
585
+ noise_dataset=val_noise_dataset,
586
+ cycle_noise_dataset=cycle_val_noise,
587
+ spike_length_samples=spike_length_samples,
588
+ )
589
+ else:
590
+ val_data = None
591
+
592
+ return train_data, val_data
593
+
594
+
595
+ @dataclasses.dataclass(kw_only=True, frozen=True)
596
+ class DecolliderDataLoader:
597
+ loader: DataLoader
598
+ sampler: Sampler | None
599
+ noise_dataset: Dataset
600
+ cycle_noise_dataset: Dataset | NoneDataset
601
+ spike_length_samples: int
602
+
603
+ def __len__(self):
604
+ return len(self.loader)
605
+
606
+ def __iter__(self):
607
+ """Flatten..."""
608
+ for batch in self.loader:
609
+ batch_flat = []
610
+ for item in batch:
611
+ if isinstance(item, (list, tuple)):
612
+ batch_flat.extend(item)
613
+ else:
614
+ batch_flat.append(item)
615
+ yield batch_flat
616
+
617
+ def refresh(self):
618
+ if hasattr(self.sampler, "refresh"):
619
+ self.sampler.refresh()
620
+ if hasattr(self.noise_dataset, "refresh"):
621
+ self.noise_dataset.refresh(self.sampler.indices)
622
+ if self.cycle_noise_dataset is not None and hasattr(
623
+ self.cycle_noise_dataset, "refresh"
624
+ ):
625
+ self.cycle_noise_dataset.refresh(self.sampler.indices)
626
+ else:
627
+ assert not hasattr(self.noise_dataset, "refresh")
628
+
629
+ def cleanup(self):
630
+ if hasattr(self.noise_dataset, "cleanup"):
631
+ self.noise_dataset.cleanup()
632
+ if self.cycle_noise_dataset is not None and hasattr(
633
+ self.cycle_noise_dataset, "cleanup"
634
+ ):
635
+ self.cycle_noise_dataset.cleanup()