DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1123 @@
1
+ import gc
2
+ import tempfile
3
+ import warnings
4
+ from collections import namedtuple
5
+ from pathlib import Path
6
+ from typing import Literal
7
+
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from spikeinterface.core import BaseRecording
11
+
12
+ from ..detect import (
13
+ convexity_filter,
14
+ detect_and_deduplicate,
15
+ singlechan_template_detect_and_deduplicate,
16
+ )
17
+ from ..transform import (
18
+ SingleChannelTemplates,
19
+ Voltage,
20
+ Waveform,
21
+ WaveformPipeline,
22
+ )
23
+ from ..util import job_util
24
+ from ..util.data_util import SpikeDataset, subsample_waveforms
25
+ from ..util.internal_config import (
26
+ FeaturizationConfig,
27
+ FitSamplingConfig,
28
+ SubtractionConfig,
29
+ WaveformConfig,
30
+ )
31
+ from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
32
+ from ..util.waveform_util import (
33
+ get_channel_index_rel_inds,
34
+ get_relative_subset,
35
+ make_channel_index,
36
+ relative_channel_subset_index,
37
+ )
38
+ from .peel_base import BasePeeler, PeelingBatchResult
39
+ from .threshold import ThresholdAndFeaturize, threshold_chunk
40
+
41
+
42
+ class SubtractionPeeler(BasePeeler):
43
+ peel_kind = "Subtraction"
44
+
45
+ def __init__(
46
+ self,
47
+ recording,
48
+ channel_index,
49
+ subtraction_denoising_pipeline,
50
+ featurization_pipeline,
51
+ subtract_channel_index=None,
52
+ trough_offset_samples=42,
53
+ spike_length_samples=121,
54
+ detection_threshold=4.0,
55
+ chunk_length_samples=30_000,
56
+ peak_sign="both",
57
+ realign_to_denoiser=False,
58
+ denoiser_realignment_channel: Literal["detection", "denoised"] = "detection",
59
+ denoiser_realignment_shift=5,
60
+ relative_peak_channel_index=None,
61
+ spatial_dedup_channel_index=None,
62
+ relative_peak_radius_samples=5,
63
+ temporal_dedup_radius_samples=7,
64
+ remove_exact_duplicates=True,
65
+ positive_temporal_dedup_radius_samples=41,
66
+ trough_priority: float | None = 2.0,
67
+ convexity_threshold=None,
68
+ convexity_radius=3,
69
+ n_seconds_fit=100,
70
+ max_waveforms_fit=50_000,
71
+ n_waveforms_fit=20_000,
72
+ fit_subsampling_random_state=0,
73
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
74
+ fit_max_reweighting=4.0,
75
+ residnorm_decrease_threshold=16.0,
76
+ decrease_objective: Literal["norm", "normsq", "deconv"] = "deconv",
77
+ growth_tolerance: float | None = 0.1,
78
+ use_singlechan_templates=False,
79
+ n_singlechan_templates=10,
80
+ singlechan_threshold=40.0,
81
+ singlechan_alignment_padding=20,
82
+ cumulant_order=None,
83
+ first_denoiser_max_waveforms_fit=250_000,
84
+ first_denoiser_thinning=0.5,
85
+ first_denoiser_temporal_jitter=3,
86
+ first_denoiser_spatial_jitter=35.0,
87
+ save_iteration=False,
88
+ save_residnorm_decrease=False,
89
+ max_iter=100,
90
+ dtype=torch.float,
91
+ ):
92
+ if realign_to_denoiser:
93
+ fixed_property_keys = ("channels", "time_shifts")
94
+ else:
95
+ fixed_property_keys = ("channels",)
96
+
97
+ super().__init__(
98
+ recording=recording,
99
+ channel_index=channel_index,
100
+ featurization_pipeline=featurization_pipeline,
101
+ chunk_length_samples=chunk_length_samples,
102
+ chunk_margin_samples=self.next_margin(2 * spike_length_samples),
103
+ n_seconds_fit=n_seconds_fit,
104
+ max_waveforms_fit=max_waveforms_fit,
105
+ fit_subsampling_random_state=fit_subsampling_random_state,
106
+ n_waveforms_fit=n_waveforms_fit,
107
+ fit_max_reweighting=fit_max_reweighting,
108
+ fit_sampling=fit_sampling,
109
+ trough_offset_samples=trough_offset_samples,
110
+ spike_length_samples=spike_length_samples,
111
+ fixed_property_keys=fixed_property_keys,
112
+ dtype=dtype,
113
+ )
114
+
115
+ self.relative_peak_radius_samples = relative_peak_radius_samples
116
+ self.peak_sign = peak_sign
117
+ self.realign_to_denoiser = realign_to_denoiser
118
+ self.denoiser_realignment_shift = denoiser_realignment_shift
119
+ self.detection_threshold = detection_threshold
120
+ self.residnorm_decrease_threshold = residnorm_decrease_threshold
121
+ self.temporal_dedup_radius_samples = temporal_dedup_radius_samples
122
+ self.remove_exact_duplicates = remove_exact_duplicates
123
+ self.positive_temporal_dedup_radius_samples = (
124
+ positive_temporal_dedup_radius_samples
125
+ )
126
+ self.trough_priority = trough_priority
127
+ self.growth_tolerance = growth_tolerance
128
+ self.convexity_threshold = convexity_threshold
129
+ self.convexity_radius = convexity_radius
130
+ self.save_iteration = save_iteration
131
+ self.save_residnorm_decrease = save_residnorm_decrease
132
+ self.max_iter = max_iter
133
+ self.decrease_objective: Literal["norm", "normsq", "deconv"] = decrease_objective
134
+
135
+ if subtract_channel_index is None:
136
+ subtract_channel_index = channel_index.clone().detach()
137
+ self.register_buffer("subtract_channel_index", subtract_channel_index)
138
+ self.denoiser_realignment_channel = denoiser_realignment_channel
139
+ self.register_buffer(
140
+ "subtract_index_rel_inds",
141
+ get_channel_index_rel_inds(subtract_channel_index),
142
+ )
143
+ self.fixed_output_data.append(
144
+ (
145
+ "subtract_channel_index",
146
+ self.b.subtract_channel_index.numpy(force=True).copy(),
147
+ ),
148
+ )
149
+ self.extract_subtract_same = torch.equal(
150
+ self.b.subtract_channel_index, self.b.channel_index
151
+ )
152
+ if not self.extract_subtract_same:
153
+ esmask = relative_channel_subset_index(
154
+ self.subtract_channel_index, self.channel_index
155
+ )
156
+ self.register_buffer("extract_subtract_mask", torch.asarray(esmask))
157
+ else:
158
+ self.extract_subtract_mask = None
159
+ self.dedup_batch_size = self.nearest_batch_length()
160
+ if spatial_dedup_channel_index is not None:
161
+ self.register_buffer(
162
+ "spatial_dedup_channel_index", spatial_dedup_channel_index
163
+ )
164
+ self.register_buffer(
165
+ "spatial_dedup_rel_inds",
166
+ get_channel_index_rel_inds(spatial_dedup_channel_index),
167
+ )
168
+ else:
169
+ self.spatial_dedup_channel_index = None
170
+ self.spatial_dedup_rel_inds = None
171
+ if relative_peak_channel_index is not None:
172
+ self.register_buffer(
173
+ "relative_peak_channel_index", relative_peak_channel_index
174
+ )
175
+ else:
176
+ self.relative_peak_channel_index = None
177
+ self.add_module(
178
+ "subtraction_denoising_pipeline", subtraction_denoising_pipeline
179
+ )
180
+
181
+ # first denoiser fitting parameters
182
+ self.first_denoiser_max_waveforms_fit = first_denoiser_max_waveforms_fit
183
+ thinning_length_cond = recording.get_total_duration() >= 2 * n_seconds_fit
184
+ self.first_denoiser_thinning = (
185
+ first_denoiser_thinning if thinning_length_cond else 0.0
186
+ )
187
+ self.first_denoiser_temporal_jitter = first_denoiser_temporal_jitter
188
+ self.first_denoiser_spatial_jitter = first_denoiser_spatial_jitter
189
+
190
+ # cumulant detection
191
+ self.cumulant_order = cumulant_order
192
+
193
+ # singlechan template based detection
194
+ self.use_singlechan_templates = use_singlechan_templates
195
+ self.have_singlechan_templates = False
196
+ self.singlechan_threshold = singlechan_threshold
197
+ if use_singlechan_templates:
198
+ sc_feat = SingleChannelTemplates(
199
+ self.channel_index,
200
+ n_centroids=n_singlechan_templates,
201
+ alignment_padding=singlechan_alignment_padding,
202
+ trough_offset_samples=self.trough_offset_samples,
203
+ max_waveforms=n_waveforms_fit,
204
+ )
205
+ if self.featurization_pipeline is None:
206
+ self.featurization_pipeline = WaveformPipeline([sc_feat])
207
+ else:
208
+ self.featurization_pipeline.transformers.insert(0, sc_feat)
209
+
210
+ def out_datasets(self):
211
+ datasets = super().out_datasets()
212
+
213
+ if self.save_iteration:
214
+ datasets.append(SpikeDataset("iteration", (), "int32"))
215
+ if self.realign_to_denoiser:
216
+ datasets.append(SpikeDataset("time_shifts", (), "int16"))
217
+ if self.save_residnorm_decrease:
218
+ datasets.append(SpikeDataset("residnorm_decreases", (), "float32"))
219
+
220
+ # we may be featurizing during subtraction, register the features
221
+ datasets.extend(self.subtraction_denoising_pipeline.spike_datasets())
222
+
223
+ return datasets
224
+
225
+ def peeling_needs_fit(self):
226
+ return self.subtraction_denoising_pipeline.needs_fit()
227
+
228
+ def peeling_needs_precompute(self):
229
+ return self.subtraction_denoising_pipeline.needs_precompute()
230
+
231
+ def save_models(self, save_folder):
232
+ super().save_models(save_folder)
233
+ sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
234
+ torch.save(self.subtraction_denoising_pipeline.state_dict(), sub_denoise_pt)
235
+
236
+ def load_models(self, save_folder):
237
+ super().load_models(save_folder)
238
+ sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
239
+ if sub_denoise_pt.exists():
240
+ state_dict = torch.load(sub_denoise_pt)
241
+ self.subtraction_denoising_pipeline.load_state_dict(state_dict)
242
+
243
+ @classmethod
244
+ def from_config(
245
+ cls,
246
+ *,
247
+ recording: BaseRecording,
248
+ waveform_cfg: WaveformConfig,
249
+ subtraction_cfg: SubtractionConfig,
250
+ featurization_cfg: FeaturizationConfig,
251
+ sampling_cfg: FitSamplingConfig,
252
+ ):
253
+ # waveform extraction channel neighborhoods
254
+ geom = torch.tensor(recording.get_channel_locations())
255
+ channel_index = make_channel_index(
256
+ geom, featurization_cfg.extract_radius, to_torch=True
257
+ )
258
+ subtract_channel_index = make_channel_index(
259
+ geom, subtraction_cfg.subtract_radius, to_torch=True
260
+ )
261
+ # per-threshold spike event deduplication channel neighborhoods
262
+ if subtraction_cfg.spatial_dedup_radius is not None:
263
+ spatial_dedup_channel_index = make_channel_index(
264
+ geom, subtraction_cfg.spatial_dedup_radius, to_torch=True
265
+ )
266
+ else:
267
+ spatial_dedup_channel_index = None
268
+
269
+ relative_peak_channel_index = None
270
+ if subtraction_cfg.relative_peak_radius_um:
271
+ relative_peak_channel_index = make_channel_index(
272
+ geom, subtraction_cfg.relative_peak_radius_um, to_torch=True
273
+ )
274
+
275
+ # construct denoising and featurization pipelines
276
+ subtraction_denoising_pipeline = WaveformPipeline.from_config(
277
+ geom=geom,
278
+ channel_index=subtract_channel_index,
279
+ featurization_cfg=subtraction_cfg.subtraction_denoising_cfg,
280
+ waveform_cfg=waveform_cfg,
281
+ sampling_frequency=recording.sampling_frequency,
282
+ )
283
+ featurization_pipeline = WaveformPipeline.from_config(
284
+ geom=geom,
285
+ channel_index=channel_index,
286
+ featurization_cfg=featurization_cfg,
287
+ waveform_cfg=waveform_cfg,
288
+ sampling_frequency=recording.sampling_frequency,
289
+ )
290
+
291
+ # waveform logic
292
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
293
+ recording.sampling_frequency
294
+ )
295
+ spike_length_samples = waveform_cfg.spike_length_samples(
296
+ recording.sampling_frequency
297
+ )
298
+
299
+ if trough_offset_samples != 42 or spike_length_samples != 121:
300
+ # temporary warning just so I can see if this happens
301
+ warnings.warn(
302
+ f"waveform_cfg {trough_offset_samples=} {spike_length_samples=} "
303
+ f"since {recording.sampling_frequency=}"
304
+ )
305
+ singlechan_alignment_padding = int(
306
+ subtraction_cfg.singlechan_alignment_padding_ms
307
+ * (recording.sampling_frequency / 1000)
308
+ )
309
+
310
+ return cls(
311
+ recording=recording,
312
+ channel_index=channel_index,
313
+ subtraction_denoising_pipeline=subtraction_denoising_pipeline,
314
+ featurization_pipeline=featurization_pipeline,
315
+ subtract_channel_index=subtract_channel_index,
316
+ trough_offset_samples=trough_offset_samples,
317
+ spike_length_samples=spike_length_samples,
318
+ detection_threshold=subtraction_cfg.detection_threshold,
319
+ chunk_length_samples=subtraction_cfg.chunk_length_samples,
320
+ peak_sign=subtraction_cfg.peak_sign,
321
+ relative_peak_radius_samples=subtraction_cfg.relative_peak_radius_samples,
322
+ realign_to_denoiser=subtraction_cfg.realign_to_denoiser,
323
+ denoiser_realignment_shift=subtraction_cfg.denoiser_realignment_shift,
324
+ relative_peak_channel_index=relative_peak_channel_index,
325
+ spatial_dedup_channel_index=spatial_dedup_channel_index,
326
+ temporal_dedup_radius_samples=subtraction_cfg.temporal_dedup_radius_samples,
327
+ remove_exact_duplicates=subtraction_cfg.remove_exact_duplicates,
328
+ positive_temporal_dedup_radius_samples=subtraction_cfg.positive_temporal_dedup_radius_samples,
329
+ n_seconds_fit=subtraction_cfg.n_seconds_fit,
330
+ max_waveforms_fit=sampling_cfg.max_waveforms_fit,
331
+ fit_sampling=sampling_cfg.fit_sampling,
332
+ fit_max_reweighting=sampling_cfg.fit_max_reweighting,
333
+ n_waveforms_fit=sampling_cfg.n_waveforms_fit,
334
+ fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
335
+ residnorm_decrease_threshold=subtraction_cfg.residnorm_decrease_threshold,
336
+ convexity_threshold=subtraction_cfg.convexity_threshold,
337
+ convexity_radius=subtraction_cfg.convexity_radius,
338
+ use_singlechan_templates=subtraction_cfg.use_singlechan_templates,
339
+ n_singlechan_templates=subtraction_cfg.n_singlechan_templates,
340
+ singlechan_threshold=subtraction_cfg.singlechan_threshold,
341
+ singlechan_alignment_padding=singlechan_alignment_padding,
342
+ cumulant_order=subtraction_cfg.cumulant_order,
343
+ first_denoiser_max_waveforms_fit=subtraction_cfg.first_denoiser_max_waveforms_fit,
344
+ first_denoiser_thinning=subtraction_cfg.first_denoiser_thinning,
345
+ first_denoiser_temporal_jitter=subtraction_cfg.first_denoiser_temporal_jitter,
346
+ first_denoiser_spatial_jitter=subtraction_cfg.first_denoiser_spatial_jitter,
347
+ growth_tolerance=subtraction_cfg.growth_tolerance,
348
+ trough_priority=subtraction_cfg.trough_priority,
349
+ save_iteration=subtraction_cfg.save_iteration,
350
+ save_residnorm_decrease=subtraction_cfg.save_residnorm_decrease,
351
+ )
352
+
353
+ def peel_chunk(
354
+ self,
355
+ traces,
356
+ *,
357
+ chunk_start_samples=0,
358
+ left_margin=0,
359
+ right_margin=0,
360
+ return_residual=False,
361
+ return_waveforms=True,
362
+ ):
363
+ del return_waveforms # always done here
364
+
365
+ extract_index = None if self.extract_subtract_same else self.channel_index
366
+ traces = traces.to(self.dtype)
367
+ if self.have_singlechan_templates:
368
+ assert self.featurization_pipeline is not None
369
+ sc_feat = self.featurization_pipeline.transformers[0]
370
+ assert isinstance(sc_feat, SingleChannelTemplates)
371
+ singlechan_kw = dict(
372
+ singlechan_templates=sc_feat.templates,
373
+ singlechan_threshold=self.singlechan_threshold,
374
+ singlechan_trough_offset=sc_feat.template_trough,
375
+ )
376
+ else:
377
+ singlechan_kw = {}
378
+
379
+ subtraction_result = subtract_chunk(
380
+ traces,
381
+ self.subtract_channel_index,
382
+ self.subtraction_denoising_pipeline,
383
+ extract_index=extract_index,
384
+ extract_mask=self.extract_subtract_mask,
385
+ trough_offset_samples=self.trough_offset_samples,
386
+ spike_length_samples=self.spike_length_samples,
387
+ left_margin=left_margin,
388
+ right_margin=right_margin,
389
+ detection_threshold=self.detection_threshold,
390
+ relative_peak_radius=self.relative_peak_radius_samples,
391
+ peak_sign=self.peak_sign,
392
+ relative_peak_channel_index=self.relative_peak_channel_index,
393
+ spatial_dedup_channel_index=self.spatial_dedup_channel_index,
394
+ dedup_batch_size=self.dedup_batch_size,
395
+ dedup_temporal_radius=self.temporal_dedup_radius_samples,
396
+ remove_exact_duplicates=self.remove_exact_duplicates,
397
+ pos_dedup_temporal_radius=self.positive_temporal_dedup_radius_samples,
398
+ residnorm_decrease_threshold=self.residnorm_decrease_threshold,
399
+ decrease_objective=self.decrease_objective,
400
+ trough_priority=self.trough_priority,
401
+ growth_tolerance=self.growth_tolerance,
402
+ cumulant_order=self.cumulant_order,
403
+ convexity_threshold=self.convexity_threshold,
404
+ convexity_radius=self.convexity_radius,
405
+ save_iteration=self.save_iteration,
406
+ save_residnorm_decrease=self.save_residnorm_decrease,
407
+ max_iter=self.max_iter,
408
+ subtract_rel_inds=self.subtract_index_rel_inds,
409
+ spatial_dedup_rel_inds=self.spatial_dedup_rel_inds,
410
+ realign_to_denoiser=self.realign_to_denoiser,
411
+ denoiser_realignment_shift=self.denoiser_realignment_shift,
412
+ denoiser_realignment_channel=self.denoiser_realignment_channel,
413
+ **singlechan_kw, # type: ignore
414
+ )
415
+
416
+ # add in chunk_start_samples
417
+ times_samples = subtraction_result.times_samples + chunk_start_samples
418
+
419
+ peel_result = PeelingBatchResult(
420
+ n_spikes=subtraction_result.n_spikes,
421
+ times_samples=times_samples,
422
+ channels=subtraction_result.channels,
423
+ collisioncleaned_waveforms=subtraction_result.collisioncleaned_waveforms,
424
+ )
425
+ peel_result.update(subtraction_result.features)
426
+ if return_residual:
427
+ peel_result["residual"] = subtraction_result.residual
428
+
429
+ return peel_result
430
+
431
+ def precompute_peeler_models(
432
+ self, save_folder, overwrite=False, computation_cfg=None
433
+ ):
434
+ self.subtraction_denoising_pipeline.precompute()
435
+
436
+ def fit_featurization_pipeline(self, tmp_dir=None, computation_cfg=None):
437
+ super().fit_featurization_pipeline(
438
+ tmp_dir=tmp_dir, computation_cfg=computation_cfg
439
+ )
440
+ if self.use_singlechan_templates:
441
+ self.have_singlechan_templates = True
442
+
443
+ def fit_peeler_models(self, save_folder, tmp_dir=None, computation_cfg=None):
444
+ # when fitting peelers for subtraction, there are basically
445
+ # two cases. fitting featurizers is easy -- they don't modify
446
+ # the waveforms. fitting denoisers is hard -- they do. each
447
+ # denoiser that needs fitting will affect any transformer
448
+ # that comes after it in the pipeline.
449
+
450
+ # but, we usually only have one denoiser that needs fitting
451
+ # in the pipeline, which is temporal PCA (after pretrained NN)
452
+ # so we will cheat for now:
453
+ # just remove all the denoisers that need fitting, run peeling,
454
+ # and fit everything
455
+
456
+ while self._fit_subtraction_transformers(
457
+ save_folder,
458
+ tmp_dir=tmp_dir,
459
+ computation_cfg=computation_cfg,
460
+ which="denoisers",
461
+ ):
462
+ pass
463
+ self._fit_subtraction_transformers(
464
+ save_folder,
465
+ tmp_dir=tmp_dir,
466
+ computation_cfg=computation_cfg,
467
+ which="featurizers",
468
+ )
469
+
470
+ gc.collect()
471
+ torch.cuda.empty_cache()
472
+
473
+ def _fit_subtraction_transformers(
474
+ self, save_folder, tmp_dir=None, computation_cfg=None, which="denoisers"
475
+ ):
476
+ """Fit models which are run during the subtraction step
477
+
478
+ These include denoisers and featurizers. Featurizers are easy, we just fit them
479
+ to the extracted waveforms output from a mini-subtraction. Denoisers are a bit
480
+ harder, since they influence the actual waveforms that are extracted. In that sense,
481
+ you need to fit them serially with a new mini subtraction each time.
482
+ """
483
+ if which == "denoisers":
484
+ needs_fit = any(
485
+ t.is_denoiser and t.needs_fit()
486
+ for t in self.subtraction_denoising_pipeline
487
+ )
488
+ elif which == "featurizers":
489
+ assert not any(
490
+ t.is_denoiser and t.needs_fit()
491
+ for t in self.subtraction_denoising_pipeline
492
+ )
493
+ needs_fit = any(
494
+ t.is_featurizer and t.needs_fit()
495
+ for t in self.subtraction_denoising_pipeline
496
+ )
497
+ else:
498
+ assert False
499
+
500
+ if not needs_fit:
501
+ return False
502
+
503
+ if computation_cfg is None:
504
+ computation_cfg = job_util.get_global_computation_config()
505
+ device = computation_cfg.actual_device()
506
+
507
+ orig_denoise = self.subtraction_denoising_pipeline
508
+ init_voltage_feature = Voltage(
509
+ channel_index=self.subtract_channel_index,
510
+ name="subtract_fit_voltages",
511
+ )
512
+ init_waveform_feature = Waveform(
513
+ channel_index=self.subtract_channel_index,
514
+ name="subtract_fit_waveforms",
515
+ )
516
+ ifeats = [init_voltage_feature, init_waveform_feature]
517
+ if which == "denoisers":
518
+ # add all the already fitted denoisers until we hit the next unfitted one
519
+ already_fitted = []
520
+ fit_feats = []
521
+ for t in orig_denoise:
522
+ if t.is_denoiser:
523
+ if t.needs_fit():
524
+ fit_feats = [t]
525
+ break
526
+ already_fitted.append(t)
527
+
528
+ # this is the sequence of transforms to actually use in fitting
529
+ fit_feats = already_fitted + fit_feats
530
+ else:
531
+ fit_feats = None
532
+ already_fitted = [t for t in orig_denoise if t.is_denoiser]
533
+
534
+ # if fitting the very first denoiser...
535
+ if which == "denoisers" and not already_fitted:
536
+ assert fit_feats is not None
537
+ fit_pipeline = WaveformPipeline(fit_feats)
538
+ self._threshold_to_fit(
539
+ tmp_dir, fit_pipeline, computation_cfg=computation_cfg
540
+ )
541
+ return True
542
+
543
+ self.subtraction_denoising_pipeline = WaveformPipeline(ifeats + already_fitted)
544
+
545
+ # and we don't need any features for this
546
+ orig_featurization_pipeline = self.featurization_pipeline
547
+ self.featurization_pipeline = WaveformPipeline([])
548
+
549
+ # run mini subtraction
550
+ with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
551
+ temp_hdf5_filename = Path(temp_dir) / f"subtraction_{which[:-1]}_fit.h5"
552
+ try:
553
+ self.run_subsampled_peeling(
554
+ temp_hdf5_filename,
555
+ computation_cfg=computation_cfg,
556
+ task_name=f"Load examples for {which[:-1]} fitting",
557
+ )
558
+
559
+ # fit featurization pipeline and reassign
560
+ # work in a try finally so we can delete the temp file
561
+ # in case of an issue or a keyboard interrupt
562
+ waveforms, fixed_properties = subsample_waveforms(
563
+ temp_hdf5_filename,
564
+ fit_sampling=self.fit_sampling,
565
+ random_state=self.fit_subsampling_random_state,
566
+ n_waveforms_fit=self.n_waveforms_fit,
567
+ fit_max_reweighting=self.fit_max_reweighting,
568
+ voltages_dataset_name="subtract_fit_voltages",
569
+ waveforms_dataset_name="subtract_fit_waveforms",
570
+ device="cpu" if which == "denoisers" else device,
571
+ )
572
+ # these are on CPU for now.
573
+ assert fit_feats is not None
574
+ fit_denoise = WaveformPipeline(fit_feats)
575
+ fit_denoise = fit_denoise.to(device)
576
+ fit_denoise.fit(
577
+ recording=self.recording, waveforms=waveforms, **fixed_properties
578
+ )
579
+ fit_denoise = fit_denoise.to("cpu")
580
+ self.subtraction_denoising_pipeline = orig_denoise
581
+ self.featurization_pipeline = orig_featurization_pipeline
582
+ finally:
583
+ self.to("cpu")
584
+ if temp_hdf5_filename.exists():
585
+ temp_hdf5_filename.unlink()
586
+ return True
587
+
588
+ def _threshold_to_fit(self, tmp_dir, fit_pipeline, computation_cfg):
589
+ geom = self.recording.get_channel_locations()
590
+ spatial_jitter_index = None
591
+ if self.first_denoiser_spatial_jitter:
592
+ spatial_jitter_index = make_channel_index(
593
+ geom, self.first_denoiser_spatial_jitter, to_torch=True
594
+ )
595
+ waveform_pipeline = WaveformPipeline([Waveform(self.subtract_channel_index)])
596
+ trainer = ThresholdAndFeaturize(
597
+ self.recording,
598
+ detection_threshold=self.detection_threshold,
599
+ channel_index=self.subtract_channel_index,
600
+ relative_peak_channel_index=self.relative_peak_channel_index,
601
+ spatial_dedup_channel_index=self.spatial_dedup_channel_index,
602
+ relative_peak_radius_samples=self.relative_peak_radius_samples,
603
+ featurization_pipeline=waveform_pipeline,
604
+ temporal_dedup_radius_samples=self.spike_length_samples,
605
+ thinning=self.first_denoiser_thinning,
606
+ time_jitter=self.first_denoiser_temporal_jitter,
607
+ spatial_jitter_channel_index=spatial_jitter_index,
608
+ peak_sign=self.peak_sign,
609
+ trough_priority=self.trough_priority,
610
+ convexity_radius=self.convexity_radius,
611
+ convexity_threshold=self.convexity_threshold,
612
+ cumulant_order=self.cumulant_order,
613
+ remove_exact_duplicates=self.remove_exact_duplicates,
614
+ )
615
+
616
+ with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
617
+ temp_hdf5_filename = Path(temp_dir) / f"subtraction_denoiser0_fit.h5"
618
+ try:
619
+ trainer.peel(
620
+ temp_hdf5_filename,
621
+ shuffle=True,
622
+ stop_after_n_waveforms=self.first_denoiser_max_waveforms_fit,
623
+ task_name=f"Load examples for initial denoiser fitting",
624
+ computation_cfg=computation_cfg,
625
+ )
626
+
627
+ # get fit weights
628
+ device = computation_cfg.actual_device()
629
+ waveforms, fixed_properties = subsample_waveforms(
630
+ temp_hdf5_filename,
631
+ fit_sampling=self.fit_sampling,
632
+ random_state=self.fit_subsampling_random_state,
633
+ n_waveforms_fit=self.first_denoiser_max_waveforms_fit,
634
+ fit_max_reweighting=self.fit_max_reweighting,
635
+ voltages_dataset_name="voltages",
636
+ waveforms_dataset_name="waveforms",
637
+ subsample_by_weighting=True,
638
+ )
639
+
640
+ # fit the thing
641
+ fit_pipeline = fit_pipeline.to(device)
642
+ fit_pipeline.fit(
643
+ recording=self.recording, waveforms=waveforms, **fixed_properties
644
+ )
645
+ fit_pipeline.to("cpu")
646
+ finally:
647
+ if temp_hdf5_filename.exists():
648
+ temp_hdf5_filename.unlink()
649
+
650
+
651
+ ChunkSubtractionResult = namedtuple(
652
+ "ChunkSubtractionResult",
653
+ [
654
+ "n_spikes",
655
+ "times_samples",
656
+ "channels",
657
+ "collisioncleaned_waveforms",
658
+ "residual",
659
+ "features",
660
+ ],
661
+ )
662
+
663
+
664
+ def subtract_chunk(
665
+ traces,
666
+ channel_index,
667
+ denoising_pipeline,
668
+ extract_index=None,
669
+ extract_mask=None,
670
+ trough_offset_samples=42,
671
+ spike_length_samples=121,
672
+ left_margin=0,
673
+ right_margin=0,
674
+ detection_threshold=4.0,
675
+ peak_sign="both",
676
+ realign_to_denoiser=False,
677
+ denoiser_realignment_shift=5,
678
+ denoiser_realignment_channel="detection",
679
+ convexity_threshold=None,
680
+ convexity_radius=3,
681
+ relative_peak_channel_index=None,
682
+ spatial_dedup_channel_index=None,
683
+ subtract_rel_inds=None,
684
+ spatial_dedup_rel_inds=None,
685
+ residnorm_decrease_threshold=16.0,
686
+ decrease_objective: Literal["norm", "normsq", "deconv"] = "deconv",
687
+ relative_peak_radius=5,
688
+ dedup_temporal_radius=7,
689
+ remove_exact_duplicates=True,
690
+ pos_dedup_temporal_radius=None,
691
+ dedup_batch_size=512,
692
+ singlechan_templates=None,
693
+ singlechan_threshold=None,
694
+ singlechan_trough_offset=None,
695
+ no_subtraction=False,
696
+ max_iter=100,
697
+ trough_priority=None,
698
+ growth_tolerance=None,
699
+ cumulant_order=None,
700
+ save_iteration=False,
701
+ save_residnorm_decrease=False,
702
+ ):
703
+ """Core peeling routine for subtraction"""
704
+ if no_subtraction:
705
+ threshold_res = threshold_chunk(
706
+ traces,
707
+ channel_index,
708
+ detection_threshold=detection_threshold,
709
+ peak_sign=peak_sign,
710
+ relative_peak_channel_index=relative_peak_channel_index,
711
+ spatial_dedup_channel_index=spatial_dedup_channel_index,
712
+ spatial_dedup_rel_inds=spatial_dedup_rel_inds,
713
+ trough_offset_samples=trough_offset_samples,
714
+ spike_length_samples=spike_length_samples,
715
+ left_margin=left_margin,
716
+ right_margin=right_margin,
717
+ relative_peak_radius=relative_peak_radius,
718
+ temporal_dedup_radius_samples=dedup_temporal_radius,
719
+ dedup_batch_size=dedup_batch_size,
720
+ remove_exact_duplicates=remove_exact_duplicates,
721
+ cumulant_order=cumulant_order,
722
+ convexity_threshold=convexity_threshold,
723
+ convexity_radius=convexity_radius,
724
+ max_spikes_per_chunk=None,
725
+ quiet=False,
726
+ )
727
+ waveforms, features = denoising_pipeline(
728
+ threshold_res["waveforms"], channels=threshold_res["channels"]
729
+ )
730
+ return ChunkSubtractionResult(
731
+ n_spikes=threshold_res["n_spikes"],
732
+ times_samples=threshold_res["times_rel"],
733
+ channels=threshold_res["channels"],
734
+ collisioncleaned_waveforms=waveforms,
735
+ residual=None,
736
+ features=features,
737
+ )
738
+
739
+ # validate arguments to avoid confusing error messages later
740
+ re_extract = extract_index is not None
741
+ if extract_index is None:
742
+ extract_index = channel_index
743
+ else:
744
+ assert extract_mask is not None
745
+ assert 0 <= left_margin < traces.shape[0]
746
+ assert 0 <= right_margin < traces.shape[0]
747
+ assert traces.shape[1] == channel_index.shape[0]
748
+ if spatial_dedup_channel_index is not None:
749
+ assert traces.shape[1] == spatial_dedup_channel_index.shape[0]
750
+
751
+ # can only subtract spikes with trough time >=trough_offset and <max_trough
752
+ post_trough_samples = spike_length_samples - trough_offset_samples
753
+ max_trough_time = traces.shape[0] - post_trough_samples
754
+
755
+ if growth_tolerance is not None:
756
+ gtol = traces.abs().add_(growth_tolerance)
757
+ else:
758
+ gtol = 0.0
759
+
760
+ # initialize residual, it needs to be padded to support
761
+ # our channel indexing convention. this copies the input.
762
+ residual = F.pad(traces, (0, 1), value=torch.nan)
763
+
764
+ subtracted_waveforms = []
765
+ spike_times = []
766
+ spike_channels = []
767
+ spike_features = []
768
+ detection_mask = torch.ones_like(residual)
769
+ dedup_temporal_ix = torch.arange(
770
+ -dedup_temporal_radius, dedup_temporal_radius + 1, device=residual.device
771
+ )
772
+ pos_dedup_temporal_ix = None
773
+ if pos_dedup_temporal_radius:
774
+ pos_dedup_temporal_ix = torch.arange(
775
+ -pos_dedup_temporal_radius,
776
+ pos_dedup_temporal_radius + 1,
777
+ device=residual.device,
778
+ )
779
+
780
+ for it in range(max_iter):
781
+ residual_det = residual[:, :-1]
782
+ if it and growth_tolerance is not None:
783
+ residual_det = residual_det.clamp(-gtol, gtol)
784
+
785
+ if singlechan_templates is None:
786
+ times_samples, channels = detect_and_deduplicate( # type: ignore
787
+ residual_det,
788
+ detection_threshold,
789
+ relative_peak_channel_index=relative_peak_channel_index,
790
+ dedup_channel_index=channel_index,
791
+ peak_sign=peak_sign,
792
+ relative_peak_radius=relative_peak_radius,
793
+ dedup_temporal_radius=spike_length_samples,
794
+ spatial_dedup_batch_size=dedup_batch_size,
795
+ remove_exact_duplicates=remove_exact_duplicates,
796
+ dedup_index_inds=subtract_rel_inds,
797
+ detection_mask=detection_mask[:, :-1] if it else None,
798
+ trough_priority=trough_priority,
799
+ cumulant_order=cumulant_order,
800
+ )
801
+ else:
802
+ times_samples, channels = singlechan_template_detect_and_deduplicate( # type: ignore
803
+ residual_det,
804
+ singlechan_templates,
805
+ threshold=singlechan_threshold, # type: ignore
806
+ trough_offset_samples=singlechan_trough_offset, # type: ignore
807
+ relative_peak_channel_index=relative_peak_channel_index,
808
+ dedup_channel_index=channel_index,
809
+ relative_peak_radius=relative_peak_radius,
810
+ dedup_temporal_radius=spike_length_samples,
811
+ detection_mask=detection_mask[:, :-1] if it else None,
812
+ )
813
+ if not times_samples.numel():
814
+ break
815
+
816
+ if it:
817
+ keep = detection_mask[times_samples, channels]
818
+ (keep,) = keep.nonzero(as_tuple=True)
819
+ times_samples = times_samples[keep]
820
+ channels = channels[keep]
821
+
822
+ if not times_samples.numel():
823
+ break
824
+
825
+ keep = convexity_filter(
826
+ residual,
827
+ times_samples,
828
+ channels,
829
+ threshold=convexity_threshold,
830
+ radius=convexity_radius,
831
+ )
832
+ times_samples = times_samples[keep]
833
+ channels = channels[keep]
834
+ if not times_samples.numel():
835
+ break
836
+
837
+ voltages = residual[times_samples, channels]
838
+
839
+ # never look at these again.
840
+ time_ix = times_samples.unsqueeze(1) + dedup_temporal_ix
841
+ time_ix = time_ix.clamp_(0, traces.shape[0] - 1)
842
+ if spatial_dedup_channel_index is not None:
843
+ chan_ix = spatial_dedup_channel_index[channels]
844
+ else:
845
+ chan_ix = channels.unsqueeze(1)
846
+ detection_mask[time_ix[:, :, None], chan_ix[:, None, :]] = 0.0
847
+
848
+ # take extra care to exclude positive peaks appearing near stronger troughs
849
+ if pos_dedup_temporal_radius:
850
+ assert pos_dedup_temporal_ix is not None
851
+ (neg,) = (voltages < 0).nonzero(as_tuple=True)
852
+ time_ix = times_samples[neg].unsqueeze(1) + pos_dedup_temporal_ix
853
+ time_ix = time_ix.clamp_(0, traces.shape[0] - 1)
854
+ if spatial_dedup_channel_index is not None:
855
+ chan_ix = spatial_dedup_channel_index[channels[neg]]
856
+ else:
857
+ chan_ix = channels[neg].unsqueeze(1)
858
+
859
+ pd_mask = torch.ones_like(detection_mask)
860
+ pd_mask[time_ix[:, :, None], chan_ix[:, None, :]] = 0
861
+ pd_mask = torch.logical_or(pd_mask, residual < 0)
862
+ detection_mask = torch.logical_and(detection_mask, pd_mask)
863
+
864
+ # throw away spikes which cannot be subtracted
865
+ keep = times_samples == times_samples.clamp(
866
+ trough_offset_samples, max_trough_time
867
+ )
868
+ (keep,) = keep.nonzero(as_tuple=True)
869
+
870
+ if not keep.numel():
871
+ break
872
+ times_samples = times_samples[keep]
873
+ channels = channels[keep]
874
+ voltages = voltages[keep]
875
+
876
+ # -- read waveforms, denoise, and test residnorm decrease
877
+ waveforms = grab_spikes(
878
+ residual,
879
+ times_samples,
880
+ channels,
881
+ channel_index,
882
+ trough_offset=trough_offset_samples,
883
+ spike_length_samples=spike_length_samples,
884
+ buffer=0,
885
+ already_padded=True,
886
+ )
887
+
888
+ if residnorm_decrease_threshold:
889
+ original_waveforms = waveforms.nan_to_num()
890
+ else:
891
+ original_waveforms = None
892
+
893
+ waveforms, features = denoising_pipeline(waveforms, channels=channels)
894
+
895
+ resid_keep, new_feats = check_residual_decrease(
896
+ original_waveforms,
897
+ waveforms,
898
+ decrease_objective=decrease_objective,
899
+ threshold=residnorm_decrease_threshold,
900
+ save_residnorm_decrease=save_residnorm_decrease,
901
+ )
902
+ features.update(new_feats)
903
+ if resid_keep is not None:
904
+ if not resid_keep.numel():
905
+ break
906
+ if resid_keep.numel() < len(original_waveforms):
907
+ waveforms = waveforms[resid_keep]
908
+ times_samples = times_samples[resid_keep]
909
+ channels = channels[resid_keep]
910
+ voltages = voltages[resid_keep]
911
+ for k, ft in features.items():
912
+ features[k] = ft[resid_keep]
913
+
914
+ # -- subtract in place
915
+ residual = subtract_spikes_(
916
+ residual,
917
+ times_samples,
918
+ channels,
919
+ channel_index,
920
+ waveforms,
921
+ trough_offset=trough_offset_samples,
922
+ buffer=0,
923
+ already_padded=True,
924
+ in_place=True,
925
+ )
926
+
927
+ # -- follow the nn's realignment advice, if requested
928
+ if realign_to_denoiser:
929
+ features["time_shifts"] = denoiser_time_shifts(
930
+ waveforms,
931
+ channels,
932
+ voltages,
933
+ subtract_rel_inds,
934
+ trough_offset_samples,
935
+ spike_length_samples,
936
+ peak_sign,
937
+ denoiser_realignment_shift,
938
+ denoiser_realignment_channel,
939
+ )
940
+
941
+ # -- store this iter's outputs
942
+ spike_times.append(times_samples)
943
+ spike_channels.append(channels)
944
+ spike_features.append(features)
945
+ if save_iteration:
946
+ spike_features[-1]["iteration"] = torch.full_like(times_samples, it)
947
+ subtracted_waveforms.append(waveforms)
948
+
949
+ # check if we got no spikes
950
+ if not spike_times:
951
+ return empty_chunk_subtraction_result(
952
+ spike_length_samples,
953
+ channel_index,
954
+ residual[left_margin : traces.shape[0] - right_margin, :-1],
955
+ )
956
+
957
+ # concatenate all of the thresholds together into single tensors
958
+ spike_times = torch.concatenate(spike_times)
959
+ spike_channels = torch.concatenate(spike_channels)
960
+ subtracted_waveforms = torch.concatenate(subtracted_waveforms)
961
+ spike_features = {
962
+ k: torch.concatenate([ff[k] for ff in spike_features])
963
+ for k in spike_features[0].keys()
964
+ }
965
+
966
+ # discard spikes in the margins and sort times_samples for caller
967
+ max_valid_t = traces.shape[0] - right_margin - 1
968
+ keep = spike_times == spike_times.clamp(left_margin, max_valid_t)
969
+ (keep,) = keep.nonzero(as_tuple=True)
970
+ if not keep.numel():
971
+ return empty_chunk_subtraction_result(
972
+ spike_length_samples,
973
+ channel_index,
974
+ residual[left_margin : traces.shape[0] - right_margin, :-1],
975
+ )
976
+
977
+ keep = keep[torch.argsort(spike_times[keep])]
978
+ subtracted_waveforms = subtracted_waveforms[keep]
979
+ spike_times = spike_times[keep]
980
+ spike_channels = spike_channels[keep]
981
+ for k in spike_features:
982
+ spike_features[k] = spike_features[k][keep]
983
+
984
+ # if extract_index != subtract_index, re-do the channels for the subtracted wfs
985
+ if re_extract:
986
+ subtracted_waveforms = get_relative_subset(
987
+ subtracted_waveforms, spike_channels, extract_mask
988
+ )
989
+
990
+ # construct collision-cleaned waveforms
991
+ collisioncleaned_waveforms = grab_spikes(
992
+ residual,
993
+ spike_times,
994
+ spike_channels,
995
+ extract_index,
996
+ trough_offset=trough_offset_samples,
997
+ spike_length_samples=spike_length_samples,
998
+ buffer=0,
999
+ already_padded=True,
1000
+ )
1001
+ collisioncleaned_waveforms += subtracted_waveforms
1002
+
1003
+ # offset spike times_samples according to margin
1004
+ spike_times -= left_margin
1005
+
1006
+ # apply time shifts
1007
+ if "time_shifts" in spike_features:
1008
+ spike_times += spike_features["time_shifts"]
1009
+
1010
+ # strip margin and padding channel off the residual
1011
+ residual = residual[left_margin : traces.shape[0] - right_margin, :-1].cpu()
1012
+
1013
+ return ChunkSubtractionResult(
1014
+ n_spikes=spike_times.numel(),
1015
+ times_samples=spike_times,
1016
+ channels=spike_channels,
1017
+ collisioncleaned_waveforms=collisioncleaned_waveforms,
1018
+ residual=residual,
1019
+ features=spike_features,
1020
+ )
1021
+
1022
+
1023
+ def empty_chunk_subtraction_result(spike_length_samples, channel_index, residual):
1024
+ empty_waveforms = torch.empty(
1025
+ (0, spike_length_samples, channel_index.shape[1]),
1026
+ dtype=residual.dtype,
1027
+ )
1028
+ empty_times_or_chans = torch.empty((0,), dtype=torch.long)
1029
+ return ChunkSubtractionResult(
1030
+ n_spikes=0,
1031
+ times_samples=empty_times_or_chans,
1032
+ channels=empty_times_or_chans,
1033
+ collisioncleaned_waveforms=empty_waveforms,
1034
+ residual=residual,
1035
+ features={},
1036
+ )
1037
+
1038
+
1039
+ # -- utilities
1040
+
1041
+
1042
+ def denoiser_time_shifts(
1043
+ waveforms,
1044
+ channels,
1045
+ voltages,
1046
+ subtract_rel_inds,
1047
+ trough_offset_samples,
1048
+ spike_length_samples,
1049
+ peak_sign,
1050
+ denoiser_realignment_shift,
1051
+ denoiser_realignment_channel,
1052
+ ):
1053
+ # extract main channel traces
1054
+ if denoiser_realignment_channel == "detection":
1055
+ assert subtract_rel_inds is not None
1056
+ main_channel_rel_inds = subtract_rel_inds[channels]
1057
+ elif denoiser_realignment_channel == "denoised":
1058
+ main_channel_rel_inds = ptp(waveforms).nan_to_num_(nan=-torch.inf).argmax(dim=1)
1059
+ else:
1060
+ assert False
1061
+ denoised_main_channel_traces = waveforms.take_along_dim(
1062
+ dim=2, indices=main_channel_rel_inds[:, None, None]
1063
+ )
1064
+ nwf = len(waveforms)
1065
+ assert denoised_main_channel_traces.shape == (nwf, spike_length_samples, 1)
1066
+ denoised_main_channel_traces = denoised_main_channel_traces[:, :, 0]
1067
+
1068
+ # extract window around trough
1069
+ start = trough_offset_samples - denoiser_realignment_shift
1070
+ end = trough_offset_samples + denoiser_realignment_shift + 1
1071
+ snips = denoised_main_channel_traces[:, start:end]
1072
+ assert snips.shape == (nwf, 2 * denoiser_realignment_shift + 1)
1073
+
1074
+ # handle the sign of the events so that we can align to maxima
1075
+ if peak_sign == "both":
1076
+ snips.mul_(torch.sign(voltages)[:, None])
1077
+ elif peak_sign == "neg":
1078
+ snips.neg_()
1079
+ else:
1080
+ assert peak_sign == "pos"
1081
+
1082
+ # find shifts just by argmax
1083
+ peaks = snips.argmax(dim=1)
1084
+ dt = peaks.sub_(denoiser_realignment_shift).to(dtype=torch.int16)
1085
+ return dt
1086
+
1087
+
1088
+ def check_residual_decrease(
1089
+ orig_wfs,
1090
+ dn_wfs,
1091
+ decrease_objective="deconv",
1092
+ threshold=10.0,
1093
+ save_residnorm_decrease=False,
1094
+ ):
1095
+ if not threshold:
1096
+ return None, {}
1097
+
1098
+ if decrease_objective == "deconv":
1099
+ dn_wfs = dn_wfs.nan_to_num()
1100
+ conv = (orig_wfs * dn_wfs).sum(dim=(1, 2))
1101
+ norm = dn_wfs.square_().sum(dim=(1, 2))
1102
+ reduction = conv.mul_(2.0).sub_(norm)
1103
+ threshold = threshold ** 2
1104
+ elif decrease_objective in ("norm", "normsq"):
1105
+ orig_decobj = orig_wfs.square().sum(dim=(1, 2))
1106
+ orig_wfs = orig_wfs.sub_(dn_wfs).nan_to_num_()
1107
+ new_decobj = orig_wfs.square_().sum(dim=(1, 2))
1108
+ if decrease_objective == "norm":
1109
+ orig_decobj = orig_decobj.sqrt_()
1110
+ new_decobj = new_decobj.sqrt_()
1111
+ else:
1112
+ threshold = threshold ** 2
1113
+ reduction = orig_decobj - new_decobj
1114
+ else:
1115
+ assert False
1116
+
1117
+ keep = threshold < reduction
1118
+ (keep,) = keep.nonzero(as_tuple=True)
1119
+ if save_residnorm_decrease:
1120
+ features = dict(residnorm_decreases=reduction)
1121
+ else:
1122
+ features = {}
1123
+ return keep, features