DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,471 @@
1
+ import warnings
2
+ from typing import Literal
3
+
4
+ import numpy as np
5
+ import torch
6
+ from spikeinterface.core import BaseRecording
7
+
8
+ from ..detect import convexity_filter, detect_and_deduplicate
9
+ from ..transform import WaveformPipeline
10
+ from ..util import spiketorch
11
+ from ..util.data_util import SpikeDataset
12
+ from ..util.internal_config import (
13
+ FeaturizationConfig,
14
+ FitSamplingConfig,
15
+ ThresholdingConfig,
16
+ WaveformConfig,
17
+ )
18
+ from ..util.waveform_util import get_channel_index_rel_inds, make_channel_index
19
+ from .peel_base import BasePeeler, PeelingBatchResult, peeling_empty_result
20
+
21
+
22
+ class ThresholdAndFeaturize(BasePeeler):
23
+ def __init__(
24
+ self,
25
+ recording,
26
+ channel_index,
27
+ featurization_pipeline=None,
28
+ trough_offset_samples=42,
29
+ spike_length_samples=121,
30
+ detection_threshold=5.0,
31
+ chunk_length_samples=30_000,
32
+ max_spikes_per_chunk=None,
33
+ peak_sign="both",
34
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
35
+ fit_max_reweighting=4.0,
36
+ relative_peak_channel_index=None,
37
+ spatial_dedup_channel_index=None,
38
+ relative_peak_radius_samples=5,
39
+ temporal_dedup_radius_samples=7,
40
+ cumulant_order=None,
41
+ convexity_threshold=None,
42
+ convexity_radius=3,
43
+ remove_exact_duplicates=True,
44
+ n_seconds_fit=100,
45
+ n_waveforms_fit=20_000,
46
+ max_waveforms_fit=50_000,
47
+ fit_subsampling_random_state=0,
48
+ thinning=0.0,
49
+ time_jitter=0,
50
+ trough_priority=None,
51
+ spatial_jitter_channel_index=None,
52
+ dtype=torch.float,
53
+ ):
54
+ super().__init__(
55
+ recording=recording,
56
+ channel_index=channel_index,
57
+ featurization_pipeline=featurization_pipeline,
58
+ chunk_length_samples=chunk_length_samples,
59
+ chunk_margin_samples=self.next_margin(spike_length_samples, factor=5),
60
+ n_seconds_fit=n_seconds_fit,
61
+ n_waveforms_fit=n_waveforms_fit,
62
+ max_waveforms_fit=max_waveforms_fit,
63
+ fit_sampling=fit_sampling,
64
+ fit_subsampling_random_state=fit_subsampling_random_state,
65
+ fit_max_reweighting=fit_max_reweighting,
66
+ dtype=dtype,
67
+ trough_offset_samples=trough_offset_samples,
68
+ spike_length_samples=spike_length_samples,
69
+ )
70
+
71
+ self.relative_peak_radius_samples = relative_peak_radius_samples
72
+ self.temporal_dedup_radius_samples = temporal_dedup_radius_samples
73
+ self.remove_exact_duplicates = remove_exact_duplicates
74
+ self.peak_sign = peak_sign
75
+ self.trough_priority = trough_priority
76
+ self.convexity_threshold = convexity_threshold
77
+ self.convexity_radius = convexity_radius
78
+ self.cumulant_order = cumulant_order
79
+ self.dedup_batch_size = self.nearest_batch_length()
80
+ if spatial_dedup_channel_index is not None:
81
+ self.register_buffer(
82
+ "spatial_dedup_channel_index", spatial_dedup_channel_index
83
+ )
84
+ self.register_buffer(
85
+ "spatial_dedup_rel_inds",
86
+ get_channel_index_rel_inds(spatial_dedup_channel_index),
87
+ )
88
+ else:
89
+ self.spatial_dedup_channel_index = None
90
+ if relative_peak_channel_index is not None:
91
+ self.register_buffer(
92
+ "relative_peak_channel_index", relative_peak_channel_index
93
+ )
94
+ else:
95
+ self.relative_peak_channel_index = None
96
+ self.detection_threshold = detection_threshold
97
+ self.max_spikes_per_chunk = max_spikes_per_chunk
98
+ self.peel_kind = f"Threshold {detection_threshold}"
99
+
100
+ # parameters for random perturbation of detections
101
+ if thinning:
102
+ assert thinning < min(
103
+ trough_offset_samples, spike_length_samples - trough_offset_samples
104
+ )
105
+ self.thinning = thinning
106
+ self.time_jitter = time_jitter
107
+ if spatial_jitter_channel_index is not None:
108
+ self.register_buffer(
109
+ "spatial_jitter_channel_index", spatial_jitter_channel_index
110
+ )
111
+ else:
112
+ self.spatial_jitter_channel_index = None
113
+ self.is_random = (
114
+ thinning or time_jitter or spatial_jitter_channel_index is not None
115
+ )
116
+
117
+ @classmethod
118
+ def from_config(
119
+ cls,
120
+ *,
121
+ recording: BaseRecording,
122
+ waveform_cfg: WaveformConfig,
123
+ thresholding_cfg: ThresholdingConfig,
124
+ featurization_cfg: FeaturizationConfig,
125
+ sampling_cfg: FitSamplingConfig,
126
+ ):
127
+ geom = torch.tensor(recording.get_channel_locations())
128
+ channel_index = make_channel_index(
129
+ geom, featurization_cfg.extract_radius, to_torch=True
130
+ )
131
+
132
+ featurization_pipeline = WaveformPipeline.from_config(
133
+ geom=geom,
134
+ channel_index=channel_index,
135
+ featurization_cfg=featurization_cfg,
136
+ waveform_cfg=waveform_cfg,
137
+ sampling_frequency=recording.sampling_frequency,
138
+ )
139
+
140
+ if thresholding_cfg.relative_peak_radius_um:
141
+ relative_peak_channel_index = make_channel_index(
142
+ geom, thresholding_cfg.relative_peak_radius_um, to_torch=True
143
+ )
144
+ else:
145
+ relative_peak_channel_index = None
146
+
147
+ if thresholding_cfg.spatial_dedup_radius:
148
+ spatial_dedup_channel_index = make_channel_index(
149
+ geom, thresholding_cfg.spatial_dedup_radius, to_torch=True
150
+ )
151
+ else:
152
+ spatial_dedup_channel_index = None
153
+
154
+ if thresholding_cfg.spatial_jitter_radius:
155
+ spatial_jitter_channel_index = make_channel_index(
156
+ geom, thresholding_cfg.spatial_jitter_radius, to_torch=True
157
+ )
158
+ else:
159
+ spatial_jitter_channel_index = None
160
+
161
+ # waveform logic
162
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
163
+ recording.sampling_frequency
164
+ )
165
+ spike_length_samples = waveform_cfg.spike_length_samples(
166
+ recording.sampling_frequency
167
+ )
168
+
169
+ return cls(
170
+ recording,
171
+ channel_index,
172
+ featurization_pipeline=featurization_pipeline,
173
+ trough_offset_samples=trough_offset_samples,
174
+ spike_length_samples=spike_length_samples,
175
+ detection_threshold=thresholding_cfg.detection_threshold,
176
+ chunk_length_samples=thresholding_cfg.chunk_length_samples,
177
+ max_spikes_per_chunk=thresholding_cfg.max_spikes_per_chunk,
178
+ peak_sign=thresholding_cfg.peak_sign,
179
+ relative_peak_channel_index=relative_peak_channel_index,
180
+ spatial_dedup_channel_index=spatial_dedup_channel_index,
181
+ relative_peak_radius_samples=thresholding_cfg.relative_peak_radius_samples,
182
+ temporal_dedup_radius_samples=thresholding_cfg.temporal_dedup_radius_samples,
183
+ remove_exact_duplicates=thresholding_cfg.remove_exact_duplicates,
184
+ cumulant_order=thresholding_cfg.cumulant_order,
185
+ convexity_threshold=thresholding_cfg.convexity_threshold,
186
+ convexity_radius=thresholding_cfg.convexity_radius,
187
+ n_seconds_fit=thresholding_cfg.n_seconds_fit,
188
+ fit_sampling=sampling_cfg.fit_sampling,
189
+ fit_max_reweighting=sampling_cfg.fit_max_reweighting,
190
+ n_waveforms_fit=sampling_cfg.n_waveforms_fit,
191
+ max_waveforms_fit=sampling_cfg.max_waveforms_fit,
192
+ fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
193
+ thinning=thresholding_cfg.thinning,
194
+ time_jitter=thresholding_cfg.time_jitter,
195
+ spatial_jitter_channel_index=spatial_jitter_channel_index,
196
+ trough_priority=thresholding_cfg.trough_priority,
197
+ )
198
+
199
+ def out_datasets(self):
200
+ datasets = super().out_datasets()
201
+ if self.is_random:
202
+ datasets.append(
203
+ SpikeDataset(name="orig_times_samples", shape_per_spike=(), dtype=float)
204
+ )
205
+ datasets.append(
206
+ SpikeDataset(name="orig_channels", shape_per_spike=(), dtype=float)
207
+ )
208
+ datasets.append(SpikeDataset(name="voltages", shape_per_spike=(), dtype=float))
209
+ return datasets
210
+
211
+ def peel_chunk(
212
+ self,
213
+ traces,
214
+ *,
215
+ chunk_start_samples=0,
216
+ left_margin=0,
217
+ right_margin=0,
218
+ return_residual=False,
219
+ return_waveforms=True,
220
+ ) -> PeelingBatchResult:
221
+ threshold_res = threshold_chunk(
222
+ traces,
223
+ self.channel_index,
224
+ detection_threshold=self.detection_threshold,
225
+ peak_sign=self.peak_sign,
226
+ spatial_dedup_channel_index=self.spatial_dedup_channel_index,
227
+ spatial_dedup_rel_inds=self.spatial_dedup_rel_inds,
228
+ dedup_batch_size=self.dedup_batch_size,
229
+ trough_offset_samples=self.trough_offset_samples,
230
+ spike_length_samples=self.spike_length_samples,
231
+ left_margin=left_margin,
232
+ right_margin=right_margin,
233
+ relative_peak_channel_index=self.relative_peak_channel_index,
234
+ temporal_dedup_radius_samples=self.temporal_dedup_radius_samples,
235
+ remove_exact_duplicates=self.remove_exact_duplicates,
236
+ cumulant_order=self.cumulant_order,
237
+ convexity_threshold=self.convexity_threshold,
238
+ convexity_radius=self.convexity_radius,
239
+ thinning=self.thinning,
240
+ time_jitter=self.time_jitter,
241
+ spatial_jitter_channel_index=self.spatial_jitter_channel_index,
242
+ rg=self.rg if self.is_random else None,
243
+ max_spikes_per_chunk=None,
244
+ return_waveforms=return_waveforms,
245
+ trough_priority=self.trough_priority,
246
+ quiet=False,
247
+ )
248
+ if not threshold_res["n_spikes"]:
249
+ return peeling_empty_result
250
+
251
+ # get absolute times
252
+ times_rel = threshold_res["times_rel"]
253
+ assert times_rel is not None
254
+ times_samples = times_rel + chunk_start_samples
255
+
256
+ peel_result = PeelingBatchResult(
257
+ n_spikes=threshold_res["n_spikes"],
258
+ times_samples=times_samples,
259
+ channels=threshold_res["channels"],
260
+ voltages=threshold_res["voltages"],
261
+ )
262
+ if return_waveforms:
263
+ assert "waveforms" in threshold_res
264
+ peel_result["collisioncleaned_waveforms"] = threshold_res["waveforms"]
265
+ if self.is_random:
266
+ orig_times_rel = threshold_res["orig_times_rel"]
267
+ assert orig_times_rel is not None
268
+ peel_result["orig_times_samples"] = orig_times_rel + chunk_start_samples
269
+ peel_result["orig_channels"] = threshold_res["orig_channels"]
270
+ if return_residual:
271
+ # note, this is same as the input.
272
+ peel_result["residual"] = traces[
273
+ left_margin : traces.shape[0] - right_margin
274
+ ]
275
+ return peel_result
276
+
277
+
278
+ def threshold_chunk(
279
+ traces,
280
+ channel_index,
281
+ detection_threshold=4.0,
282
+ peak_sign="both",
283
+ relative_peak_channel_index=None,
284
+ spatial_dedup_channel_index=None,
285
+ spatial_dedup_rel_inds=None,
286
+ dedup_batch_size=512,
287
+ trough_offset_samples=42,
288
+ spike_length_samples=121,
289
+ left_margin=0,
290
+ right_margin=0,
291
+ relative_peak_radius=5,
292
+ temporal_dedup_radius_samples=7,
293
+ remove_exact_duplicates=True,
294
+ max_spikes_per_chunk=None,
295
+ thinning=0.0,
296
+ time_jitter=0,
297
+ trough_priority=None,
298
+ spatial_jitter_channel_index=None,
299
+ cumulant_order=None,
300
+ convexity_threshold=None,
301
+ convexity_radius=3,
302
+ return_waveforms=True,
303
+ rg=None,
304
+ quiet=False,
305
+ ) -> PeelingBatchResult:
306
+ n_index = channel_index.shape[1]
307
+ times_rel, channels, energies = detect_and_deduplicate( # type: ignore
308
+ traces,
309
+ detection_threshold,
310
+ relative_peak_channel_index=relative_peak_channel_index,
311
+ dedup_channel_index=spatial_dedup_channel_index,
312
+ dedup_index_inds=spatial_dedup_rel_inds,
313
+ spatial_dedup_batch_size=dedup_batch_size,
314
+ peak_sign=peak_sign,
315
+ dedup_temporal_radius=temporal_dedup_radius_samples,
316
+ remove_exact_duplicates=remove_exact_duplicates,
317
+ relative_peak_radius=relative_peak_radius,
318
+ return_energies=True,
319
+ trough_priority=trough_priority,
320
+ cumulant_order=cumulant_order,
321
+ )
322
+ if not times_rel.numel():
323
+ return PeelingBatchResult(
324
+ n_spikes=0,
325
+ orig_times_rel=times_rel,
326
+ times_rel=times_rel,
327
+ orig_channels=channels,
328
+ channels=channels,
329
+ voltages=energies,
330
+ waveforms=energies.view(-1, spike_length_samples, n_index),
331
+ )
332
+ keep = convexity_filter(
333
+ traces,
334
+ times_rel,
335
+ channels,
336
+ threshold=convexity_threshold,
337
+ radius=convexity_radius,
338
+ )
339
+ times_rel = times_rel[keep]
340
+ channels = channels[keep]
341
+ energies = energies[keep]
342
+
343
+ orig_times_rel = times_rel
344
+ orig_channels = channels
345
+ keep, times_rel, channels = perturb_detections(
346
+ times_rel,
347
+ channels,
348
+ thinning=thinning,
349
+ time_jitter=time_jitter,
350
+ spatial_jitter_channel_index=spatial_jitter_channel_index,
351
+ rg=rg,
352
+ )
353
+ orig_times_rel = orig_times_rel[keep]
354
+ orig_channels = orig_channels[keep]
355
+ energies = energies[keep]
356
+
357
+ # want only peaks in the chunk
358
+ min_time = left_margin + trough_offset_samples
359
+ tail_samples = spike_length_samples - trough_offset_samples
360
+ max_time = traces.shape[0] - right_margin - tail_samples - 1
361
+ valid = times_rel == times_rel.clamp(min_time, max_time)
362
+ (valid,) = valid.nonzero(as_tuple=True)
363
+ orig_times_rel = orig_times_rel[valid]
364
+ times_rel = times_rel[valid]
365
+ channels = channels[valid]
366
+ orig_channels = orig_channels[valid]
367
+ voltages = traces[orig_times_rel, orig_channels]
368
+ n_detect = times_rel.numel()
369
+ if not n_detect:
370
+ return PeelingBatchResult(
371
+ n_spikes=0,
372
+ times_rel=times_rel,
373
+ channels=channels,
374
+ voltages=energies,
375
+ orig_times_rel=orig_times_rel,
376
+ orig_channels=orig_channels,
377
+ )
378
+
379
+ if max_spikes_per_chunk is not None:
380
+ if n_detect > max_spikes_per_chunk and not quiet:
381
+ warnings.warn(
382
+ f"{n_detect} spikes in chunk was larger than "
383
+ f"{max_spikes_per_chunk=}. Keeping the top ones."
384
+ )
385
+ energies = energies[valid]
386
+ best = torch.argsort(energies)[-max_spikes_per_chunk:]
387
+ best = best.sort().values
388
+ del energies
389
+
390
+ times_rel = times_rel[best]
391
+ channels = channels[best]
392
+ voltages = voltages[best]
393
+ orig_channels = orig_channels[best]
394
+ orig_times_rel = orig_times_rel[best]
395
+
396
+ # load up the waveforms for this chunk
397
+ if return_waveforms:
398
+ waveforms = spiketorch.grab_spikes(
399
+ traces,
400
+ times_rel,
401
+ channels,
402
+ channel_index,
403
+ trough_offset=trough_offset_samples,
404
+ spike_length_samples=spike_length_samples,
405
+ already_padded=False,
406
+ pad_value=torch.nan,
407
+ )
408
+ else:
409
+ waveforms = None
410
+
411
+ # offset times for caller
412
+ orig_times_rel -= left_margin
413
+ times_rel -= left_margin
414
+
415
+ res = PeelingBatchResult(
416
+ n_spikes=times_rel.numel(),
417
+ orig_times_rel=orig_times_rel,
418
+ orig_channels=orig_channels,
419
+ times_rel=times_rel,
420
+ channels=channels,
421
+ voltages=voltages,
422
+ )
423
+ if waveforms is not None:
424
+ res["waveforms"] = waveforms
425
+ return res
426
+
427
+
428
+ def perturb_detections(
429
+ times_rel,
430
+ channels,
431
+ thinning: float = 0,
432
+ time_jitter=0,
433
+ spatial_jitter_channel_index=None,
434
+ rg: np.random.Generator | None = None,
435
+ ):
436
+ keep = slice(None)
437
+ if not (thinning or time_jitter or spatial_jitter_channel_index is not None):
438
+ return keep, times_rel, channels
439
+
440
+ n = len(times_rel)
441
+ if not n:
442
+ return keep, times_rel, channels
443
+
444
+ if thinning:
445
+ assert 0 <= thinning <= 1
446
+ assert rg is not None
447
+ keep = rg.binomial(n=1, p=1.0 - thinning, size=n)
448
+ keep = torch.from_numpy(np.flatnonzero(keep))
449
+ keep = keep.to(times_rel)
450
+
451
+ times_rel = times_rel[keep]
452
+ channels = channels[keep]
453
+
454
+ n = len(times_rel)
455
+ if time_jitter:
456
+ assert rg is not None
457
+ jitter = rg.integers(low=-time_jitter, high=time_jitter + 1)
458
+ times_rel = times_rel + torch.asarray(
459
+ jitter, dtype=times_rel.dtype, device=times_rel.device
460
+ )
461
+
462
+ if spatial_jitter_channel_index is not None:
463
+ assert rg is not None
464
+ n_channels = len(spatial_jitter_channel_index)
465
+ n_valid = (spatial_jitter_channel_index < n_channels).sum(1)
466
+ n_valid = n_valid[channels].cpu()
467
+ rel_ix = rg.integers(0, high=n_valid)
468
+ rel_ix = torch.from_numpy(rel_ix).to(channels)
469
+ channels = spatial_jitter_channel_index[channels, rel_ix]
470
+
471
+ return keep, times_rel, channels
@@ -0,0 +1,191 @@
1
+ from typing import Literal
2
+ import numpy as np
3
+ import torch
4
+
5
+ from ..templates.template_util import LowRankTemplates, compressed_upsampled_templates
6
+ from ..transform import WaveformPipeline
7
+ from ..util import universal_util, waveform_util
8
+ from ..util.internal_config import (
9
+ UniversalMatchingConfig,
10
+ FitSamplingConfig,
11
+ FeaturizationConfig,
12
+ WaveformConfig,
13
+ )
14
+ from .matching import ObjectiveUpdateTemplateMatchingPeeler
15
+ from .matching_util.compressed_upsampled import CompressedUpsampledMatchingTemplates
16
+ from .matching_util.pairwise import SeparablePairwiseConv
17
+
18
+
19
+ class UniversalTemplatesMatchingPeeler(ObjectiveUpdateTemplateMatchingPeeler):
20
+ """KS-style universal-templates-from-data detection
21
+
22
+ This tries to rephrase their algorithm as faithfully as possible
23
+ using dartsort tools, for comparison purposes with our algorithms.
24
+
25
+ The idea is to estimate some (they use 6, it turns out) single-channel
26
+ shapes via K means applied to single-channel waveforms. These
27
+ are then expanded out into a full template library by spatial
28
+ convs with various Gaussians. Then, throw them into the matcher.
29
+ Since KS' matcher has scale_std --> infty, we can put a large
30
+ scale prior variance to match the spirit of the thing.
31
+
32
+ TODO maybe I should implement scale_prior->infty in our matcher?
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ recording,
38
+ channel_index,
39
+ featurization_pipeline,
40
+ threshold=100.0,
41
+ trough_offset_samples=42,
42
+ spike_length_samples=121,
43
+ amplitude_scaling_variance=100.0,
44
+ amplitude_scaling_boundary=500.0,
45
+ detection_threshold=6.0,
46
+ alignment_padding=20,
47
+ n_centroids=6,
48
+ pca_rank=8,
49
+ taper=True,
50
+ n_sigmas=5,
51
+ min_template_size=10.0,
52
+ max_distance=32.0,
53
+ dx=32.0,
54
+ chunk_length_samples=1000,
55
+ n_seconds_fit=40,
56
+ max_waveforms_fit=50_000,
57
+ n_waveforms_fit=20_000,
58
+ fit_subsampling_random_state=0,
59
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
60
+ fit_max_reweighting=4.0,
61
+ refractory_radius_frames=10,
62
+ max_iter=1000,
63
+ dtype=torch.float,
64
+ ):
65
+ shapes, footprints, template_data = (
66
+ universal_util.universal_templates_from_data(
67
+ rec=recording,
68
+ detection_threshold=detection_threshold,
69
+ trough_offset_samples=trough_offset_samples,
70
+ spike_length_samples=spike_length_samples,
71
+ alignment_padding=alignment_padding,
72
+ n_centroids=n_centroids,
73
+ pca_rank=pca_rank,
74
+ n_waveforms_fit=n_waveforms_fit,
75
+ taper=taper,
76
+ taper_start=alignment_padding // 2,
77
+ taper_end=alignment_padding // 2,
78
+ random_seed=fit_subsampling_random_state,
79
+ n_sigmas=n_sigmas,
80
+ min_template_size=min_template_size,
81
+ max_distance=max_distance,
82
+ dx=dx,
83
+ # let's not worry about exposing these
84
+ deduplication_radius=150.0,
85
+ kmeanspp_initial="random",
86
+ )
87
+ )
88
+
89
+ # TODO: this should all be put in a matchingtemplate+builder
90
+ # so that it is lazy and resuming works better.
91
+ Nf = len(footprints)
92
+ Ns = len(shapes)
93
+ shapes_ixd = torch.asarray(shapes)[None]
94
+ shapes_ixd = shapes_ixd.broadcast_to((Nf, Ns, *shapes.shape[1:]))
95
+ shapes_ixd = shapes_ixd.reshape(Nf * Ns, *shapes.shape[1:], 1)
96
+ footprints_ixd = torch.asarray(footprints)[:, None]
97
+ footprints_ixd = footprints_ixd.broadcast_to((Nf, Ns, *footprints.shape[1:]))
98
+ footprints_ixd = footprints_ixd.reshape(Nf * Ns, 1, *footprints.shape[1:])
99
+ low_rank_templates = LowRankTemplates(
100
+ unit_ids=np.arange(Nf * Ns),
101
+ temporal_components=shapes_ixd.numpy(),
102
+ singular_values=shapes_ixd.new_ones(Nf * Ns, 1).numpy(),
103
+ spatial_components=footprints_ixd.numpy(),
104
+ spike_counts_by_channel=np.broadcast_to(
105
+ np.atleast_2d([100]), (Nf * Ns, footprints.shape[1])
106
+ ),
107
+ )
108
+ pairwise_conv_db = SeparablePairwiseConv(footprints, shapes)
109
+
110
+ cupt = compressed_upsampled_templates(
111
+ low_rank_templates.temporal_components, max_upsample=1
112
+ )
113
+
114
+ # TODO: re-using the individually compressed basis algorithm here
115
+ # which is general enough that it works, but for this separable
116
+ # basis there is a faster algorithm.
117
+ matching_templates = CompressedUpsampledMatchingTemplates(
118
+ trough_offset_samples=trough_offset_samples,
119
+ low_rank_templates=low_rank_templates,
120
+ pconv_db=pairwise_conv_db,
121
+ compressed_upsampled_temporal=cupt,
122
+ )
123
+
124
+ super().__init__(
125
+ recording=recording,
126
+ matching_templates=matching_templates,
127
+ channel_index=channel_index,
128
+ featurization_pipeline=featurization_pipeline,
129
+ threshold=threshold,
130
+ amplitude_scaling_variance=amplitude_scaling_variance,
131
+ amplitude_scaling_boundary=amplitude_scaling_boundary,
132
+ trough_offset_samples=trough_offset_samples,
133
+ chunk_length_samples=chunk_length_samples,
134
+ n_seconds_fit=n_seconds_fit,
135
+ max_waveforms_fit=max_waveforms_fit,
136
+ n_waveforms_fit=n_waveforms_fit,
137
+ fit_subsampling_random_state=fit_subsampling_random_state,
138
+ fit_sampling=fit_sampling,
139
+ fit_max_reweighting=fit_max_reweighting,
140
+ dtype=dtype,
141
+ refractory_radius_frames=refractory_radius_frames,
142
+ max_iter=max_iter,
143
+ )
144
+
145
+ @classmethod
146
+ def from_config( # type: ignore[reportIncompatibleOverride]
147
+ cls,
148
+ recording,
149
+ *,
150
+ universal_cfg: UniversalMatchingConfig,
151
+ featurization_cfg: FeaturizationConfig,
152
+ waveform_cfg: WaveformConfig,
153
+ sampling_cfg: FitSamplingConfig,
154
+ ):
155
+ geom = torch.tensor(recording.get_channel_locations())
156
+ channel_index = waveform_util.make_channel_index(
157
+ geom, featurization_cfg.extract_radius, to_torch=True
158
+ )
159
+ featurization_pipeline = WaveformPipeline.from_config(
160
+ geom=geom,
161
+ channel_index=channel_index,
162
+ featurization_cfg=featurization_cfg,
163
+ waveform_cfg=waveform_cfg,
164
+ sampling_frequency=recording.sampling_frequency,
165
+ )
166
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
167
+ recording.sampling_frequency
168
+ )
169
+ spike_length_samples = waveform_cfg.spike_length_samples(
170
+ recording.sampling_frequency
171
+ )
172
+ fs_ms = recording.sampling_frequency / 1000
173
+ alignment_padding = int(universal_cfg.alignment_padding_ms * fs_ms)
174
+ return cls(
175
+ recording,
176
+ chunk_length_samples=universal_cfg.chunk_length_samples,
177
+ n_seconds_fit=universal_cfg.n_seconds_fit,
178
+ n_waveforms_fit=sampling_cfg.n_waveforms_fit,
179
+ max_waveforms_fit=sampling_cfg.max_waveforms_fit,
180
+ fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
181
+ fit_sampling=sampling_cfg.fit_sampling,
182
+ fit_max_reweighting=sampling_cfg.fit_max_reweighting,
183
+ threshold=universal_cfg.threshold,
184
+ detection_threshold=universal_cfg.detection_threshold,
185
+ n_sigmas=universal_cfg.n_sigmas,
186
+ channel_index=channel_index,
187
+ alignment_padding=alignment_padding,
188
+ featurization_pipeline=featurization_pipeline,
189
+ trough_offset_samples=trough_offset_samples,
190
+ spike_length_samples=spike_length_samples,
191
+ )
Binary file
Binary file
Binary file
@@ -0,0 +1,11 @@
1
+ from .templates import TemplateData
2
+ from .template_util import (
3
+ svd_compress_templates,
4
+ compressed_upsampled_templates,
5
+ CompressedUpsampledTemplates,
6
+ LowRankTemplates,
7
+ templates_at_time,
8
+ )
9
+ from ..peel.matching_util.pairwise import CompressedPairwiseConv, SeparablePairwiseConv
10
+ from .realignment import realign, estimate_offset
11
+ from .postprocess_util import estimate_template_library