DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,628 @@
1
+ """A simple residual updating template matcher."""
2
+
3
+ from pathlib import Path
4
+ from typing import Literal, Self
5
+
6
+ import numpy as np
7
+ import torch
8
+ import torch.nn.functional as F
9
+ from scipy.stats import norm
10
+ from spikeinterface.core import BaseRecording
11
+ from torch import Tensor
12
+
13
+ from ..templates import LowRankTemplates, TemplateData
14
+ from ..transform import WaveformPipeline
15
+ from ..util.data_util import SpikeDataset
16
+ from ..util.internal_config import (
17
+ ComputationConfig,
18
+ FeaturizationConfig,
19
+ FitSamplingConfig,
20
+ MatchingConfig,
21
+ WaveformConfig,
22
+ )
23
+ from ..util.logging_util import get_logger
24
+ from ..util.noise_util import whitener_from_hdf5
25
+ from ..util.waveform_util import make_channel_index
26
+ from .matching_util import (
27
+ ChunkTemplateData,
28
+ MatchingPeaks,
29
+ MatchingTemplates,
30
+ MatchingTemplatesBuilder,
31
+ )
32
+ from .peel_base import BasePeeler, PeelingBatchResult
33
+
34
+ logger = get_logger(__name__)
35
+
36
+
37
+ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
38
+ peel_kind = "TemplateMatching"
39
+
40
+ def __init__(
41
+ self,
42
+ recording,
43
+ channel_index,
44
+ featurization_pipeline,
45
+ matching_templates: MatchingTemplates | None = None,
46
+ matching_templates_builder: MatchingTemplatesBuilder | None = None,
47
+ refractory_radius_frames=10,
48
+ amplitude_scaling_variance=0.0,
49
+ amplitude_scaling_boundary=0.5,
50
+ margin_factor=2,
51
+ up_factor: int = 1,
52
+ trough_offset_samples=42,
53
+ threshold: float | Literal["fp_control"] = 100.0,
54
+ obj_spike_counts=None,
55
+ max_fp_per_input_spike=2.5,
56
+ chunk_length_samples=30_000,
57
+ n_seconds_fit=40,
58
+ max_waveforms_fit=50_000,
59
+ n_waveforms_fit=20_000,
60
+ fit_max_reweighting=4.0,
61
+ channel_selection: Literal["template", "amplitude"] = "template",
62
+ channel_selection_index: Tensor | None = None,
63
+ fit_subsampling_random_state=0,
64
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
65
+ max_iter=1000,
66
+ max_spikes_per_second=16384,
67
+ cd_iter=0,
68
+ coarse_cd=True,
69
+ parent_sorting_hdf5_path: str | Path | None = None,
70
+ dtype=torch.float,
71
+ ):
72
+ if matching_templates is not None:
73
+ spike_length_samples = matching_templates.spike_length_samples
74
+ elif matching_templates_builder is not None:
75
+ spike_length_samples = matching_templates_builder.spike_length_samples
76
+ else:
77
+ raise ValueError(f"Need either a MatchingTemplates or a builder.")
78
+
79
+ super().__init__(
80
+ recording=recording,
81
+ channel_index=channel_index,
82
+ featurization_pipeline=featurization_pipeline,
83
+ chunk_length_samples=chunk_length_samples,
84
+ chunk_margin_samples=margin_factor * spike_length_samples,
85
+ n_seconds_fit=n_seconds_fit,
86
+ max_waveforms_fit=max_waveforms_fit,
87
+ fit_subsampling_random_state=fit_subsampling_random_state,
88
+ fit_sampling=fit_sampling,
89
+ n_waveforms_fit=n_waveforms_fit,
90
+ fit_max_reweighting=fit_max_reweighting,
91
+ trough_offset_samples=trough_offset_samples,
92
+ spike_length_samples=spike_length_samples,
93
+ dtype=dtype,
94
+ )
95
+
96
+ self.matching_templates = matching_templates
97
+ self.matching_templates_builder = matching_templates_builder
98
+ self.threshold = threshold
99
+ self.thresholdsq = None # set in precompute. usually just ^ squared.
100
+ self.refractory_radius_frames = refractory_radius_frames
101
+ self.max_iter = max_iter
102
+ self.cd_iter = cd_iter
103
+ self.coarse_cd = coarse_cd
104
+ self.max_spikes_per_second = max_spikes_per_second
105
+ self.obj_spike_counts = obj_spike_counts
106
+
107
+ # fp control threshold params (remove?)
108
+ self.max_fp_per_input_spike = max_fp_per_input_spike
109
+ self.parent_sorting_hdf5_path = parent_sorting_hdf5_path
110
+
111
+ assert channel_selection in ("template", "amplitude")
112
+ self.channel_selection: Literal["template", "amplitude"] = channel_selection
113
+ if channel_selection_index is None:
114
+ assert self.channel_selection == "template"
115
+ self.channel_selection_index = None
116
+ else:
117
+ self.register_buffer("channel_selection_index", channel_selection_index)
118
+ self.picking_channels = self.channel_selection != "template"
119
+
120
+ self.up_factor = up_factor
121
+ self.is_upsampling = up_factor > 1
122
+
123
+ # amplitude scaling properties
124
+ self.is_scaling = amplitude_scaling_variance > 0
125
+ self.amplitude_scaling_variance = amplitude_scaling_variance
126
+ self.inv_lambda = (
127
+ 1.0 / amplitude_scaling_variance if self.is_scaling else float("inf")
128
+ )
129
+ self.amp_scale_max = 1.0 + amplitude_scaling_boundary
130
+ self.amp_scale_min = 1.0 / self.amp_scale_max
131
+ self.obj_pad_len = max(refractory_radius_frames, self.spike_length_samples - 1)
132
+
133
+ conv_len = (
134
+ self.chunk_length_samples
135
+ + 2 * self.chunk_margin_samples
136
+ + 2 * self.obj_pad_len
137
+ )
138
+ self.register_buffer("obj_arange", torch.arange(conv_len))
139
+
140
+ def peeling_needs_precompute(self):
141
+ return self.matching_templates is None or self.thresholdsq is None
142
+
143
+ def precompute_peeling_data(
144
+ self,
145
+ save_folder,
146
+ overwrite=False,
147
+ computation_cfg: ComputationConfig | None = None,
148
+ ):
149
+ if self.matching_templates is None:
150
+ assert self.matching_templates_builder is not None
151
+ self.matching_templates = self.matching_templates_builder.build(
152
+ save_folder, computation_cfg=computation_cfg, overwrite=overwrite
153
+ )
154
+ self.pick_threshold()
155
+
156
+ def out_datasets(self):
157
+ datasets = super().out_datasets()
158
+ datasets.extend(
159
+ [
160
+ SpikeDataset(name="template_inds", shape_per_spike=(), dtype=np.int64),
161
+ SpikeDataset(name="labels", shape_per_spike=(), dtype=np.int64),
162
+ SpikeDataset(name="scores", shape_per_spike=(), dtype=float),
163
+ ]
164
+ )
165
+ if self.is_scaling:
166
+ datasets.append(
167
+ SpikeDataset(name="scalings", shape_per_spike=(), dtype=float),
168
+ )
169
+ if self.is_upsampling:
170
+ datasets.append(
171
+ SpikeDataset(name="up_inds", shape_per_spike=(), dtype=np.int64)
172
+ )
173
+ datasets.append(
174
+ SpikeDataset(name="time_shifts", shape_per_spike=(), dtype=np.int8),
175
+ )
176
+ return datasets
177
+
178
+ @classmethod
179
+ def from_config(
180
+ cls,
181
+ recording: BaseRecording,
182
+ *,
183
+ waveform_cfg: WaveformConfig,
184
+ matching_cfg: MatchingConfig,
185
+ featurization_cfg: FeaturizationConfig,
186
+ sampling_cfg: FitSamplingConfig,
187
+ template_data: TemplateData | None,
188
+ motion_est=None,
189
+ parent_sorting_hdf5_path=None,
190
+ ) -> Self:
191
+ geom = torch.tensor(recording.get_channel_locations())
192
+ channel_index = make_channel_index(
193
+ geom, featurization_cfg.extract_radius, to_torch=True
194
+ )
195
+ channel_selection = "template"
196
+ channel_selection_index = None
197
+ if matching_cfg.channel_selection_radius:
198
+ channel_selection = "amplitude"
199
+ channel_selection_index = make_channel_index(
200
+ geom, matching_cfg.channel_selection_radius, to_torch=True
201
+ )
202
+ featurization_pipeline = WaveformPipeline.from_config(
203
+ geom=geom,
204
+ channel_index=channel_index,
205
+ featurization_cfg=featurization_cfg,
206
+ waveform_cfg=waveform_cfg,
207
+ sampling_frequency=recording.sampling_frequency,
208
+ )
209
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
210
+ recording.sampling_frequency
211
+ )
212
+
213
+ if template_data is None:
214
+ assert matching_cfg.precomputed_templates_npz is not None
215
+ template_data = TemplateData.from_npz(
216
+ matching_cfg.precomputed_templates_npz
217
+ )
218
+ assert trough_offset_samples == template_data.trough_offset_samples
219
+
220
+ if matching_cfg.whiten:
221
+ # TODO defer computation of this
222
+ whitener = whitener_from_hdf5(
223
+ parent_sorting_hdf5_path,
224
+ motion_est=motion_est,
225
+ interp_params=matching_cfg.drift_interp_params,
226
+ rgeom=template_data.registered_geom,
227
+ )
228
+ whitener = torch.tensor(whitener)
229
+ else:
230
+ whitener = None
231
+
232
+ builder = MatchingTemplatesBuilder(
233
+ recording=recording,
234
+ template_data=template_data,
235
+ matching_cfg=matching_cfg,
236
+ motion_est=motion_est,
237
+ whitener=whitener,
238
+ )
239
+
240
+ logger.info(
241
+ "Constructing a matcher with template kind %s, drift %senabled, "
242
+ "scaling variance %s, compression rank %s, and upsampling factor %s.",
243
+ matching_cfg.template_type,
244
+ "not " if motion_est is None else "",
245
+ matching_cfg.amplitude_scaling_variance,
246
+ matching_cfg.template_svd_compression_rank,
247
+ matching_cfg.template_temporal_upsampling_factor,
248
+ )
249
+
250
+ return cls(
251
+ recording=recording,
252
+ matching_templates_builder=builder,
253
+ channel_index=channel_index,
254
+ featurization_pipeline=featurization_pipeline,
255
+ refractory_radius_frames=matching_cfg.refractory_radius_frames,
256
+ amplitude_scaling_variance=matching_cfg.amplitude_scaling_variance,
257
+ amplitude_scaling_boundary=matching_cfg.amplitude_scaling_boundary,
258
+ trough_offset_samples=trough_offset_samples,
259
+ threshold=matching_cfg.threshold,
260
+ channel_selection=channel_selection,
261
+ channel_selection_index=channel_selection_index,
262
+ chunk_length_samples=matching_cfg.chunk_length_samples,
263
+ n_seconds_fit=matching_cfg.n_seconds_fit,
264
+ max_waveforms_fit=sampling_cfg.max_waveforms_fit,
265
+ fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
266
+ n_waveforms_fit=sampling_cfg.n_waveforms_fit,
267
+ fit_sampling=sampling_cfg.fit_sampling,
268
+ fit_max_reweighting=sampling_cfg.fit_max_reweighting,
269
+ up_factor=matching_cfg.template_temporal_upsampling_factor,
270
+ max_iter=matching_cfg.max_iter,
271
+ max_spikes_per_second=matching_cfg.max_spikes_per_second,
272
+ cd_iter=matching_cfg.cd_iter,
273
+ coarse_cd=matching_cfg.coarse_cd,
274
+ parent_sorting_hdf5_path=parent_sorting_hdf5_path,
275
+ obj_spike_counts=template_data.coarsen().spike_counts
276
+ if matching_cfg.threshold == "fp_control"
277
+ else None,
278
+ )
279
+
280
+ def peel_chunk(
281
+ self,
282
+ traces,
283
+ *,
284
+ chunk_start_samples=0,
285
+ left_margin=0,
286
+ right_margin=0,
287
+ return_residual=False,
288
+ return_waveforms=True,
289
+ return_conv=False,
290
+ ) -> PeelingBatchResult:
291
+ assert self.matching_templates is not None
292
+ # get chunk center time and template info at that time
293
+ chunk_center_samples = chunk_start_samples + self.chunk_length_samples // 2
294
+ segment = self.recording._recording_segments[0]
295
+ chunk_center_seconds = segment.sample_index_to_time(chunk_center_samples)
296
+ chunk_template_data = self.matching_templates.data_at_time(
297
+ t_s=chunk_center_seconds,
298
+ scaling=self.is_scaling,
299
+ inv_lambda=self.inv_lambda,
300
+ scale_min=self.amp_scale_min,
301
+ scale_max=self.amp_scale_max,
302
+ )
303
+
304
+ # deconvolve
305
+ match_results = self.match_chunk(
306
+ traces,
307
+ chunk_template_data,
308
+ left_margin=left_margin,
309
+ right_margin=right_margin,
310
+ return_residual=return_residual,
311
+ return_conv=return_conv,
312
+ return_collisioncleaned_waveforms=return_waveforms,
313
+ )
314
+
315
+ # process spike times and create return result
316
+ if match_results["n_spikes"]:
317
+ match_results["times_samples"] += chunk_start_samples - left_margin # type: ignore
318
+ if match_results["n_spikes"] > self.max_spikes_per_second: # type: ignore
319
+ raise ValueError(
320
+ f"Too many spikes {match_results['n_spikes']} > {self.max_spikes_per_second}."
321
+ )
322
+
323
+ return match_results
324
+
325
+ def match_chunk(
326
+ self,
327
+ traces: Tensor,
328
+ chunk_template_data: ChunkTemplateData,
329
+ left_margin=0,
330
+ right_margin=0,
331
+ return_collisioncleaned_waveforms=True,
332
+ return_residual=False,
333
+ return_conv=False,
334
+ unit_mask=None,
335
+ max_iter: int | None = None,
336
+ ) -> PeelingBatchResult:
337
+ """Core peeling routine for subtraction"""
338
+ if max_iter is None:
339
+ max_iter = self.max_iter
340
+ # initialize residual, it needs to be padded to support our channel
341
+ # indexing convention (used later to extract small channel
342
+ # neighborhoods). this copies the input.
343
+ residual_padded = F.pad(traces, (0, 1), value=torch.nan)
344
+ residual = residual_padded[:, :-1]
345
+
346
+ # name objective variables so that we can update them in-place later
347
+ # padded objective has an extra unit (for group_index) and refractory
348
+ # padding (for easier implementation of enforce_refractory)
349
+ valid_len = traces.shape[0] - self.spike_length_samples + 1
350
+ padded_obj_len = valid_len + 2 * self.obj_pad_len
351
+ padded_conv = traces.new_zeros(
352
+ chunk_template_data.obj_n_templates, padded_obj_len
353
+ )
354
+ if self.is_scaling:
355
+ padded_scalings = padded_conv.clone()
356
+ else:
357
+ padded_scalings = None
358
+ padded_objective = traces.new_zeros(
359
+ chunk_template_data.obj_n_templates + 1, padded_obj_len
360
+ )
361
+ refrac_mask = torch.zeros_like(padded_objective)
362
+
363
+ # initialize convolution
364
+ chunk_template_data.convolve(
365
+ residual.T, padding=self.obj_pad_len, out=padded_conv
366
+ )
367
+
368
+ # main loop
369
+ previous_peaks = current_peaks = prev_refrac_mask = None
370
+ prev_update_residual = None
371
+ for cd_it in range(self.cd_iter + 1):
372
+ initializing_cd = not cd_it
373
+ coarse_only = self.coarse_cd and cd_it < self.cd_iter
374
+
375
+ # we always need to update the residual in the final iteration
376
+ # in "cd iterations", we may not need to update the residual.
377
+ update_residual = not coarse_only
378
+
379
+ if not initializing_cd:
380
+ refrac_mask = torch.zeros_like(refrac_mask)
381
+
382
+ current_peaks = []
383
+ for match_it in range(max_iter):
384
+ if not initializing_cd:
385
+ assert previous_peaks is not None
386
+ if (
387
+ not initializing_cd
388
+ and previous_peaks is not None
389
+ and len(previous_peaks)
390
+ ):
391
+ assert prev_refrac_mask is not None
392
+ assert prev_update_residual is not None
393
+
394
+ prev_peaks = previous_peaks.pop()
395
+ if prev_update_residual:
396
+ chunk_template_data.unsubtract(residual_padded, prev_peaks)
397
+ chunk_template_data.unsubtract_conv(
398
+ padded_conv, prev_peaks, padding=self.obj_pad_len
399
+ )
400
+ chunk_template_data.forget_refractory(
401
+ prev_refrac_mask, prev_peaks, offset=self.obj_pad_len
402
+ )
403
+ apply_refrac_mask = refrac_mask + prev_refrac_mask
404
+ else:
405
+ apply_refrac_mask = refrac_mask
406
+
407
+ # find spikes
408
+ new_peaks = self.find_peaks(
409
+ residual=residual,
410
+ padded_conv=padded_conv,
411
+ padded_scalings=padded_scalings,
412
+ padded_objective=padded_objective,
413
+ refrac_mask=apply_refrac_mask,
414
+ chunk_template_data=chunk_template_data,
415
+ unit_mask=unit_mask,
416
+ coarse_only=coarse_only,
417
+ )
418
+ if new_peaks is None or not new_peaks.n_spikes:
419
+ break
420
+
421
+ # enforce refractoriness
422
+ chunk_template_data.enforce_refractory(
423
+ refrac_mask, new_peaks, offset=self.obj_pad_len
424
+ )
425
+
426
+ # subtract them
427
+ if update_residual:
428
+ chunk_template_data.subtract(residual_padded, new_peaks)
429
+ chunk_template_data.subtract_conv(
430
+ padded_conv, new_peaks, padding=self.obj_pad_len
431
+ )
432
+
433
+ # update spike train
434
+ current_peaks.append(new_peaks)
435
+
436
+ # some of this round's peaks will be added back in before
437
+ # each iteration in the next round
438
+ previous_peaks = current_peaks
439
+ previous_peaks.reverse()
440
+ prev_refrac_mask = refrac_mask
441
+ prev_update_residual = update_residual
442
+
443
+ # peaks are then the final current_peaks
444
+ assert current_peaks is not None
445
+ peaks = MatchingPeaks.concatenate(current_peaks)
446
+ if not peaks.n_spikes:
447
+ res = PeelingBatchResult(n_spikes=0)
448
+ if return_residual:
449
+ residual = residual[left_margin : traces.shape[0] - right_margin]
450
+ res["residual"] = residual
451
+ if return_conv:
452
+ res["conv"] = padded_conv
453
+ return res
454
+
455
+ # subset to peaks inside the margin and sort for the caller
456
+ max_time = traces.shape[0] - right_margin - 1
457
+ peaks = peaks.subset_by_time(
458
+ left_margin, max_time, offset=self.trough_offset_samples
459
+ )
460
+ assert peaks.times is not None
461
+ assert peaks.template_inds is not None
462
+
463
+ # construct return value
464
+ res = peaks_to_batch_result(
465
+ peaks=peaks,
466
+ trough_offset_samples=self.trough_offset_samples,
467
+ unit_ids=chunk_template_data.unit_ids,
468
+ trough_shifts=peaks.time_shifts,
469
+ )
470
+ # extract collision-cleaned waveforms on small neighborhoods
471
+ if return_collisioncleaned_waveforms or self.picking_channels:
472
+ cc = chunk_template_data.get_collisioncleaned_waveforms(
473
+ residual_padded=residual_padded,
474
+ peaks=peaks,
475
+ channels=self.channel_selection,
476
+ channel_index=self.b.channel_index,
477
+ channel_selection_index=self.channel_selection_index,
478
+ )
479
+ channels, waveforms = cc
480
+ else:
481
+ assert self.channel_selection == "template"
482
+ channels = chunk_template_data.main_channels[peaks.template_inds]
483
+ waveforms = None
484
+ res["channels"] = channels
485
+ if return_collisioncleaned_waveforms:
486
+ assert waveforms is not None
487
+ res["collisioncleaned_waveforms"] = waveforms
488
+ if return_residual:
489
+ residual = residual[left_margin : traces.shape[0] - right_margin]
490
+ res["residual"] = residual
491
+ if return_conv:
492
+ res["conv"] = padded_conv
493
+ return res
494
+
495
+ def find_peaks(
496
+ self,
497
+ *,
498
+ residual: Tensor,
499
+ padded_conv: Tensor,
500
+ padded_objective: Tensor,
501
+ padded_scalings: Tensor | None,
502
+ refrac_mask: Tensor,
503
+ chunk_template_data: ChunkTemplateData,
504
+ unit_mask=None,
505
+ coarse_only=False,
506
+ ):
507
+ # update the coarse objective
508
+ chunk_template_data.obj_from_conv(
509
+ conv=padded_conv, out=padded_objective[:-1], scalings_out=padded_scalings
510
+ )
511
+
512
+ # enforce refractoriness
513
+ objective = (padded_objective + refrac_mask)[
514
+ :-1, self.obj_pad_len : -self.obj_pad_len
515
+ ]
516
+ if unit_mask is not None:
517
+ objective[torch.logical_not(unit_mask)] = -torch.inf
518
+
519
+ # find peaks in the coarse objective
520
+ assert self.thresholdsq is not None
521
+ if padded_scalings is None:
522
+ scalings = None
523
+ else:
524
+ scalings = padded_scalings[:, self.obj_pad_len : -self.obj_pad_len]
525
+ coarse_peaks = chunk_template_data.coarse_match(
526
+ objective=objective,
527
+ scalings=scalings,
528
+ thresholdsq=self.thresholdsq,
529
+ obj_arange=self.b.obj_arange,
530
+ )
531
+ if coarse_only or not coarse_peaks.n_spikes:
532
+ return coarse_peaks
533
+
534
+ # high-res peaks (upsampled or grouped)
535
+ fine_peaks = chunk_template_data.fine_match(
536
+ peaks=coarse_peaks,
537
+ residual=residual,
538
+ conv=padded_conv,
539
+ padding=self.obj_pad_len,
540
+ )
541
+
542
+ return fine_peaks
543
+
544
+ def pick_threshold(self):
545
+ if self.is_scaling and self.amplitude_scaling_variance < torch.inf:
546
+ # adjust threshold by the scaling prior's constant term
547
+ # nb, everything is x2 so halves are gone.
548
+ scstd = np.sqrt(self.amplitude_scaling_variance)
549
+ norm_const = -(np.log(2.0) + np.log(np.pi) + 2.0 * np.log(scstd))
550
+
551
+ # adjust by boundary
552
+ nm = norm(loc=1.0, scale=scstd)
553
+ p_up = nm.cdf(self.amp_scale_max)
554
+ p_lo = nm.cdf(self.amp_scale_min)
555
+ Z = p_up - p_lo
556
+
557
+ scale_const = norm_const - 2.0 * np.log(scstd) * np.log(Z)
558
+
559
+ if isinstance(self.threshold, float):
560
+ tb = np.sqrt(self.threshold**2 - scale_const)
561
+ _msg = f"In norm units, that's from {self.threshold:0.2f}->{tb:0.2f}"
562
+ else:
563
+ _msg = ""
564
+ logger.dartsortdebug(
565
+ f"matching: Amplitude scaling with std {scstd:0.4f} adjusts "
566
+ f"theshold by {scale_const:0.4f} in squared units. {_msg}"
567
+ )
568
+ else:
569
+ scale_const = 0.0
570
+
571
+ if isinstance(self.threshold, float):
572
+ self.thresholdsq = self.threshold**2 - scale_const
573
+ return
574
+
575
+ assert self.threshold == "fp_control"
576
+ from ..util import noise_util
577
+
578
+ # fit noise to residuals from the previous detection step
579
+ assert self.parent_sorting_hdf5_path is not None
580
+ assert self.matching_templates is not None
581
+ lrt: LowRankTemplates = self.matching_templates.obj_lrts # type: ignore
582
+ self.thresholdsq = noise_util.fp_control_threshold_from_h5(
583
+ hdf5_path=self.parent_sorting_hdf5_path,
584
+ low_rank_templates=lrt.shift_to_best_channels(
585
+ self.recording.get_channel_locations(), self.matching_templates.rgeom
586
+ ),
587
+ rg=self.fit_subsampling_random_state,
588
+ refractory_radius_frames=self.refractory_radius_frames,
589
+ num_frames=self.recording.get_num_samples(), # TODO: wrong in subsampled case.
590
+ max_fp_per_input_spike=self.max_fp_per_input_spike,
591
+ unit_ids=self.matching_templates.obj_unit_ids,
592
+ spike_counts=self.obj_spike_counts,
593
+ )
594
+ self.thresholdsq -= scale_const
595
+ logger.info(
596
+ f"Matcher picked threshold^2 {self.thresholdsq} for strategy fp_control."
597
+ )
598
+
599
+
600
+ def peaks_to_batch_result(
601
+ peaks: MatchingPeaks,
602
+ trough_offset_samples: int,
603
+ unit_ids: Tensor,
604
+ trough_shifts: Tensor | None,
605
+ ) -> PeelingBatchResult:
606
+ if not peaks.n_spikes:
607
+ return PeelingBatchResult(n_spikes=0)
608
+
609
+ assert peaks.times is not None
610
+ assert peaks.template_inds is not None
611
+ times_samples = peaks.times + trough_offset_samples
612
+ if trough_shifts is not None:
613
+ times_samples += trough_shifts
614
+ res = PeelingBatchResult(
615
+ n_spikes=peaks.n_spikes,
616
+ times_samples=times_samples,
617
+ labels=unit_ids[peaks.template_inds],
618
+ template_inds=peaks.template_inds,
619
+ )
620
+ if peaks.up_inds is not None:
621
+ res["up_inds"] = peaks.up_inds
622
+ if trough_shifts is not None:
623
+ res["time_shifts"] = trough_shifts
624
+ if peaks.scalings is not None:
625
+ res["scalings"] = peaks.scalings
626
+ if peaks.scores is not None:
627
+ res["scores"] = peaks.scores
628
+ return res
@@ -0,0 +1,3 @@
1
+ from .matching_base import *
2
+ from .compressed_upsampled import *
3
+ from .drifty import *