DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,628 @@
|
|
|
1
|
+
"""A simple residual updating template matcher."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Literal, Self
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import torch
|
|
8
|
+
import torch.nn.functional as F
|
|
9
|
+
from scipy.stats import norm
|
|
10
|
+
from spikeinterface.core import BaseRecording
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
|
|
13
|
+
from ..templates import LowRankTemplates, TemplateData
|
|
14
|
+
from ..transform import WaveformPipeline
|
|
15
|
+
from ..util.data_util import SpikeDataset
|
|
16
|
+
from ..util.internal_config import (
|
|
17
|
+
ComputationConfig,
|
|
18
|
+
FeaturizationConfig,
|
|
19
|
+
FitSamplingConfig,
|
|
20
|
+
MatchingConfig,
|
|
21
|
+
WaveformConfig,
|
|
22
|
+
)
|
|
23
|
+
from ..util.logging_util import get_logger
|
|
24
|
+
from ..util.noise_util import whitener_from_hdf5
|
|
25
|
+
from ..util.waveform_util import make_channel_index
|
|
26
|
+
from .matching_util import (
|
|
27
|
+
ChunkTemplateData,
|
|
28
|
+
MatchingPeaks,
|
|
29
|
+
MatchingTemplates,
|
|
30
|
+
MatchingTemplatesBuilder,
|
|
31
|
+
)
|
|
32
|
+
from .peel_base import BasePeeler, PeelingBatchResult
|
|
33
|
+
|
|
34
|
+
logger = get_logger(__name__)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
38
|
+
peel_kind = "TemplateMatching"
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
recording,
|
|
43
|
+
channel_index,
|
|
44
|
+
featurization_pipeline,
|
|
45
|
+
matching_templates: MatchingTemplates | None = None,
|
|
46
|
+
matching_templates_builder: MatchingTemplatesBuilder | None = None,
|
|
47
|
+
refractory_radius_frames=10,
|
|
48
|
+
amplitude_scaling_variance=0.0,
|
|
49
|
+
amplitude_scaling_boundary=0.5,
|
|
50
|
+
margin_factor=2,
|
|
51
|
+
up_factor: int = 1,
|
|
52
|
+
trough_offset_samples=42,
|
|
53
|
+
threshold: float | Literal["fp_control"] = 100.0,
|
|
54
|
+
obj_spike_counts=None,
|
|
55
|
+
max_fp_per_input_spike=2.5,
|
|
56
|
+
chunk_length_samples=30_000,
|
|
57
|
+
n_seconds_fit=40,
|
|
58
|
+
max_waveforms_fit=50_000,
|
|
59
|
+
n_waveforms_fit=20_000,
|
|
60
|
+
fit_max_reweighting=4.0,
|
|
61
|
+
channel_selection: Literal["template", "amplitude"] = "template",
|
|
62
|
+
channel_selection_index: Tensor | None = None,
|
|
63
|
+
fit_subsampling_random_state=0,
|
|
64
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
65
|
+
max_iter=1000,
|
|
66
|
+
max_spikes_per_second=16384,
|
|
67
|
+
cd_iter=0,
|
|
68
|
+
coarse_cd=True,
|
|
69
|
+
parent_sorting_hdf5_path: str | Path | None = None,
|
|
70
|
+
dtype=torch.float,
|
|
71
|
+
):
|
|
72
|
+
if matching_templates is not None:
|
|
73
|
+
spike_length_samples = matching_templates.spike_length_samples
|
|
74
|
+
elif matching_templates_builder is not None:
|
|
75
|
+
spike_length_samples = matching_templates_builder.spike_length_samples
|
|
76
|
+
else:
|
|
77
|
+
raise ValueError(f"Need either a MatchingTemplates or a builder.")
|
|
78
|
+
|
|
79
|
+
super().__init__(
|
|
80
|
+
recording=recording,
|
|
81
|
+
channel_index=channel_index,
|
|
82
|
+
featurization_pipeline=featurization_pipeline,
|
|
83
|
+
chunk_length_samples=chunk_length_samples,
|
|
84
|
+
chunk_margin_samples=margin_factor * spike_length_samples,
|
|
85
|
+
n_seconds_fit=n_seconds_fit,
|
|
86
|
+
max_waveforms_fit=max_waveforms_fit,
|
|
87
|
+
fit_subsampling_random_state=fit_subsampling_random_state,
|
|
88
|
+
fit_sampling=fit_sampling,
|
|
89
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
90
|
+
fit_max_reweighting=fit_max_reweighting,
|
|
91
|
+
trough_offset_samples=trough_offset_samples,
|
|
92
|
+
spike_length_samples=spike_length_samples,
|
|
93
|
+
dtype=dtype,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
self.matching_templates = matching_templates
|
|
97
|
+
self.matching_templates_builder = matching_templates_builder
|
|
98
|
+
self.threshold = threshold
|
|
99
|
+
self.thresholdsq = None # set in precompute. usually just ^ squared.
|
|
100
|
+
self.refractory_radius_frames = refractory_radius_frames
|
|
101
|
+
self.max_iter = max_iter
|
|
102
|
+
self.cd_iter = cd_iter
|
|
103
|
+
self.coarse_cd = coarse_cd
|
|
104
|
+
self.max_spikes_per_second = max_spikes_per_second
|
|
105
|
+
self.obj_spike_counts = obj_spike_counts
|
|
106
|
+
|
|
107
|
+
# fp control threshold params (remove?)
|
|
108
|
+
self.max_fp_per_input_spike = max_fp_per_input_spike
|
|
109
|
+
self.parent_sorting_hdf5_path = parent_sorting_hdf5_path
|
|
110
|
+
|
|
111
|
+
assert channel_selection in ("template", "amplitude")
|
|
112
|
+
self.channel_selection: Literal["template", "amplitude"] = channel_selection
|
|
113
|
+
if channel_selection_index is None:
|
|
114
|
+
assert self.channel_selection == "template"
|
|
115
|
+
self.channel_selection_index = None
|
|
116
|
+
else:
|
|
117
|
+
self.register_buffer("channel_selection_index", channel_selection_index)
|
|
118
|
+
self.picking_channels = self.channel_selection != "template"
|
|
119
|
+
|
|
120
|
+
self.up_factor = up_factor
|
|
121
|
+
self.is_upsampling = up_factor > 1
|
|
122
|
+
|
|
123
|
+
# amplitude scaling properties
|
|
124
|
+
self.is_scaling = amplitude_scaling_variance > 0
|
|
125
|
+
self.amplitude_scaling_variance = amplitude_scaling_variance
|
|
126
|
+
self.inv_lambda = (
|
|
127
|
+
1.0 / amplitude_scaling_variance if self.is_scaling else float("inf")
|
|
128
|
+
)
|
|
129
|
+
self.amp_scale_max = 1.0 + amplitude_scaling_boundary
|
|
130
|
+
self.amp_scale_min = 1.0 / self.amp_scale_max
|
|
131
|
+
self.obj_pad_len = max(refractory_radius_frames, self.spike_length_samples - 1)
|
|
132
|
+
|
|
133
|
+
conv_len = (
|
|
134
|
+
self.chunk_length_samples
|
|
135
|
+
+ 2 * self.chunk_margin_samples
|
|
136
|
+
+ 2 * self.obj_pad_len
|
|
137
|
+
)
|
|
138
|
+
self.register_buffer("obj_arange", torch.arange(conv_len))
|
|
139
|
+
|
|
140
|
+
def peeling_needs_precompute(self):
|
|
141
|
+
return self.matching_templates is None or self.thresholdsq is None
|
|
142
|
+
|
|
143
|
+
def precompute_peeling_data(
|
|
144
|
+
self,
|
|
145
|
+
save_folder,
|
|
146
|
+
overwrite=False,
|
|
147
|
+
computation_cfg: ComputationConfig | None = None,
|
|
148
|
+
):
|
|
149
|
+
if self.matching_templates is None:
|
|
150
|
+
assert self.matching_templates_builder is not None
|
|
151
|
+
self.matching_templates = self.matching_templates_builder.build(
|
|
152
|
+
save_folder, computation_cfg=computation_cfg, overwrite=overwrite
|
|
153
|
+
)
|
|
154
|
+
self.pick_threshold()
|
|
155
|
+
|
|
156
|
+
def out_datasets(self):
|
|
157
|
+
datasets = super().out_datasets()
|
|
158
|
+
datasets.extend(
|
|
159
|
+
[
|
|
160
|
+
SpikeDataset(name="template_inds", shape_per_spike=(), dtype=np.int64),
|
|
161
|
+
SpikeDataset(name="labels", shape_per_spike=(), dtype=np.int64),
|
|
162
|
+
SpikeDataset(name="scores", shape_per_spike=(), dtype=float),
|
|
163
|
+
]
|
|
164
|
+
)
|
|
165
|
+
if self.is_scaling:
|
|
166
|
+
datasets.append(
|
|
167
|
+
SpikeDataset(name="scalings", shape_per_spike=(), dtype=float),
|
|
168
|
+
)
|
|
169
|
+
if self.is_upsampling:
|
|
170
|
+
datasets.append(
|
|
171
|
+
SpikeDataset(name="up_inds", shape_per_spike=(), dtype=np.int64)
|
|
172
|
+
)
|
|
173
|
+
datasets.append(
|
|
174
|
+
SpikeDataset(name="time_shifts", shape_per_spike=(), dtype=np.int8),
|
|
175
|
+
)
|
|
176
|
+
return datasets
|
|
177
|
+
|
|
178
|
+
@classmethod
|
|
179
|
+
def from_config(
|
|
180
|
+
cls,
|
|
181
|
+
recording: BaseRecording,
|
|
182
|
+
*,
|
|
183
|
+
waveform_cfg: WaveformConfig,
|
|
184
|
+
matching_cfg: MatchingConfig,
|
|
185
|
+
featurization_cfg: FeaturizationConfig,
|
|
186
|
+
sampling_cfg: FitSamplingConfig,
|
|
187
|
+
template_data: TemplateData | None,
|
|
188
|
+
motion_est=None,
|
|
189
|
+
parent_sorting_hdf5_path=None,
|
|
190
|
+
) -> Self:
|
|
191
|
+
geom = torch.tensor(recording.get_channel_locations())
|
|
192
|
+
channel_index = make_channel_index(
|
|
193
|
+
geom, featurization_cfg.extract_radius, to_torch=True
|
|
194
|
+
)
|
|
195
|
+
channel_selection = "template"
|
|
196
|
+
channel_selection_index = None
|
|
197
|
+
if matching_cfg.channel_selection_radius:
|
|
198
|
+
channel_selection = "amplitude"
|
|
199
|
+
channel_selection_index = make_channel_index(
|
|
200
|
+
geom, matching_cfg.channel_selection_radius, to_torch=True
|
|
201
|
+
)
|
|
202
|
+
featurization_pipeline = WaveformPipeline.from_config(
|
|
203
|
+
geom=geom,
|
|
204
|
+
channel_index=channel_index,
|
|
205
|
+
featurization_cfg=featurization_cfg,
|
|
206
|
+
waveform_cfg=waveform_cfg,
|
|
207
|
+
sampling_frequency=recording.sampling_frequency,
|
|
208
|
+
)
|
|
209
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(
|
|
210
|
+
recording.sampling_frequency
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
if template_data is None:
|
|
214
|
+
assert matching_cfg.precomputed_templates_npz is not None
|
|
215
|
+
template_data = TemplateData.from_npz(
|
|
216
|
+
matching_cfg.precomputed_templates_npz
|
|
217
|
+
)
|
|
218
|
+
assert trough_offset_samples == template_data.trough_offset_samples
|
|
219
|
+
|
|
220
|
+
if matching_cfg.whiten:
|
|
221
|
+
# TODO defer computation of this
|
|
222
|
+
whitener = whitener_from_hdf5(
|
|
223
|
+
parent_sorting_hdf5_path,
|
|
224
|
+
motion_est=motion_est,
|
|
225
|
+
interp_params=matching_cfg.drift_interp_params,
|
|
226
|
+
rgeom=template_data.registered_geom,
|
|
227
|
+
)
|
|
228
|
+
whitener = torch.tensor(whitener)
|
|
229
|
+
else:
|
|
230
|
+
whitener = None
|
|
231
|
+
|
|
232
|
+
builder = MatchingTemplatesBuilder(
|
|
233
|
+
recording=recording,
|
|
234
|
+
template_data=template_data,
|
|
235
|
+
matching_cfg=matching_cfg,
|
|
236
|
+
motion_est=motion_est,
|
|
237
|
+
whitener=whitener,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
logger.info(
|
|
241
|
+
"Constructing a matcher with template kind %s, drift %senabled, "
|
|
242
|
+
"scaling variance %s, compression rank %s, and upsampling factor %s.",
|
|
243
|
+
matching_cfg.template_type,
|
|
244
|
+
"not " if motion_est is None else "",
|
|
245
|
+
matching_cfg.amplitude_scaling_variance,
|
|
246
|
+
matching_cfg.template_svd_compression_rank,
|
|
247
|
+
matching_cfg.template_temporal_upsampling_factor,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
return cls(
|
|
251
|
+
recording=recording,
|
|
252
|
+
matching_templates_builder=builder,
|
|
253
|
+
channel_index=channel_index,
|
|
254
|
+
featurization_pipeline=featurization_pipeline,
|
|
255
|
+
refractory_radius_frames=matching_cfg.refractory_radius_frames,
|
|
256
|
+
amplitude_scaling_variance=matching_cfg.amplitude_scaling_variance,
|
|
257
|
+
amplitude_scaling_boundary=matching_cfg.amplitude_scaling_boundary,
|
|
258
|
+
trough_offset_samples=trough_offset_samples,
|
|
259
|
+
threshold=matching_cfg.threshold,
|
|
260
|
+
channel_selection=channel_selection,
|
|
261
|
+
channel_selection_index=channel_selection_index,
|
|
262
|
+
chunk_length_samples=matching_cfg.chunk_length_samples,
|
|
263
|
+
n_seconds_fit=matching_cfg.n_seconds_fit,
|
|
264
|
+
max_waveforms_fit=sampling_cfg.max_waveforms_fit,
|
|
265
|
+
fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
|
|
266
|
+
n_waveforms_fit=sampling_cfg.n_waveforms_fit,
|
|
267
|
+
fit_sampling=sampling_cfg.fit_sampling,
|
|
268
|
+
fit_max_reweighting=sampling_cfg.fit_max_reweighting,
|
|
269
|
+
up_factor=matching_cfg.template_temporal_upsampling_factor,
|
|
270
|
+
max_iter=matching_cfg.max_iter,
|
|
271
|
+
max_spikes_per_second=matching_cfg.max_spikes_per_second,
|
|
272
|
+
cd_iter=matching_cfg.cd_iter,
|
|
273
|
+
coarse_cd=matching_cfg.coarse_cd,
|
|
274
|
+
parent_sorting_hdf5_path=parent_sorting_hdf5_path,
|
|
275
|
+
obj_spike_counts=template_data.coarsen().spike_counts
|
|
276
|
+
if matching_cfg.threshold == "fp_control"
|
|
277
|
+
else None,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
def peel_chunk(
|
|
281
|
+
self,
|
|
282
|
+
traces,
|
|
283
|
+
*,
|
|
284
|
+
chunk_start_samples=0,
|
|
285
|
+
left_margin=0,
|
|
286
|
+
right_margin=0,
|
|
287
|
+
return_residual=False,
|
|
288
|
+
return_waveforms=True,
|
|
289
|
+
return_conv=False,
|
|
290
|
+
) -> PeelingBatchResult:
|
|
291
|
+
assert self.matching_templates is not None
|
|
292
|
+
# get chunk center time and template info at that time
|
|
293
|
+
chunk_center_samples = chunk_start_samples + self.chunk_length_samples // 2
|
|
294
|
+
segment = self.recording._recording_segments[0]
|
|
295
|
+
chunk_center_seconds = segment.sample_index_to_time(chunk_center_samples)
|
|
296
|
+
chunk_template_data = self.matching_templates.data_at_time(
|
|
297
|
+
t_s=chunk_center_seconds,
|
|
298
|
+
scaling=self.is_scaling,
|
|
299
|
+
inv_lambda=self.inv_lambda,
|
|
300
|
+
scale_min=self.amp_scale_min,
|
|
301
|
+
scale_max=self.amp_scale_max,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
# deconvolve
|
|
305
|
+
match_results = self.match_chunk(
|
|
306
|
+
traces,
|
|
307
|
+
chunk_template_data,
|
|
308
|
+
left_margin=left_margin,
|
|
309
|
+
right_margin=right_margin,
|
|
310
|
+
return_residual=return_residual,
|
|
311
|
+
return_conv=return_conv,
|
|
312
|
+
return_collisioncleaned_waveforms=return_waveforms,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
# process spike times and create return result
|
|
316
|
+
if match_results["n_spikes"]:
|
|
317
|
+
match_results["times_samples"] += chunk_start_samples - left_margin # type: ignore
|
|
318
|
+
if match_results["n_spikes"] > self.max_spikes_per_second: # type: ignore
|
|
319
|
+
raise ValueError(
|
|
320
|
+
f"Too many spikes {match_results['n_spikes']} > {self.max_spikes_per_second}."
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
return match_results
|
|
324
|
+
|
|
325
|
+
def match_chunk(
|
|
326
|
+
self,
|
|
327
|
+
traces: Tensor,
|
|
328
|
+
chunk_template_data: ChunkTemplateData,
|
|
329
|
+
left_margin=0,
|
|
330
|
+
right_margin=0,
|
|
331
|
+
return_collisioncleaned_waveforms=True,
|
|
332
|
+
return_residual=False,
|
|
333
|
+
return_conv=False,
|
|
334
|
+
unit_mask=None,
|
|
335
|
+
max_iter: int | None = None,
|
|
336
|
+
) -> PeelingBatchResult:
|
|
337
|
+
"""Core peeling routine for subtraction"""
|
|
338
|
+
if max_iter is None:
|
|
339
|
+
max_iter = self.max_iter
|
|
340
|
+
# initialize residual, it needs to be padded to support our channel
|
|
341
|
+
# indexing convention (used later to extract small channel
|
|
342
|
+
# neighborhoods). this copies the input.
|
|
343
|
+
residual_padded = F.pad(traces, (0, 1), value=torch.nan)
|
|
344
|
+
residual = residual_padded[:, :-1]
|
|
345
|
+
|
|
346
|
+
# name objective variables so that we can update them in-place later
|
|
347
|
+
# padded objective has an extra unit (for group_index) and refractory
|
|
348
|
+
# padding (for easier implementation of enforce_refractory)
|
|
349
|
+
valid_len = traces.shape[0] - self.spike_length_samples + 1
|
|
350
|
+
padded_obj_len = valid_len + 2 * self.obj_pad_len
|
|
351
|
+
padded_conv = traces.new_zeros(
|
|
352
|
+
chunk_template_data.obj_n_templates, padded_obj_len
|
|
353
|
+
)
|
|
354
|
+
if self.is_scaling:
|
|
355
|
+
padded_scalings = padded_conv.clone()
|
|
356
|
+
else:
|
|
357
|
+
padded_scalings = None
|
|
358
|
+
padded_objective = traces.new_zeros(
|
|
359
|
+
chunk_template_data.obj_n_templates + 1, padded_obj_len
|
|
360
|
+
)
|
|
361
|
+
refrac_mask = torch.zeros_like(padded_objective)
|
|
362
|
+
|
|
363
|
+
# initialize convolution
|
|
364
|
+
chunk_template_data.convolve(
|
|
365
|
+
residual.T, padding=self.obj_pad_len, out=padded_conv
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
# main loop
|
|
369
|
+
previous_peaks = current_peaks = prev_refrac_mask = None
|
|
370
|
+
prev_update_residual = None
|
|
371
|
+
for cd_it in range(self.cd_iter + 1):
|
|
372
|
+
initializing_cd = not cd_it
|
|
373
|
+
coarse_only = self.coarse_cd and cd_it < self.cd_iter
|
|
374
|
+
|
|
375
|
+
# we always need to update the residual in the final iteration
|
|
376
|
+
# in "cd iterations", we may not need to update the residual.
|
|
377
|
+
update_residual = not coarse_only
|
|
378
|
+
|
|
379
|
+
if not initializing_cd:
|
|
380
|
+
refrac_mask = torch.zeros_like(refrac_mask)
|
|
381
|
+
|
|
382
|
+
current_peaks = []
|
|
383
|
+
for match_it in range(max_iter):
|
|
384
|
+
if not initializing_cd:
|
|
385
|
+
assert previous_peaks is not None
|
|
386
|
+
if (
|
|
387
|
+
not initializing_cd
|
|
388
|
+
and previous_peaks is not None
|
|
389
|
+
and len(previous_peaks)
|
|
390
|
+
):
|
|
391
|
+
assert prev_refrac_mask is not None
|
|
392
|
+
assert prev_update_residual is not None
|
|
393
|
+
|
|
394
|
+
prev_peaks = previous_peaks.pop()
|
|
395
|
+
if prev_update_residual:
|
|
396
|
+
chunk_template_data.unsubtract(residual_padded, prev_peaks)
|
|
397
|
+
chunk_template_data.unsubtract_conv(
|
|
398
|
+
padded_conv, prev_peaks, padding=self.obj_pad_len
|
|
399
|
+
)
|
|
400
|
+
chunk_template_data.forget_refractory(
|
|
401
|
+
prev_refrac_mask, prev_peaks, offset=self.obj_pad_len
|
|
402
|
+
)
|
|
403
|
+
apply_refrac_mask = refrac_mask + prev_refrac_mask
|
|
404
|
+
else:
|
|
405
|
+
apply_refrac_mask = refrac_mask
|
|
406
|
+
|
|
407
|
+
# find spikes
|
|
408
|
+
new_peaks = self.find_peaks(
|
|
409
|
+
residual=residual,
|
|
410
|
+
padded_conv=padded_conv,
|
|
411
|
+
padded_scalings=padded_scalings,
|
|
412
|
+
padded_objective=padded_objective,
|
|
413
|
+
refrac_mask=apply_refrac_mask,
|
|
414
|
+
chunk_template_data=chunk_template_data,
|
|
415
|
+
unit_mask=unit_mask,
|
|
416
|
+
coarse_only=coarse_only,
|
|
417
|
+
)
|
|
418
|
+
if new_peaks is None or not new_peaks.n_spikes:
|
|
419
|
+
break
|
|
420
|
+
|
|
421
|
+
# enforce refractoriness
|
|
422
|
+
chunk_template_data.enforce_refractory(
|
|
423
|
+
refrac_mask, new_peaks, offset=self.obj_pad_len
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
# subtract them
|
|
427
|
+
if update_residual:
|
|
428
|
+
chunk_template_data.subtract(residual_padded, new_peaks)
|
|
429
|
+
chunk_template_data.subtract_conv(
|
|
430
|
+
padded_conv, new_peaks, padding=self.obj_pad_len
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
# update spike train
|
|
434
|
+
current_peaks.append(new_peaks)
|
|
435
|
+
|
|
436
|
+
# some of this round's peaks will be added back in before
|
|
437
|
+
# each iteration in the next round
|
|
438
|
+
previous_peaks = current_peaks
|
|
439
|
+
previous_peaks.reverse()
|
|
440
|
+
prev_refrac_mask = refrac_mask
|
|
441
|
+
prev_update_residual = update_residual
|
|
442
|
+
|
|
443
|
+
# peaks are then the final current_peaks
|
|
444
|
+
assert current_peaks is not None
|
|
445
|
+
peaks = MatchingPeaks.concatenate(current_peaks)
|
|
446
|
+
if not peaks.n_spikes:
|
|
447
|
+
res = PeelingBatchResult(n_spikes=0)
|
|
448
|
+
if return_residual:
|
|
449
|
+
residual = residual[left_margin : traces.shape[0] - right_margin]
|
|
450
|
+
res["residual"] = residual
|
|
451
|
+
if return_conv:
|
|
452
|
+
res["conv"] = padded_conv
|
|
453
|
+
return res
|
|
454
|
+
|
|
455
|
+
# subset to peaks inside the margin and sort for the caller
|
|
456
|
+
max_time = traces.shape[0] - right_margin - 1
|
|
457
|
+
peaks = peaks.subset_by_time(
|
|
458
|
+
left_margin, max_time, offset=self.trough_offset_samples
|
|
459
|
+
)
|
|
460
|
+
assert peaks.times is not None
|
|
461
|
+
assert peaks.template_inds is not None
|
|
462
|
+
|
|
463
|
+
# construct return value
|
|
464
|
+
res = peaks_to_batch_result(
|
|
465
|
+
peaks=peaks,
|
|
466
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
467
|
+
unit_ids=chunk_template_data.unit_ids,
|
|
468
|
+
trough_shifts=peaks.time_shifts,
|
|
469
|
+
)
|
|
470
|
+
# extract collision-cleaned waveforms on small neighborhoods
|
|
471
|
+
if return_collisioncleaned_waveforms or self.picking_channels:
|
|
472
|
+
cc = chunk_template_data.get_collisioncleaned_waveforms(
|
|
473
|
+
residual_padded=residual_padded,
|
|
474
|
+
peaks=peaks,
|
|
475
|
+
channels=self.channel_selection,
|
|
476
|
+
channel_index=self.b.channel_index,
|
|
477
|
+
channel_selection_index=self.channel_selection_index,
|
|
478
|
+
)
|
|
479
|
+
channels, waveforms = cc
|
|
480
|
+
else:
|
|
481
|
+
assert self.channel_selection == "template"
|
|
482
|
+
channels = chunk_template_data.main_channels[peaks.template_inds]
|
|
483
|
+
waveforms = None
|
|
484
|
+
res["channels"] = channels
|
|
485
|
+
if return_collisioncleaned_waveforms:
|
|
486
|
+
assert waveforms is not None
|
|
487
|
+
res["collisioncleaned_waveforms"] = waveforms
|
|
488
|
+
if return_residual:
|
|
489
|
+
residual = residual[left_margin : traces.shape[0] - right_margin]
|
|
490
|
+
res["residual"] = residual
|
|
491
|
+
if return_conv:
|
|
492
|
+
res["conv"] = padded_conv
|
|
493
|
+
return res
|
|
494
|
+
|
|
495
|
+
def find_peaks(
|
|
496
|
+
self,
|
|
497
|
+
*,
|
|
498
|
+
residual: Tensor,
|
|
499
|
+
padded_conv: Tensor,
|
|
500
|
+
padded_objective: Tensor,
|
|
501
|
+
padded_scalings: Tensor | None,
|
|
502
|
+
refrac_mask: Tensor,
|
|
503
|
+
chunk_template_data: ChunkTemplateData,
|
|
504
|
+
unit_mask=None,
|
|
505
|
+
coarse_only=False,
|
|
506
|
+
):
|
|
507
|
+
# update the coarse objective
|
|
508
|
+
chunk_template_data.obj_from_conv(
|
|
509
|
+
conv=padded_conv, out=padded_objective[:-1], scalings_out=padded_scalings
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
# enforce refractoriness
|
|
513
|
+
objective = (padded_objective + refrac_mask)[
|
|
514
|
+
:-1, self.obj_pad_len : -self.obj_pad_len
|
|
515
|
+
]
|
|
516
|
+
if unit_mask is not None:
|
|
517
|
+
objective[torch.logical_not(unit_mask)] = -torch.inf
|
|
518
|
+
|
|
519
|
+
# find peaks in the coarse objective
|
|
520
|
+
assert self.thresholdsq is not None
|
|
521
|
+
if padded_scalings is None:
|
|
522
|
+
scalings = None
|
|
523
|
+
else:
|
|
524
|
+
scalings = padded_scalings[:, self.obj_pad_len : -self.obj_pad_len]
|
|
525
|
+
coarse_peaks = chunk_template_data.coarse_match(
|
|
526
|
+
objective=objective,
|
|
527
|
+
scalings=scalings,
|
|
528
|
+
thresholdsq=self.thresholdsq,
|
|
529
|
+
obj_arange=self.b.obj_arange,
|
|
530
|
+
)
|
|
531
|
+
if coarse_only or not coarse_peaks.n_spikes:
|
|
532
|
+
return coarse_peaks
|
|
533
|
+
|
|
534
|
+
# high-res peaks (upsampled or grouped)
|
|
535
|
+
fine_peaks = chunk_template_data.fine_match(
|
|
536
|
+
peaks=coarse_peaks,
|
|
537
|
+
residual=residual,
|
|
538
|
+
conv=padded_conv,
|
|
539
|
+
padding=self.obj_pad_len,
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
return fine_peaks
|
|
543
|
+
|
|
544
|
+
def pick_threshold(self):
|
|
545
|
+
if self.is_scaling and self.amplitude_scaling_variance < torch.inf:
|
|
546
|
+
# adjust threshold by the scaling prior's constant term
|
|
547
|
+
# nb, everything is x2 so halves are gone.
|
|
548
|
+
scstd = np.sqrt(self.amplitude_scaling_variance)
|
|
549
|
+
norm_const = -(np.log(2.0) + np.log(np.pi) + 2.0 * np.log(scstd))
|
|
550
|
+
|
|
551
|
+
# adjust by boundary
|
|
552
|
+
nm = norm(loc=1.0, scale=scstd)
|
|
553
|
+
p_up = nm.cdf(self.amp_scale_max)
|
|
554
|
+
p_lo = nm.cdf(self.amp_scale_min)
|
|
555
|
+
Z = p_up - p_lo
|
|
556
|
+
|
|
557
|
+
scale_const = norm_const - 2.0 * np.log(scstd) * np.log(Z)
|
|
558
|
+
|
|
559
|
+
if isinstance(self.threshold, float):
|
|
560
|
+
tb = np.sqrt(self.threshold**2 - scale_const)
|
|
561
|
+
_msg = f"In norm units, that's from {self.threshold:0.2f}->{tb:0.2f}"
|
|
562
|
+
else:
|
|
563
|
+
_msg = ""
|
|
564
|
+
logger.dartsortdebug(
|
|
565
|
+
f"matching: Amplitude scaling with std {scstd:0.4f} adjusts "
|
|
566
|
+
f"theshold by {scale_const:0.4f} in squared units. {_msg}"
|
|
567
|
+
)
|
|
568
|
+
else:
|
|
569
|
+
scale_const = 0.0
|
|
570
|
+
|
|
571
|
+
if isinstance(self.threshold, float):
|
|
572
|
+
self.thresholdsq = self.threshold**2 - scale_const
|
|
573
|
+
return
|
|
574
|
+
|
|
575
|
+
assert self.threshold == "fp_control"
|
|
576
|
+
from ..util import noise_util
|
|
577
|
+
|
|
578
|
+
# fit noise to residuals from the previous detection step
|
|
579
|
+
assert self.parent_sorting_hdf5_path is not None
|
|
580
|
+
assert self.matching_templates is not None
|
|
581
|
+
lrt: LowRankTemplates = self.matching_templates.obj_lrts # type: ignore
|
|
582
|
+
self.thresholdsq = noise_util.fp_control_threshold_from_h5(
|
|
583
|
+
hdf5_path=self.parent_sorting_hdf5_path,
|
|
584
|
+
low_rank_templates=lrt.shift_to_best_channels(
|
|
585
|
+
self.recording.get_channel_locations(), self.matching_templates.rgeom
|
|
586
|
+
),
|
|
587
|
+
rg=self.fit_subsampling_random_state,
|
|
588
|
+
refractory_radius_frames=self.refractory_radius_frames,
|
|
589
|
+
num_frames=self.recording.get_num_samples(), # TODO: wrong in subsampled case.
|
|
590
|
+
max_fp_per_input_spike=self.max_fp_per_input_spike,
|
|
591
|
+
unit_ids=self.matching_templates.obj_unit_ids,
|
|
592
|
+
spike_counts=self.obj_spike_counts,
|
|
593
|
+
)
|
|
594
|
+
self.thresholdsq -= scale_const
|
|
595
|
+
logger.info(
|
|
596
|
+
f"Matcher picked threshold^2 {self.thresholdsq} for strategy fp_control."
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def peaks_to_batch_result(
|
|
601
|
+
peaks: MatchingPeaks,
|
|
602
|
+
trough_offset_samples: int,
|
|
603
|
+
unit_ids: Tensor,
|
|
604
|
+
trough_shifts: Tensor | None,
|
|
605
|
+
) -> PeelingBatchResult:
|
|
606
|
+
if not peaks.n_spikes:
|
|
607
|
+
return PeelingBatchResult(n_spikes=0)
|
|
608
|
+
|
|
609
|
+
assert peaks.times is not None
|
|
610
|
+
assert peaks.template_inds is not None
|
|
611
|
+
times_samples = peaks.times + trough_offset_samples
|
|
612
|
+
if trough_shifts is not None:
|
|
613
|
+
times_samples += trough_shifts
|
|
614
|
+
res = PeelingBatchResult(
|
|
615
|
+
n_spikes=peaks.n_spikes,
|
|
616
|
+
times_samples=times_samples,
|
|
617
|
+
labels=unit_ids[peaks.template_inds],
|
|
618
|
+
template_inds=peaks.template_inds,
|
|
619
|
+
)
|
|
620
|
+
if peaks.up_inds is not None:
|
|
621
|
+
res["up_inds"] = peaks.up_inds
|
|
622
|
+
if trough_shifts is not None:
|
|
623
|
+
res["time_shifts"] = trough_shifts
|
|
624
|
+
if peaks.scalings is not None:
|
|
625
|
+
res["scalings"] = peaks.scalings
|
|
626
|
+
if peaks.scores is not None:
|
|
627
|
+
res["scores"] = peaks.scores
|
|
628
|
+
return res
|