DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1123 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import tempfile
|
|
3
|
+
import warnings
|
|
4
|
+
from collections import namedtuple
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
import torch.nn.functional as F
|
|
10
|
+
from spikeinterface.core import BaseRecording
|
|
11
|
+
|
|
12
|
+
from ..detect import (
|
|
13
|
+
convexity_filter,
|
|
14
|
+
detect_and_deduplicate,
|
|
15
|
+
singlechan_template_detect_and_deduplicate,
|
|
16
|
+
)
|
|
17
|
+
from ..transform import (
|
|
18
|
+
SingleChannelTemplates,
|
|
19
|
+
Voltage,
|
|
20
|
+
Waveform,
|
|
21
|
+
WaveformPipeline,
|
|
22
|
+
)
|
|
23
|
+
from ..util import job_util
|
|
24
|
+
from ..util.data_util import SpikeDataset, subsample_waveforms
|
|
25
|
+
from ..util.internal_config import (
|
|
26
|
+
FeaturizationConfig,
|
|
27
|
+
FitSamplingConfig,
|
|
28
|
+
SubtractionConfig,
|
|
29
|
+
WaveformConfig,
|
|
30
|
+
)
|
|
31
|
+
from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
|
|
32
|
+
from ..util.waveform_util import (
|
|
33
|
+
get_channel_index_rel_inds,
|
|
34
|
+
get_relative_subset,
|
|
35
|
+
make_channel_index,
|
|
36
|
+
relative_channel_subset_index,
|
|
37
|
+
)
|
|
38
|
+
from .peel_base import BasePeeler, PeelingBatchResult
|
|
39
|
+
from .threshold import ThresholdAndFeaturize, threshold_chunk
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class SubtractionPeeler(BasePeeler):
|
|
43
|
+
peel_kind = "Subtraction"
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
recording,
|
|
48
|
+
channel_index,
|
|
49
|
+
subtraction_denoising_pipeline,
|
|
50
|
+
featurization_pipeline,
|
|
51
|
+
subtract_channel_index=None,
|
|
52
|
+
trough_offset_samples=42,
|
|
53
|
+
spike_length_samples=121,
|
|
54
|
+
detection_threshold=4.0,
|
|
55
|
+
chunk_length_samples=30_000,
|
|
56
|
+
peak_sign="both",
|
|
57
|
+
realign_to_denoiser=False,
|
|
58
|
+
denoiser_realignment_channel: Literal["detection", "denoised"] = "detection",
|
|
59
|
+
denoiser_realignment_shift=5,
|
|
60
|
+
relative_peak_channel_index=None,
|
|
61
|
+
spatial_dedup_channel_index=None,
|
|
62
|
+
relative_peak_radius_samples=5,
|
|
63
|
+
temporal_dedup_radius_samples=7,
|
|
64
|
+
remove_exact_duplicates=True,
|
|
65
|
+
positive_temporal_dedup_radius_samples=41,
|
|
66
|
+
trough_priority: float | None = 2.0,
|
|
67
|
+
convexity_threshold=None,
|
|
68
|
+
convexity_radius=3,
|
|
69
|
+
n_seconds_fit=100,
|
|
70
|
+
max_waveforms_fit=50_000,
|
|
71
|
+
n_waveforms_fit=20_000,
|
|
72
|
+
fit_subsampling_random_state=0,
|
|
73
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
74
|
+
fit_max_reweighting=4.0,
|
|
75
|
+
residnorm_decrease_threshold=16.0,
|
|
76
|
+
decrease_objective: Literal["norm", "normsq", "deconv"] = "deconv",
|
|
77
|
+
growth_tolerance: float | None = 0.1,
|
|
78
|
+
use_singlechan_templates=False,
|
|
79
|
+
n_singlechan_templates=10,
|
|
80
|
+
singlechan_threshold=40.0,
|
|
81
|
+
singlechan_alignment_padding=20,
|
|
82
|
+
cumulant_order=None,
|
|
83
|
+
first_denoiser_max_waveforms_fit=250_000,
|
|
84
|
+
first_denoiser_thinning=0.5,
|
|
85
|
+
first_denoiser_temporal_jitter=3,
|
|
86
|
+
first_denoiser_spatial_jitter=35.0,
|
|
87
|
+
save_iteration=False,
|
|
88
|
+
save_residnorm_decrease=False,
|
|
89
|
+
max_iter=100,
|
|
90
|
+
dtype=torch.float,
|
|
91
|
+
):
|
|
92
|
+
if realign_to_denoiser:
|
|
93
|
+
fixed_property_keys = ("channels", "time_shifts")
|
|
94
|
+
else:
|
|
95
|
+
fixed_property_keys = ("channels",)
|
|
96
|
+
|
|
97
|
+
super().__init__(
|
|
98
|
+
recording=recording,
|
|
99
|
+
channel_index=channel_index,
|
|
100
|
+
featurization_pipeline=featurization_pipeline,
|
|
101
|
+
chunk_length_samples=chunk_length_samples,
|
|
102
|
+
chunk_margin_samples=self.next_margin(2 * spike_length_samples),
|
|
103
|
+
n_seconds_fit=n_seconds_fit,
|
|
104
|
+
max_waveforms_fit=max_waveforms_fit,
|
|
105
|
+
fit_subsampling_random_state=fit_subsampling_random_state,
|
|
106
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
107
|
+
fit_max_reweighting=fit_max_reweighting,
|
|
108
|
+
fit_sampling=fit_sampling,
|
|
109
|
+
trough_offset_samples=trough_offset_samples,
|
|
110
|
+
spike_length_samples=spike_length_samples,
|
|
111
|
+
fixed_property_keys=fixed_property_keys,
|
|
112
|
+
dtype=dtype,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
self.relative_peak_radius_samples = relative_peak_radius_samples
|
|
116
|
+
self.peak_sign = peak_sign
|
|
117
|
+
self.realign_to_denoiser = realign_to_denoiser
|
|
118
|
+
self.denoiser_realignment_shift = denoiser_realignment_shift
|
|
119
|
+
self.detection_threshold = detection_threshold
|
|
120
|
+
self.residnorm_decrease_threshold = residnorm_decrease_threshold
|
|
121
|
+
self.temporal_dedup_radius_samples = temporal_dedup_radius_samples
|
|
122
|
+
self.remove_exact_duplicates = remove_exact_duplicates
|
|
123
|
+
self.positive_temporal_dedup_radius_samples = (
|
|
124
|
+
positive_temporal_dedup_radius_samples
|
|
125
|
+
)
|
|
126
|
+
self.trough_priority = trough_priority
|
|
127
|
+
self.growth_tolerance = growth_tolerance
|
|
128
|
+
self.convexity_threshold = convexity_threshold
|
|
129
|
+
self.convexity_radius = convexity_radius
|
|
130
|
+
self.save_iteration = save_iteration
|
|
131
|
+
self.save_residnorm_decrease = save_residnorm_decrease
|
|
132
|
+
self.max_iter = max_iter
|
|
133
|
+
self.decrease_objective: Literal["norm", "normsq", "deconv"] = decrease_objective
|
|
134
|
+
|
|
135
|
+
if subtract_channel_index is None:
|
|
136
|
+
subtract_channel_index = channel_index.clone().detach()
|
|
137
|
+
self.register_buffer("subtract_channel_index", subtract_channel_index)
|
|
138
|
+
self.denoiser_realignment_channel = denoiser_realignment_channel
|
|
139
|
+
self.register_buffer(
|
|
140
|
+
"subtract_index_rel_inds",
|
|
141
|
+
get_channel_index_rel_inds(subtract_channel_index),
|
|
142
|
+
)
|
|
143
|
+
self.fixed_output_data.append(
|
|
144
|
+
(
|
|
145
|
+
"subtract_channel_index",
|
|
146
|
+
self.b.subtract_channel_index.numpy(force=True).copy(),
|
|
147
|
+
),
|
|
148
|
+
)
|
|
149
|
+
self.extract_subtract_same = torch.equal(
|
|
150
|
+
self.b.subtract_channel_index, self.b.channel_index
|
|
151
|
+
)
|
|
152
|
+
if not self.extract_subtract_same:
|
|
153
|
+
esmask = relative_channel_subset_index(
|
|
154
|
+
self.subtract_channel_index, self.channel_index
|
|
155
|
+
)
|
|
156
|
+
self.register_buffer("extract_subtract_mask", torch.asarray(esmask))
|
|
157
|
+
else:
|
|
158
|
+
self.extract_subtract_mask = None
|
|
159
|
+
self.dedup_batch_size = self.nearest_batch_length()
|
|
160
|
+
if spatial_dedup_channel_index is not None:
|
|
161
|
+
self.register_buffer(
|
|
162
|
+
"spatial_dedup_channel_index", spatial_dedup_channel_index
|
|
163
|
+
)
|
|
164
|
+
self.register_buffer(
|
|
165
|
+
"spatial_dedup_rel_inds",
|
|
166
|
+
get_channel_index_rel_inds(spatial_dedup_channel_index),
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
self.spatial_dedup_channel_index = None
|
|
170
|
+
self.spatial_dedup_rel_inds = None
|
|
171
|
+
if relative_peak_channel_index is not None:
|
|
172
|
+
self.register_buffer(
|
|
173
|
+
"relative_peak_channel_index", relative_peak_channel_index
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
self.relative_peak_channel_index = None
|
|
177
|
+
self.add_module(
|
|
178
|
+
"subtraction_denoising_pipeline", subtraction_denoising_pipeline
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
# first denoiser fitting parameters
|
|
182
|
+
self.first_denoiser_max_waveforms_fit = first_denoiser_max_waveforms_fit
|
|
183
|
+
thinning_length_cond = recording.get_total_duration() >= 2 * n_seconds_fit
|
|
184
|
+
self.first_denoiser_thinning = (
|
|
185
|
+
first_denoiser_thinning if thinning_length_cond else 0.0
|
|
186
|
+
)
|
|
187
|
+
self.first_denoiser_temporal_jitter = first_denoiser_temporal_jitter
|
|
188
|
+
self.first_denoiser_spatial_jitter = first_denoiser_spatial_jitter
|
|
189
|
+
|
|
190
|
+
# cumulant detection
|
|
191
|
+
self.cumulant_order = cumulant_order
|
|
192
|
+
|
|
193
|
+
# singlechan template based detection
|
|
194
|
+
self.use_singlechan_templates = use_singlechan_templates
|
|
195
|
+
self.have_singlechan_templates = False
|
|
196
|
+
self.singlechan_threshold = singlechan_threshold
|
|
197
|
+
if use_singlechan_templates:
|
|
198
|
+
sc_feat = SingleChannelTemplates(
|
|
199
|
+
self.channel_index,
|
|
200
|
+
n_centroids=n_singlechan_templates,
|
|
201
|
+
alignment_padding=singlechan_alignment_padding,
|
|
202
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
203
|
+
max_waveforms=n_waveforms_fit,
|
|
204
|
+
)
|
|
205
|
+
if self.featurization_pipeline is None:
|
|
206
|
+
self.featurization_pipeline = WaveformPipeline([sc_feat])
|
|
207
|
+
else:
|
|
208
|
+
self.featurization_pipeline.transformers.insert(0, sc_feat)
|
|
209
|
+
|
|
210
|
+
def out_datasets(self):
|
|
211
|
+
datasets = super().out_datasets()
|
|
212
|
+
|
|
213
|
+
if self.save_iteration:
|
|
214
|
+
datasets.append(SpikeDataset("iteration", (), "int32"))
|
|
215
|
+
if self.realign_to_denoiser:
|
|
216
|
+
datasets.append(SpikeDataset("time_shifts", (), "int16"))
|
|
217
|
+
if self.save_residnorm_decrease:
|
|
218
|
+
datasets.append(SpikeDataset("residnorm_decreases", (), "float32"))
|
|
219
|
+
|
|
220
|
+
# we may be featurizing during subtraction, register the features
|
|
221
|
+
datasets.extend(self.subtraction_denoising_pipeline.spike_datasets())
|
|
222
|
+
|
|
223
|
+
return datasets
|
|
224
|
+
|
|
225
|
+
def peeling_needs_fit(self):
|
|
226
|
+
return self.subtraction_denoising_pipeline.needs_fit()
|
|
227
|
+
|
|
228
|
+
def peeling_needs_precompute(self):
|
|
229
|
+
return self.subtraction_denoising_pipeline.needs_precompute()
|
|
230
|
+
|
|
231
|
+
def save_models(self, save_folder):
|
|
232
|
+
super().save_models(save_folder)
|
|
233
|
+
sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
|
|
234
|
+
torch.save(self.subtraction_denoising_pipeline.state_dict(), sub_denoise_pt)
|
|
235
|
+
|
|
236
|
+
def load_models(self, save_folder):
|
|
237
|
+
super().load_models(save_folder)
|
|
238
|
+
sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
|
|
239
|
+
if sub_denoise_pt.exists():
|
|
240
|
+
state_dict = torch.load(sub_denoise_pt)
|
|
241
|
+
self.subtraction_denoising_pipeline.load_state_dict(state_dict)
|
|
242
|
+
|
|
243
|
+
@classmethod
|
|
244
|
+
def from_config(
|
|
245
|
+
cls,
|
|
246
|
+
*,
|
|
247
|
+
recording: BaseRecording,
|
|
248
|
+
waveform_cfg: WaveformConfig,
|
|
249
|
+
subtraction_cfg: SubtractionConfig,
|
|
250
|
+
featurization_cfg: FeaturizationConfig,
|
|
251
|
+
sampling_cfg: FitSamplingConfig,
|
|
252
|
+
):
|
|
253
|
+
# waveform extraction channel neighborhoods
|
|
254
|
+
geom = torch.tensor(recording.get_channel_locations())
|
|
255
|
+
channel_index = make_channel_index(
|
|
256
|
+
geom, featurization_cfg.extract_radius, to_torch=True
|
|
257
|
+
)
|
|
258
|
+
subtract_channel_index = make_channel_index(
|
|
259
|
+
geom, subtraction_cfg.subtract_radius, to_torch=True
|
|
260
|
+
)
|
|
261
|
+
# per-threshold spike event deduplication channel neighborhoods
|
|
262
|
+
if subtraction_cfg.spatial_dedup_radius is not None:
|
|
263
|
+
spatial_dedup_channel_index = make_channel_index(
|
|
264
|
+
geom, subtraction_cfg.spatial_dedup_radius, to_torch=True
|
|
265
|
+
)
|
|
266
|
+
else:
|
|
267
|
+
spatial_dedup_channel_index = None
|
|
268
|
+
|
|
269
|
+
relative_peak_channel_index = None
|
|
270
|
+
if subtraction_cfg.relative_peak_radius_um:
|
|
271
|
+
relative_peak_channel_index = make_channel_index(
|
|
272
|
+
geom, subtraction_cfg.relative_peak_radius_um, to_torch=True
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# construct denoising and featurization pipelines
|
|
276
|
+
subtraction_denoising_pipeline = WaveformPipeline.from_config(
|
|
277
|
+
geom=geom,
|
|
278
|
+
channel_index=subtract_channel_index,
|
|
279
|
+
featurization_cfg=subtraction_cfg.subtraction_denoising_cfg,
|
|
280
|
+
waveform_cfg=waveform_cfg,
|
|
281
|
+
sampling_frequency=recording.sampling_frequency,
|
|
282
|
+
)
|
|
283
|
+
featurization_pipeline = WaveformPipeline.from_config(
|
|
284
|
+
geom=geom,
|
|
285
|
+
channel_index=channel_index,
|
|
286
|
+
featurization_cfg=featurization_cfg,
|
|
287
|
+
waveform_cfg=waveform_cfg,
|
|
288
|
+
sampling_frequency=recording.sampling_frequency,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
# waveform logic
|
|
292
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(
|
|
293
|
+
recording.sampling_frequency
|
|
294
|
+
)
|
|
295
|
+
spike_length_samples = waveform_cfg.spike_length_samples(
|
|
296
|
+
recording.sampling_frequency
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
if trough_offset_samples != 42 or spike_length_samples != 121:
|
|
300
|
+
# temporary warning just so I can see if this happens
|
|
301
|
+
warnings.warn(
|
|
302
|
+
f"waveform_cfg {trough_offset_samples=} {spike_length_samples=} "
|
|
303
|
+
f"since {recording.sampling_frequency=}"
|
|
304
|
+
)
|
|
305
|
+
singlechan_alignment_padding = int(
|
|
306
|
+
subtraction_cfg.singlechan_alignment_padding_ms
|
|
307
|
+
* (recording.sampling_frequency / 1000)
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
return cls(
|
|
311
|
+
recording=recording,
|
|
312
|
+
channel_index=channel_index,
|
|
313
|
+
subtraction_denoising_pipeline=subtraction_denoising_pipeline,
|
|
314
|
+
featurization_pipeline=featurization_pipeline,
|
|
315
|
+
subtract_channel_index=subtract_channel_index,
|
|
316
|
+
trough_offset_samples=trough_offset_samples,
|
|
317
|
+
spike_length_samples=spike_length_samples,
|
|
318
|
+
detection_threshold=subtraction_cfg.detection_threshold,
|
|
319
|
+
chunk_length_samples=subtraction_cfg.chunk_length_samples,
|
|
320
|
+
peak_sign=subtraction_cfg.peak_sign,
|
|
321
|
+
relative_peak_radius_samples=subtraction_cfg.relative_peak_radius_samples,
|
|
322
|
+
realign_to_denoiser=subtraction_cfg.realign_to_denoiser,
|
|
323
|
+
denoiser_realignment_shift=subtraction_cfg.denoiser_realignment_shift,
|
|
324
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
325
|
+
spatial_dedup_channel_index=spatial_dedup_channel_index,
|
|
326
|
+
temporal_dedup_radius_samples=subtraction_cfg.temporal_dedup_radius_samples,
|
|
327
|
+
remove_exact_duplicates=subtraction_cfg.remove_exact_duplicates,
|
|
328
|
+
positive_temporal_dedup_radius_samples=subtraction_cfg.positive_temporal_dedup_radius_samples,
|
|
329
|
+
n_seconds_fit=subtraction_cfg.n_seconds_fit,
|
|
330
|
+
max_waveforms_fit=sampling_cfg.max_waveforms_fit,
|
|
331
|
+
fit_sampling=sampling_cfg.fit_sampling,
|
|
332
|
+
fit_max_reweighting=sampling_cfg.fit_max_reweighting,
|
|
333
|
+
n_waveforms_fit=sampling_cfg.n_waveforms_fit,
|
|
334
|
+
fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
|
|
335
|
+
residnorm_decrease_threshold=subtraction_cfg.residnorm_decrease_threshold,
|
|
336
|
+
convexity_threshold=subtraction_cfg.convexity_threshold,
|
|
337
|
+
convexity_radius=subtraction_cfg.convexity_radius,
|
|
338
|
+
use_singlechan_templates=subtraction_cfg.use_singlechan_templates,
|
|
339
|
+
n_singlechan_templates=subtraction_cfg.n_singlechan_templates,
|
|
340
|
+
singlechan_threshold=subtraction_cfg.singlechan_threshold,
|
|
341
|
+
singlechan_alignment_padding=singlechan_alignment_padding,
|
|
342
|
+
cumulant_order=subtraction_cfg.cumulant_order,
|
|
343
|
+
first_denoiser_max_waveforms_fit=subtraction_cfg.first_denoiser_max_waveforms_fit,
|
|
344
|
+
first_denoiser_thinning=subtraction_cfg.first_denoiser_thinning,
|
|
345
|
+
first_denoiser_temporal_jitter=subtraction_cfg.first_denoiser_temporal_jitter,
|
|
346
|
+
first_denoiser_spatial_jitter=subtraction_cfg.first_denoiser_spatial_jitter,
|
|
347
|
+
growth_tolerance=subtraction_cfg.growth_tolerance,
|
|
348
|
+
trough_priority=subtraction_cfg.trough_priority,
|
|
349
|
+
save_iteration=subtraction_cfg.save_iteration,
|
|
350
|
+
save_residnorm_decrease=subtraction_cfg.save_residnorm_decrease,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
def peel_chunk(
|
|
354
|
+
self,
|
|
355
|
+
traces,
|
|
356
|
+
*,
|
|
357
|
+
chunk_start_samples=0,
|
|
358
|
+
left_margin=0,
|
|
359
|
+
right_margin=0,
|
|
360
|
+
return_residual=False,
|
|
361
|
+
return_waveforms=True,
|
|
362
|
+
):
|
|
363
|
+
del return_waveforms # always done here
|
|
364
|
+
|
|
365
|
+
extract_index = None if self.extract_subtract_same else self.channel_index
|
|
366
|
+
traces = traces.to(self.dtype)
|
|
367
|
+
if self.have_singlechan_templates:
|
|
368
|
+
assert self.featurization_pipeline is not None
|
|
369
|
+
sc_feat = self.featurization_pipeline.transformers[0]
|
|
370
|
+
assert isinstance(sc_feat, SingleChannelTemplates)
|
|
371
|
+
singlechan_kw = dict(
|
|
372
|
+
singlechan_templates=sc_feat.templates,
|
|
373
|
+
singlechan_threshold=self.singlechan_threshold,
|
|
374
|
+
singlechan_trough_offset=sc_feat.template_trough,
|
|
375
|
+
)
|
|
376
|
+
else:
|
|
377
|
+
singlechan_kw = {}
|
|
378
|
+
|
|
379
|
+
subtraction_result = subtract_chunk(
|
|
380
|
+
traces,
|
|
381
|
+
self.subtract_channel_index,
|
|
382
|
+
self.subtraction_denoising_pipeline,
|
|
383
|
+
extract_index=extract_index,
|
|
384
|
+
extract_mask=self.extract_subtract_mask,
|
|
385
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
386
|
+
spike_length_samples=self.spike_length_samples,
|
|
387
|
+
left_margin=left_margin,
|
|
388
|
+
right_margin=right_margin,
|
|
389
|
+
detection_threshold=self.detection_threshold,
|
|
390
|
+
relative_peak_radius=self.relative_peak_radius_samples,
|
|
391
|
+
peak_sign=self.peak_sign,
|
|
392
|
+
relative_peak_channel_index=self.relative_peak_channel_index,
|
|
393
|
+
spatial_dedup_channel_index=self.spatial_dedup_channel_index,
|
|
394
|
+
dedup_batch_size=self.dedup_batch_size,
|
|
395
|
+
dedup_temporal_radius=self.temporal_dedup_radius_samples,
|
|
396
|
+
remove_exact_duplicates=self.remove_exact_duplicates,
|
|
397
|
+
pos_dedup_temporal_radius=self.positive_temporal_dedup_radius_samples,
|
|
398
|
+
residnorm_decrease_threshold=self.residnorm_decrease_threshold,
|
|
399
|
+
decrease_objective=self.decrease_objective,
|
|
400
|
+
trough_priority=self.trough_priority,
|
|
401
|
+
growth_tolerance=self.growth_tolerance,
|
|
402
|
+
cumulant_order=self.cumulant_order,
|
|
403
|
+
convexity_threshold=self.convexity_threshold,
|
|
404
|
+
convexity_radius=self.convexity_radius,
|
|
405
|
+
save_iteration=self.save_iteration,
|
|
406
|
+
save_residnorm_decrease=self.save_residnorm_decrease,
|
|
407
|
+
max_iter=self.max_iter,
|
|
408
|
+
subtract_rel_inds=self.subtract_index_rel_inds,
|
|
409
|
+
spatial_dedup_rel_inds=self.spatial_dedup_rel_inds,
|
|
410
|
+
realign_to_denoiser=self.realign_to_denoiser,
|
|
411
|
+
denoiser_realignment_shift=self.denoiser_realignment_shift,
|
|
412
|
+
denoiser_realignment_channel=self.denoiser_realignment_channel,
|
|
413
|
+
**singlechan_kw, # type: ignore
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
# add in chunk_start_samples
|
|
417
|
+
times_samples = subtraction_result.times_samples + chunk_start_samples
|
|
418
|
+
|
|
419
|
+
peel_result = PeelingBatchResult(
|
|
420
|
+
n_spikes=subtraction_result.n_spikes,
|
|
421
|
+
times_samples=times_samples,
|
|
422
|
+
channels=subtraction_result.channels,
|
|
423
|
+
collisioncleaned_waveforms=subtraction_result.collisioncleaned_waveforms,
|
|
424
|
+
)
|
|
425
|
+
peel_result.update(subtraction_result.features)
|
|
426
|
+
if return_residual:
|
|
427
|
+
peel_result["residual"] = subtraction_result.residual
|
|
428
|
+
|
|
429
|
+
return peel_result
|
|
430
|
+
|
|
431
|
+
def precompute_peeler_models(
|
|
432
|
+
self, save_folder, overwrite=False, computation_cfg=None
|
|
433
|
+
):
|
|
434
|
+
self.subtraction_denoising_pipeline.precompute()
|
|
435
|
+
|
|
436
|
+
def fit_featurization_pipeline(self, tmp_dir=None, computation_cfg=None):
|
|
437
|
+
super().fit_featurization_pipeline(
|
|
438
|
+
tmp_dir=tmp_dir, computation_cfg=computation_cfg
|
|
439
|
+
)
|
|
440
|
+
if self.use_singlechan_templates:
|
|
441
|
+
self.have_singlechan_templates = True
|
|
442
|
+
|
|
443
|
+
def fit_peeler_models(self, save_folder, tmp_dir=None, computation_cfg=None):
|
|
444
|
+
# when fitting peelers for subtraction, there are basically
|
|
445
|
+
# two cases. fitting featurizers is easy -- they don't modify
|
|
446
|
+
# the waveforms. fitting denoisers is hard -- they do. each
|
|
447
|
+
# denoiser that needs fitting will affect any transformer
|
|
448
|
+
# that comes after it in the pipeline.
|
|
449
|
+
|
|
450
|
+
# but, we usually only have one denoiser that needs fitting
|
|
451
|
+
# in the pipeline, which is temporal PCA (after pretrained NN)
|
|
452
|
+
# so we will cheat for now:
|
|
453
|
+
# just remove all the denoisers that need fitting, run peeling,
|
|
454
|
+
# and fit everything
|
|
455
|
+
|
|
456
|
+
while self._fit_subtraction_transformers(
|
|
457
|
+
save_folder,
|
|
458
|
+
tmp_dir=tmp_dir,
|
|
459
|
+
computation_cfg=computation_cfg,
|
|
460
|
+
which="denoisers",
|
|
461
|
+
):
|
|
462
|
+
pass
|
|
463
|
+
self._fit_subtraction_transformers(
|
|
464
|
+
save_folder,
|
|
465
|
+
tmp_dir=tmp_dir,
|
|
466
|
+
computation_cfg=computation_cfg,
|
|
467
|
+
which="featurizers",
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
gc.collect()
|
|
471
|
+
torch.cuda.empty_cache()
|
|
472
|
+
|
|
473
|
+
def _fit_subtraction_transformers(
|
|
474
|
+
self, save_folder, tmp_dir=None, computation_cfg=None, which="denoisers"
|
|
475
|
+
):
|
|
476
|
+
"""Fit models which are run during the subtraction step
|
|
477
|
+
|
|
478
|
+
These include denoisers and featurizers. Featurizers are easy, we just fit them
|
|
479
|
+
to the extracted waveforms output from a mini-subtraction. Denoisers are a bit
|
|
480
|
+
harder, since they influence the actual waveforms that are extracted. In that sense,
|
|
481
|
+
you need to fit them serially with a new mini subtraction each time.
|
|
482
|
+
"""
|
|
483
|
+
if which == "denoisers":
|
|
484
|
+
needs_fit = any(
|
|
485
|
+
t.is_denoiser and t.needs_fit()
|
|
486
|
+
for t in self.subtraction_denoising_pipeline
|
|
487
|
+
)
|
|
488
|
+
elif which == "featurizers":
|
|
489
|
+
assert not any(
|
|
490
|
+
t.is_denoiser and t.needs_fit()
|
|
491
|
+
for t in self.subtraction_denoising_pipeline
|
|
492
|
+
)
|
|
493
|
+
needs_fit = any(
|
|
494
|
+
t.is_featurizer and t.needs_fit()
|
|
495
|
+
for t in self.subtraction_denoising_pipeline
|
|
496
|
+
)
|
|
497
|
+
else:
|
|
498
|
+
assert False
|
|
499
|
+
|
|
500
|
+
if not needs_fit:
|
|
501
|
+
return False
|
|
502
|
+
|
|
503
|
+
if computation_cfg is None:
|
|
504
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
505
|
+
device = computation_cfg.actual_device()
|
|
506
|
+
|
|
507
|
+
orig_denoise = self.subtraction_denoising_pipeline
|
|
508
|
+
init_voltage_feature = Voltage(
|
|
509
|
+
channel_index=self.subtract_channel_index,
|
|
510
|
+
name="subtract_fit_voltages",
|
|
511
|
+
)
|
|
512
|
+
init_waveform_feature = Waveform(
|
|
513
|
+
channel_index=self.subtract_channel_index,
|
|
514
|
+
name="subtract_fit_waveforms",
|
|
515
|
+
)
|
|
516
|
+
ifeats = [init_voltage_feature, init_waveform_feature]
|
|
517
|
+
if which == "denoisers":
|
|
518
|
+
# add all the already fitted denoisers until we hit the next unfitted one
|
|
519
|
+
already_fitted = []
|
|
520
|
+
fit_feats = []
|
|
521
|
+
for t in orig_denoise:
|
|
522
|
+
if t.is_denoiser:
|
|
523
|
+
if t.needs_fit():
|
|
524
|
+
fit_feats = [t]
|
|
525
|
+
break
|
|
526
|
+
already_fitted.append(t)
|
|
527
|
+
|
|
528
|
+
# this is the sequence of transforms to actually use in fitting
|
|
529
|
+
fit_feats = already_fitted + fit_feats
|
|
530
|
+
else:
|
|
531
|
+
fit_feats = None
|
|
532
|
+
already_fitted = [t for t in orig_denoise if t.is_denoiser]
|
|
533
|
+
|
|
534
|
+
# if fitting the very first denoiser...
|
|
535
|
+
if which == "denoisers" and not already_fitted:
|
|
536
|
+
assert fit_feats is not None
|
|
537
|
+
fit_pipeline = WaveformPipeline(fit_feats)
|
|
538
|
+
self._threshold_to_fit(
|
|
539
|
+
tmp_dir, fit_pipeline, computation_cfg=computation_cfg
|
|
540
|
+
)
|
|
541
|
+
return True
|
|
542
|
+
|
|
543
|
+
self.subtraction_denoising_pipeline = WaveformPipeline(ifeats + already_fitted)
|
|
544
|
+
|
|
545
|
+
# and we don't need any features for this
|
|
546
|
+
orig_featurization_pipeline = self.featurization_pipeline
|
|
547
|
+
self.featurization_pipeline = WaveformPipeline([])
|
|
548
|
+
|
|
549
|
+
# run mini subtraction
|
|
550
|
+
with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
|
|
551
|
+
temp_hdf5_filename = Path(temp_dir) / f"subtraction_{which[:-1]}_fit.h5"
|
|
552
|
+
try:
|
|
553
|
+
self.run_subsampled_peeling(
|
|
554
|
+
temp_hdf5_filename,
|
|
555
|
+
computation_cfg=computation_cfg,
|
|
556
|
+
task_name=f"Load examples for {which[:-1]} fitting",
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
# fit featurization pipeline and reassign
|
|
560
|
+
# work in a try finally so we can delete the temp file
|
|
561
|
+
# in case of an issue or a keyboard interrupt
|
|
562
|
+
waveforms, fixed_properties = subsample_waveforms(
|
|
563
|
+
temp_hdf5_filename,
|
|
564
|
+
fit_sampling=self.fit_sampling,
|
|
565
|
+
random_state=self.fit_subsampling_random_state,
|
|
566
|
+
n_waveforms_fit=self.n_waveforms_fit,
|
|
567
|
+
fit_max_reweighting=self.fit_max_reweighting,
|
|
568
|
+
voltages_dataset_name="subtract_fit_voltages",
|
|
569
|
+
waveforms_dataset_name="subtract_fit_waveforms",
|
|
570
|
+
device="cpu" if which == "denoisers" else device,
|
|
571
|
+
)
|
|
572
|
+
# these are on CPU for now.
|
|
573
|
+
assert fit_feats is not None
|
|
574
|
+
fit_denoise = WaveformPipeline(fit_feats)
|
|
575
|
+
fit_denoise = fit_denoise.to(device)
|
|
576
|
+
fit_denoise.fit(
|
|
577
|
+
recording=self.recording, waveforms=waveforms, **fixed_properties
|
|
578
|
+
)
|
|
579
|
+
fit_denoise = fit_denoise.to("cpu")
|
|
580
|
+
self.subtraction_denoising_pipeline = orig_denoise
|
|
581
|
+
self.featurization_pipeline = orig_featurization_pipeline
|
|
582
|
+
finally:
|
|
583
|
+
self.to("cpu")
|
|
584
|
+
if temp_hdf5_filename.exists():
|
|
585
|
+
temp_hdf5_filename.unlink()
|
|
586
|
+
return True
|
|
587
|
+
|
|
588
|
+
def _threshold_to_fit(self, tmp_dir, fit_pipeline, computation_cfg):
|
|
589
|
+
geom = self.recording.get_channel_locations()
|
|
590
|
+
spatial_jitter_index = None
|
|
591
|
+
if self.first_denoiser_spatial_jitter:
|
|
592
|
+
spatial_jitter_index = make_channel_index(
|
|
593
|
+
geom, self.first_denoiser_spatial_jitter, to_torch=True
|
|
594
|
+
)
|
|
595
|
+
waveform_pipeline = WaveformPipeline([Waveform(self.subtract_channel_index)])
|
|
596
|
+
trainer = ThresholdAndFeaturize(
|
|
597
|
+
self.recording,
|
|
598
|
+
detection_threshold=self.detection_threshold,
|
|
599
|
+
channel_index=self.subtract_channel_index,
|
|
600
|
+
relative_peak_channel_index=self.relative_peak_channel_index,
|
|
601
|
+
spatial_dedup_channel_index=self.spatial_dedup_channel_index,
|
|
602
|
+
relative_peak_radius_samples=self.relative_peak_radius_samples,
|
|
603
|
+
featurization_pipeline=waveform_pipeline,
|
|
604
|
+
temporal_dedup_radius_samples=self.spike_length_samples,
|
|
605
|
+
thinning=self.first_denoiser_thinning,
|
|
606
|
+
time_jitter=self.first_denoiser_temporal_jitter,
|
|
607
|
+
spatial_jitter_channel_index=spatial_jitter_index,
|
|
608
|
+
peak_sign=self.peak_sign,
|
|
609
|
+
trough_priority=self.trough_priority,
|
|
610
|
+
convexity_radius=self.convexity_radius,
|
|
611
|
+
convexity_threshold=self.convexity_threshold,
|
|
612
|
+
cumulant_order=self.cumulant_order,
|
|
613
|
+
remove_exact_duplicates=self.remove_exact_duplicates,
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
|
|
617
|
+
temp_hdf5_filename = Path(temp_dir) / f"subtraction_denoiser0_fit.h5"
|
|
618
|
+
try:
|
|
619
|
+
trainer.peel(
|
|
620
|
+
temp_hdf5_filename,
|
|
621
|
+
shuffle=True,
|
|
622
|
+
stop_after_n_waveforms=self.first_denoiser_max_waveforms_fit,
|
|
623
|
+
task_name=f"Load examples for initial denoiser fitting",
|
|
624
|
+
computation_cfg=computation_cfg,
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
# get fit weights
|
|
628
|
+
device = computation_cfg.actual_device()
|
|
629
|
+
waveforms, fixed_properties = subsample_waveforms(
|
|
630
|
+
temp_hdf5_filename,
|
|
631
|
+
fit_sampling=self.fit_sampling,
|
|
632
|
+
random_state=self.fit_subsampling_random_state,
|
|
633
|
+
n_waveforms_fit=self.first_denoiser_max_waveforms_fit,
|
|
634
|
+
fit_max_reweighting=self.fit_max_reweighting,
|
|
635
|
+
voltages_dataset_name="voltages",
|
|
636
|
+
waveforms_dataset_name="waveforms",
|
|
637
|
+
subsample_by_weighting=True,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
# fit the thing
|
|
641
|
+
fit_pipeline = fit_pipeline.to(device)
|
|
642
|
+
fit_pipeline.fit(
|
|
643
|
+
recording=self.recording, waveforms=waveforms, **fixed_properties
|
|
644
|
+
)
|
|
645
|
+
fit_pipeline.to("cpu")
|
|
646
|
+
finally:
|
|
647
|
+
if temp_hdf5_filename.exists():
|
|
648
|
+
temp_hdf5_filename.unlink()
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
ChunkSubtractionResult = namedtuple(
|
|
652
|
+
"ChunkSubtractionResult",
|
|
653
|
+
[
|
|
654
|
+
"n_spikes",
|
|
655
|
+
"times_samples",
|
|
656
|
+
"channels",
|
|
657
|
+
"collisioncleaned_waveforms",
|
|
658
|
+
"residual",
|
|
659
|
+
"features",
|
|
660
|
+
],
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def subtract_chunk(
|
|
665
|
+
traces,
|
|
666
|
+
channel_index,
|
|
667
|
+
denoising_pipeline,
|
|
668
|
+
extract_index=None,
|
|
669
|
+
extract_mask=None,
|
|
670
|
+
trough_offset_samples=42,
|
|
671
|
+
spike_length_samples=121,
|
|
672
|
+
left_margin=0,
|
|
673
|
+
right_margin=0,
|
|
674
|
+
detection_threshold=4.0,
|
|
675
|
+
peak_sign="both",
|
|
676
|
+
realign_to_denoiser=False,
|
|
677
|
+
denoiser_realignment_shift=5,
|
|
678
|
+
denoiser_realignment_channel="detection",
|
|
679
|
+
convexity_threshold=None,
|
|
680
|
+
convexity_radius=3,
|
|
681
|
+
relative_peak_channel_index=None,
|
|
682
|
+
spatial_dedup_channel_index=None,
|
|
683
|
+
subtract_rel_inds=None,
|
|
684
|
+
spatial_dedup_rel_inds=None,
|
|
685
|
+
residnorm_decrease_threshold=16.0,
|
|
686
|
+
decrease_objective: Literal["norm", "normsq", "deconv"] = "deconv",
|
|
687
|
+
relative_peak_radius=5,
|
|
688
|
+
dedup_temporal_radius=7,
|
|
689
|
+
remove_exact_duplicates=True,
|
|
690
|
+
pos_dedup_temporal_radius=None,
|
|
691
|
+
dedup_batch_size=512,
|
|
692
|
+
singlechan_templates=None,
|
|
693
|
+
singlechan_threshold=None,
|
|
694
|
+
singlechan_trough_offset=None,
|
|
695
|
+
no_subtraction=False,
|
|
696
|
+
max_iter=100,
|
|
697
|
+
trough_priority=None,
|
|
698
|
+
growth_tolerance=None,
|
|
699
|
+
cumulant_order=None,
|
|
700
|
+
save_iteration=False,
|
|
701
|
+
save_residnorm_decrease=False,
|
|
702
|
+
):
|
|
703
|
+
"""Core peeling routine for subtraction"""
|
|
704
|
+
if no_subtraction:
|
|
705
|
+
threshold_res = threshold_chunk(
|
|
706
|
+
traces,
|
|
707
|
+
channel_index,
|
|
708
|
+
detection_threshold=detection_threshold,
|
|
709
|
+
peak_sign=peak_sign,
|
|
710
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
711
|
+
spatial_dedup_channel_index=spatial_dedup_channel_index,
|
|
712
|
+
spatial_dedup_rel_inds=spatial_dedup_rel_inds,
|
|
713
|
+
trough_offset_samples=trough_offset_samples,
|
|
714
|
+
spike_length_samples=spike_length_samples,
|
|
715
|
+
left_margin=left_margin,
|
|
716
|
+
right_margin=right_margin,
|
|
717
|
+
relative_peak_radius=relative_peak_radius,
|
|
718
|
+
temporal_dedup_radius_samples=dedup_temporal_radius,
|
|
719
|
+
dedup_batch_size=dedup_batch_size,
|
|
720
|
+
remove_exact_duplicates=remove_exact_duplicates,
|
|
721
|
+
cumulant_order=cumulant_order,
|
|
722
|
+
convexity_threshold=convexity_threshold,
|
|
723
|
+
convexity_radius=convexity_radius,
|
|
724
|
+
max_spikes_per_chunk=None,
|
|
725
|
+
quiet=False,
|
|
726
|
+
)
|
|
727
|
+
waveforms, features = denoising_pipeline(
|
|
728
|
+
threshold_res["waveforms"], channels=threshold_res["channels"]
|
|
729
|
+
)
|
|
730
|
+
return ChunkSubtractionResult(
|
|
731
|
+
n_spikes=threshold_res["n_spikes"],
|
|
732
|
+
times_samples=threshold_res["times_rel"],
|
|
733
|
+
channels=threshold_res["channels"],
|
|
734
|
+
collisioncleaned_waveforms=waveforms,
|
|
735
|
+
residual=None,
|
|
736
|
+
features=features,
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
# validate arguments to avoid confusing error messages later
|
|
740
|
+
re_extract = extract_index is not None
|
|
741
|
+
if extract_index is None:
|
|
742
|
+
extract_index = channel_index
|
|
743
|
+
else:
|
|
744
|
+
assert extract_mask is not None
|
|
745
|
+
assert 0 <= left_margin < traces.shape[0]
|
|
746
|
+
assert 0 <= right_margin < traces.shape[0]
|
|
747
|
+
assert traces.shape[1] == channel_index.shape[0]
|
|
748
|
+
if spatial_dedup_channel_index is not None:
|
|
749
|
+
assert traces.shape[1] == spatial_dedup_channel_index.shape[0]
|
|
750
|
+
|
|
751
|
+
# can only subtract spikes with trough time >=trough_offset and <max_trough
|
|
752
|
+
post_trough_samples = spike_length_samples - trough_offset_samples
|
|
753
|
+
max_trough_time = traces.shape[0] - post_trough_samples
|
|
754
|
+
|
|
755
|
+
if growth_tolerance is not None:
|
|
756
|
+
gtol = traces.abs().add_(growth_tolerance)
|
|
757
|
+
else:
|
|
758
|
+
gtol = 0.0
|
|
759
|
+
|
|
760
|
+
# initialize residual, it needs to be padded to support
|
|
761
|
+
# our channel indexing convention. this copies the input.
|
|
762
|
+
residual = F.pad(traces, (0, 1), value=torch.nan)
|
|
763
|
+
|
|
764
|
+
subtracted_waveforms = []
|
|
765
|
+
spike_times = []
|
|
766
|
+
spike_channels = []
|
|
767
|
+
spike_features = []
|
|
768
|
+
detection_mask = torch.ones_like(residual)
|
|
769
|
+
dedup_temporal_ix = torch.arange(
|
|
770
|
+
-dedup_temporal_radius, dedup_temporal_radius + 1, device=residual.device
|
|
771
|
+
)
|
|
772
|
+
pos_dedup_temporal_ix = None
|
|
773
|
+
if pos_dedup_temporal_radius:
|
|
774
|
+
pos_dedup_temporal_ix = torch.arange(
|
|
775
|
+
-pos_dedup_temporal_radius,
|
|
776
|
+
pos_dedup_temporal_radius + 1,
|
|
777
|
+
device=residual.device,
|
|
778
|
+
)
|
|
779
|
+
|
|
780
|
+
for it in range(max_iter):
|
|
781
|
+
residual_det = residual[:, :-1]
|
|
782
|
+
if it and growth_tolerance is not None:
|
|
783
|
+
residual_det = residual_det.clamp(-gtol, gtol)
|
|
784
|
+
|
|
785
|
+
if singlechan_templates is None:
|
|
786
|
+
times_samples, channels = detect_and_deduplicate( # type: ignore
|
|
787
|
+
residual_det,
|
|
788
|
+
detection_threshold,
|
|
789
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
790
|
+
dedup_channel_index=channel_index,
|
|
791
|
+
peak_sign=peak_sign,
|
|
792
|
+
relative_peak_radius=relative_peak_radius,
|
|
793
|
+
dedup_temporal_radius=spike_length_samples,
|
|
794
|
+
spatial_dedup_batch_size=dedup_batch_size,
|
|
795
|
+
remove_exact_duplicates=remove_exact_duplicates,
|
|
796
|
+
dedup_index_inds=subtract_rel_inds,
|
|
797
|
+
detection_mask=detection_mask[:, :-1] if it else None,
|
|
798
|
+
trough_priority=trough_priority,
|
|
799
|
+
cumulant_order=cumulant_order,
|
|
800
|
+
)
|
|
801
|
+
else:
|
|
802
|
+
times_samples, channels = singlechan_template_detect_and_deduplicate( # type: ignore
|
|
803
|
+
residual_det,
|
|
804
|
+
singlechan_templates,
|
|
805
|
+
threshold=singlechan_threshold, # type: ignore
|
|
806
|
+
trough_offset_samples=singlechan_trough_offset, # type: ignore
|
|
807
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
808
|
+
dedup_channel_index=channel_index,
|
|
809
|
+
relative_peak_radius=relative_peak_radius,
|
|
810
|
+
dedup_temporal_radius=spike_length_samples,
|
|
811
|
+
detection_mask=detection_mask[:, :-1] if it else None,
|
|
812
|
+
)
|
|
813
|
+
if not times_samples.numel():
|
|
814
|
+
break
|
|
815
|
+
|
|
816
|
+
if it:
|
|
817
|
+
keep = detection_mask[times_samples, channels]
|
|
818
|
+
(keep,) = keep.nonzero(as_tuple=True)
|
|
819
|
+
times_samples = times_samples[keep]
|
|
820
|
+
channels = channels[keep]
|
|
821
|
+
|
|
822
|
+
if not times_samples.numel():
|
|
823
|
+
break
|
|
824
|
+
|
|
825
|
+
keep = convexity_filter(
|
|
826
|
+
residual,
|
|
827
|
+
times_samples,
|
|
828
|
+
channels,
|
|
829
|
+
threshold=convexity_threshold,
|
|
830
|
+
radius=convexity_radius,
|
|
831
|
+
)
|
|
832
|
+
times_samples = times_samples[keep]
|
|
833
|
+
channels = channels[keep]
|
|
834
|
+
if not times_samples.numel():
|
|
835
|
+
break
|
|
836
|
+
|
|
837
|
+
voltages = residual[times_samples, channels]
|
|
838
|
+
|
|
839
|
+
# never look at these again.
|
|
840
|
+
time_ix = times_samples.unsqueeze(1) + dedup_temporal_ix
|
|
841
|
+
time_ix = time_ix.clamp_(0, traces.shape[0] - 1)
|
|
842
|
+
if spatial_dedup_channel_index is not None:
|
|
843
|
+
chan_ix = spatial_dedup_channel_index[channels]
|
|
844
|
+
else:
|
|
845
|
+
chan_ix = channels.unsqueeze(1)
|
|
846
|
+
detection_mask[time_ix[:, :, None], chan_ix[:, None, :]] = 0.0
|
|
847
|
+
|
|
848
|
+
# take extra care to exclude positive peaks appearing near stronger troughs
|
|
849
|
+
if pos_dedup_temporal_radius:
|
|
850
|
+
assert pos_dedup_temporal_ix is not None
|
|
851
|
+
(neg,) = (voltages < 0).nonzero(as_tuple=True)
|
|
852
|
+
time_ix = times_samples[neg].unsqueeze(1) + pos_dedup_temporal_ix
|
|
853
|
+
time_ix = time_ix.clamp_(0, traces.shape[0] - 1)
|
|
854
|
+
if spatial_dedup_channel_index is not None:
|
|
855
|
+
chan_ix = spatial_dedup_channel_index[channels[neg]]
|
|
856
|
+
else:
|
|
857
|
+
chan_ix = channels[neg].unsqueeze(1)
|
|
858
|
+
|
|
859
|
+
pd_mask = torch.ones_like(detection_mask)
|
|
860
|
+
pd_mask[time_ix[:, :, None], chan_ix[:, None, :]] = 0
|
|
861
|
+
pd_mask = torch.logical_or(pd_mask, residual < 0)
|
|
862
|
+
detection_mask = torch.logical_and(detection_mask, pd_mask)
|
|
863
|
+
|
|
864
|
+
# throw away spikes which cannot be subtracted
|
|
865
|
+
keep = times_samples == times_samples.clamp(
|
|
866
|
+
trough_offset_samples, max_trough_time
|
|
867
|
+
)
|
|
868
|
+
(keep,) = keep.nonzero(as_tuple=True)
|
|
869
|
+
|
|
870
|
+
if not keep.numel():
|
|
871
|
+
break
|
|
872
|
+
times_samples = times_samples[keep]
|
|
873
|
+
channels = channels[keep]
|
|
874
|
+
voltages = voltages[keep]
|
|
875
|
+
|
|
876
|
+
# -- read waveforms, denoise, and test residnorm decrease
|
|
877
|
+
waveforms = grab_spikes(
|
|
878
|
+
residual,
|
|
879
|
+
times_samples,
|
|
880
|
+
channels,
|
|
881
|
+
channel_index,
|
|
882
|
+
trough_offset=trough_offset_samples,
|
|
883
|
+
spike_length_samples=spike_length_samples,
|
|
884
|
+
buffer=0,
|
|
885
|
+
already_padded=True,
|
|
886
|
+
)
|
|
887
|
+
|
|
888
|
+
if residnorm_decrease_threshold:
|
|
889
|
+
original_waveforms = waveforms.nan_to_num()
|
|
890
|
+
else:
|
|
891
|
+
original_waveforms = None
|
|
892
|
+
|
|
893
|
+
waveforms, features = denoising_pipeline(waveforms, channels=channels)
|
|
894
|
+
|
|
895
|
+
resid_keep, new_feats = check_residual_decrease(
|
|
896
|
+
original_waveforms,
|
|
897
|
+
waveforms,
|
|
898
|
+
decrease_objective=decrease_objective,
|
|
899
|
+
threshold=residnorm_decrease_threshold,
|
|
900
|
+
save_residnorm_decrease=save_residnorm_decrease,
|
|
901
|
+
)
|
|
902
|
+
features.update(new_feats)
|
|
903
|
+
if resid_keep is not None:
|
|
904
|
+
if not resid_keep.numel():
|
|
905
|
+
break
|
|
906
|
+
if resid_keep.numel() < len(original_waveforms):
|
|
907
|
+
waveforms = waveforms[resid_keep]
|
|
908
|
+
times_samples = times_samples[resid_keep]
|
|
909
|
+
channels = channels[resid_keep]
|
|
910
|
+
voltages = voltages[resid_keep]
|
|
911
|
+
for k, ft in features.items():
|
|
912
|
+
features[k] = ft[resid_keep]
|
|
913
|
+
|
|
914
|
+
# -- subtract in place
|
|
915
|
+
residual = subtract_spikes_(
|
|
916
|
+
residual,
|
|
917
|
+
times_samples,
|
|
918
|
+
channels,
|
|
919
|
+
channel_index,
|
|
920
|
+
waveforms,
|
|
921
|
+
trough_offset=trough_offset_samples,
|
|
922
|
+
buffer=0,
|
|
923
|
+
already_padded=True,
|
|
924
|
+
in_place=True,
|
|
925
|
+
)
|
|
926
|
+
|
|
927
|
+
# -- follow the nn's realignment advice, if requested
|
|
928
|
+
if realign_to_denoiser:
|
|
929
|
+
features["time_shifts"] = denoiser_time_shifts(
|
|
930
|
+
waveforms,
|
|
931
|
+
channels,
|
|
932
|
+
voltages,
|
|
933
|
+
subtract_rel_inds,
|
|
934
|
+
trough_offset_samples,
|
|
935
|
+
spike_length_samples,
|
|
936
|
+
peak_sign,
|
|
937
|
+
denoiser_realignment_shift,
|
|
938
|
+
denoiser_realignment_channel,
|
|
939
|
+
)
|
|
940
|
+
|
|
941
|
+
# -- store this iter's outputs
|
|
942
|
+
spike_times.append(times_samples)
|
|
943
|
+
spike_channels.append(channels)
|
|
944
|
+
spike_features.append(features)
|
|
945
|
+
if save_iteration:
|
|
946
|
+
spike_features[-1]["iteration"] = torch.full_like(times_samples, it)
|
|
947
|
+
subtracted_waveforms.append(waveforms)
|
|
948
|
+
|
|
949
|
+
# check if we got no spikes
|
|
950
|
+
if not spike_times:
|
|
951
|
+
return empty_chunk_subtraction_result(
|
|
952
|
+
spike_length_samples,
|
|
953
|
+
channel_index,
|
|
954
|
+
residual[left_margin : traces.shape[0] - right_margin, :-1],
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
# concatenate all of the thresholds together into single tensors
|
|
958
|
+
spike_times = torch.concatenate(spike_times)
|
|
959
|
+
spike_channels = torch.concatenate(spike_channels)
|
|
960
|
+
subtracted_waveforms = torch.concatenate(subtracted_waveforms)
|
|
961
|
+
spike_features = {
|
|
962
|
+
k: torch.concatenate([ff[k] for ff in spike_features])
|
|
963
|
+
for k in spike_features[0].keys()
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
# discard spikes in the margins and sort times_samples for caller
|
|
967
|
+
max_valid_t = traces.shape[0] - right_margin - 1
|
|
968
|
+
keep = spike_times == spike_times.clamp(left_margin, max_valid_t)
|
|
969
|
+
(keep,) = keep.nonzero(as_tuple=True)
|
|
970
|
+
if not keep.numel():
|
|
971
|
+
return empty_chunk_subtraction_result(
|
|
972
|
+
spike_length_samples,
|
|
973
|
+
channel_index,
|
|
974
|
+
residual[left_margin : traces.shape[0] - right_margin, :-1],
|
|
975
|
+
)
|
|
976
|
+
|
|
977
|
+
keep = keep[torch.argsort(spike_times[keep])]
|
|
978
|
+
subtracted_waveforms = subtracted_waveforms[keep]
|
|
979
|
+
spike_times = spike_times[keep]
|
|
980
|
+
spike_channels = spike_channels[keep]
|
|
981
|
+
for k in spike_features:
|
|
982
|
+
spike_features[k] = spike_features[k][keep]
|
|
983
|
+
|
|
984
|
+
# if extract_index != subtract_index, re-do the channels for the subtracted wfs
|
|
985
|
+
if re_extract:
|
|
986
|
+
subtracted_waveforms = get_relative_subset(
|
|
987
|
+
subtracted_waveforms, spike_channels, extract_mask
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
# construct collision-cleaned waveforms
|
|
991
|
+
collisioncleaned_waveforms = grab_spikes(
|
|
992
|
+
residual,
|
|
993
|
+
spike_times,
|
|
994
|
+
spike_channels,
|
|
995
|
+
extract_index,
|
|
996
|
+
trough_offset=trough_offset_samples,
|
|
997
|
+
spike_length_samples=spike_length_samples,
|
|
998
|
+
buffer=0,
|
|
999
|
+
already_padded=True,
|
|
1000
|
+
)
|
|
1001
|
+
collisioncleaned_waveforms += subtracted_waveforms
|
|
1002
|
+
|
|
1003
|
+
# offset spike times_samples according to margin
|
|
1004
|
+
spike_times -= left_margin
|
|
1005
|
+
|
|
1006
|
+
# apply time shifts
|
|
1007
|
+
if "time_shifts" in spike_features:
|
|
1008
|
+
spike_times += spike_features["time_shifts"]
|
|
1009
|
+
|
|
1010
|
+
# strip margin and padding channel off the residual
|
|
1011
|
+
residual = residual[left_margin : traces.shape[0] - right_margin, :-1].cpu()
|
|
1012
|
+
|
|
1013
|
+
return ChunkSubtractionResult(
|
|
1014
|
+
n_spikes=spike_times.numel(),
|
|
1015
|
+
times_samples=spike_times,
|
|
1016
|
+
channels=spike_channels,
|
|
1017
|
+
collisioncleaned_waveforms=collisioncleaned_waveforms,
|
|
1018
|
+
residual=residual,
|
|
1019
|
+
features=spike_features,
|
|
1020
|
+
)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def empty_chunk_subtraction_result(spike_length_samples, channel_index, residual):
|
|
1024
|
+
empty_waveforms = torch.empty(
|
|
1025
|
+
(0, spike_length_samples, channel_index.shape[1]),
|
|
1026
|
+
dtype=residual.dtype,
|
|
1027
|
+
)
|
|
1028
|
+
empty_times_or_chans = torch.empty((0,), dtype=torch.long)
|
|
1029
|
+
return ChunkSubtractionResult(
|
|
1030
|
+
n_spikes=0,
|
|
1031
|
+
times_samples=empty_times_or_chans,
|
|
1032
|
+
channels=empty_times_or_chans,
|
|
1033
|
+
collisioncleaned_waveforms=empty_waveforms,
|
|
1034
|
+
residual=residual,
|
|
1035
|
+
features={},
|
|
1036
|
+
)
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
# -- utilities
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def denoiser_time_shifts(
|
|
1043
|
+
waveforms,
|
|
1044
|
+
channels,
|
|
1045
|
+
voltages,
|
|
1046
|
+
subtract_rel_inds,
|
|
1047
|
+
trough_offset_samples,
|
|
1048
|
+
spike_length_samples,
|
|
1049
|
+
peak_sign,
|
|
1050
|
+
denoiser_realignment_shift,
|
|
1051
|
+
denoiser_realignment_channel,
|
|
1052
|
+
):
|
|
1053
|
+
# extract main channel traces
|
|
1054
|
+
if denoiser_realignment_channel == "detection":
|
|
1055
|
+
assert subtract_rel_inds is not None
|
|
1056
|
+
main_channel_rel_inds = subtract_rel_inds[channels]
|
|
1057
|
+
elif denoiser_realignment_channel == "denoised":
|
|
1058
|
+
main_channel_rel_inds = ptp(waveforms).nan_to_num_(nan=-torch.inf).argmax(dim=1)
|
|
1059
|
+
else:
|
|
1060
|
+
assert False
|
|
1061
|
+
denoised_main_channel_traces = waveforms.take_along_dim(
|
|
1062
|
+
dim=2, indices=main_channel_rel_inds[:, None, None]
|
|
1063
|
+
)
|
|
1064
|
+
nwf = len(waveforms)
|
|
1065
|
+
assert denoised_main_channel_traces.shape == (nwf, spike_length_samples, 1)
|
|
1066
|
+
denoised_main_channel_traces = denoised_main_channel_traces[:, :, 0]
|
|
1067
|
+
|
|
1068
|
+
# extract window around trough
|
|
1069
|
+
start = trough_offset_samples - denoiser_realignment_shift
|
|
1070
|
+
end = trough_offset_samples + denoiser_realignment_shift + 1
|
|
1071
|
+
snips = denoised_main_channel_traces[:, start:end]
|
|
1072
|
+
assert snips.shape == (nwf, 2 * denoiser_realignment_shift + 1)
|
|
1073
|
+
|
|
1074
|
+
# handle the sign of the events so that we can align to maxima
|
|
1075
|
+
if peak_sign == "both":
|
|
1076
|
+
snips.mul_(torch.sign(voltages)[:, None])
|
|
1077
|
+
elif peak_sign == "neg":
|
|
1078
|
+
snips.neg_()
|
|
1079
|
+
else:
|
|
1080
|
+
assert peak_sign == "pos"
|
|
1081
|
+
|
|
1082
|
+
# find shifts just by argmax
|
|
1083
|
+
peaks = snips.argmax(dim=1)
|
|
1084
|
+
dt = peaks.sub_(denoiser_realignment_shift).to(dtype=torch.int16)
|
|
1085
|
+
return dt
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def check_residual_decrease(
|
|
1089
|
+
orig_wfs,
|
|
1090
|
+
dn_wfs,
|
|
1091
|
+
decrease_objective="deconv",
|
|
1092
|
+
threshold=10.0,
|
|
1093
|
+
save_residnorm_decrease=False,
|
|
1094
|
+
):
|
|
1095
|
+
if not threshold:
|
|
1096
|
+
return None, {}
|
|
1097
|
+
|
|
1098
|
+
if decrease_objective == "deconv":
|
|
1099
|
+
dn_wfs = dn_wfs.nan_to_num()
|
|
1100
|
+
conv = (orig_wfs * dn_wfs).sum(dim=(1, 2))
|
|
1101
|
+
norm = dn_wfs.square_().sum(dim=(1, 2))
|
|
1102
|
+
reduction = conv.mul_(2.0).sub_(norm)
|
|
1103
|
+
threshold = threshold ** 2
|
|
1104
|
+
elif decrease_objective in ("norm", "normsq"):
|
|
1105
|
+
orig_decobj = orig_wfs.square().sum(dim=(1, 2))
|
|
1106
|
+
orig_wfs = orig_wfs.sub_(dn_wfs).nan_to_num_()
|
|
1107
|
+
new_decobj = orig_wfs.square_().sum(dim=(1, 2))
|
|
1108
|
+
if decrease_objective == "norm":
|
|
1109
|
+
orig_decobj = orig_decobj.sqrt_()
|
|
1110
|
+
new_decobj = new_decobj.sqrt_()
|
|
1111
|
+
else:
|
|
1112
|
+
threshold = threshold ** 2
|
|
1113
|
+
reduction = orig_decobj - new_decobj
|
|
1114
|
+
else:
|
|
1115
|
+
assert False
|
|
1116
|
+
|
|
1117
|
+
keep = threshold < reduction
|
|
1118
|
+
(keep,) = keep.nonzero(as_tuple=True)
|
|
1119
|
+
if save_residnorm_decrease:
|
|
1120
|
+
features = dict(residnorm_decreases=reduction)
|
|
1121
|
+
else:
|
|
1122
|
+
features = {}
|
|
1123
|
+
return keep, features
|