DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
from spikeinterface.core import BaseRecording
|
|
7
|
+
|
|
8
|
+
from ..detect import convexity_filter, detect_and_deduplicate
|
|
9
|
+
from ..transform import WaveformPipeline
|
|
10
|
+
from ..util import spiketorch
|
|
11
|
+
from ..util.data_util import SpikeDataset
|
|
12
|
+
from ..util.internal_config import (
|
|
13
|
+
FeaturizationConfig,
|
|
14
|
+
FitSamplingConfig,
|
|
15
|
+
ThresholdingConfig,
|
|
16
|
+
WaveformConfig,
|
|
17
|
+
)
|
|
18
|
+
from ..util.waveform_util import get_channel_index_rel_inds, make_channel_index
|
|
19
|
+
from .peel_base import BasePeeler, PeelingBatchResult, peeling_empty_result
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ThresholdAndFeaturize(BasePeeler):
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
recording,
|
|
26
|
+
channel_index,
|
|
27
|
+
featurization_pipeline=None,
|
|
28
|
+
trough_offset_samples=42,
|
|
29
|
+
spike_length_samples=121,
|
|
30
|
+
detection_threshold=5.0,
|
|
31
|
+
chunk_length_samples=30_000,
|
|
32
|
+
max_spikes_per_chunk=None,
|
|
33
|
+
peak_sign="both",
|
|
34
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
35
|
+
fit_max_reweighting=4.0,
|
|
36
|
+
relative_peak_channel_index=None,
|
|
37
|
+
spatial_dedup_channel_index=None,
|
|
38
|
+
relative_peak_radius_samples=5,
|
|
39
|
+
temporal_dedup_radius_samples=7,
|
|
40
|
+
cumulant_order=None,
|
|
41
|
+
convexity_threshold=None,
|
|
42
|
+
convexity_radius=3,
|
|
43
|
+
remove_exact_duplicates=True,
|
|
44
|
+
n_seconds_fit=100,
|
|
45
|
+
n_waveforms_fit=20_000,
|
|
46
|
+
max_waveforms_fit=50_000,
|
|
47
|
+
fit_subsampling_random_state=0,
|
|
48
|
+
thinning=0.0,
|
|
49
|
+
time_jitter=0,
|
|
50
|
+
trough_priority=None,
|
|
51
|
+
spatial_jitter_channel_index=None,
|
|
52
|
+
dtype=torch.float,
|
|
53
|
+
):
|
|
54
|
+
super().__init__(
|
|
55
|
+
recording=recording,
|
|
56
|
+
channel_index=channel_index,
|
|
57
|
+
featurization_pipeline=featurization_pipeline,
|
|
58
|
+
chunk_length_samples=chunk_length_samples,
|
|
59
|
+
chunk_margin_samples=self.next_margin(spike_length_samples, factor=5),
|
|
60
|
+
n_seconds_fit=n_seconds_fit,
|
|
61
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
62
|
+
max_waveforms_fit=max_waveforms_fit,
|
|
63
|
+
fit_sampling=fit_sampling,
|
|
64
|
+
fit_subsampling_random_state=fit_subsampling_random_state,
|
|
65
|
+
fit_max_reweighting=fit_max_reweighting,
|
|
66
|
+
dtype=dtype,
|
|
67
|
+
trough_offset_samples=trough_offset_samples,
|
|
68
|
+
spike_length_samples=spike_length_samples,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
self.relative_peak_radius_samples = relative_peak_radius_samples
|
|
72
|
+
self.temporal_dedup_radius_samples = temporal_dedup_radius_samples
|
|
73
|
+
self.remove_exact_duplicates = remove_exact_duplicates
|
|
74
|
+
self.peak_sign = peak_sign
|
|
75
|
+
self.trough_priority = trough_priority
|
|
76
|
+
self.convexity_threshold = convexity_threshold
|
|
77
|
+
self.convexity_radius = convexity_radius
|
|
78
|
+
self.cumulant_order = cumulant_order
|
|
79
|
+
self.dedup_batch_size = self.nearest_batch_length()
|
|
80
|
+
if spatial_dedup_channel_index is not None:
|
|
81
|
+
self.register_buffer(
|
|
82
|
+
"spatial_dedup_channel_index", spatial_dedup_channel_index
|
|
83
|
+
)
|
|
84
|
+
self.register_buffer(
|
|
85
|
+
"spatial_dedup_rel_inds",
|
|
86
|
+
get_channel_index_rel_inds(spatial_dedup_channel_index),
|
|
87
|
+
)
|
|
88
|
+
else:
|
|
89
|
+
self.spatial_dedup_channel_index = None
|
|
90
|
+
if relative_peak_channel_index is not None:
|
|
91
|
+
self.register_buffer(
|
|
92
|
+
"relative_peak_channel_index", relative_peak_channel_index
|
|
93
|
+
)
|
|
94
|
+
else:
|
|
95
|
+
self.relative_peak_channel_index = None
|
|
96
|
+
self.detection_threshold = detection_threshold
|
|
97
|
+
self.max_spikes_per_chunk = max_spikes_per_chunk
|
|
98
|
+
self.peel_kind = f"Threshold {detection_threshold}"
|
|
99
|
+
|
|
100
|
+
# parameters for random perturbation of detections
|
|
101
|
+
if thinning:
|
|
102
|
+
assert thinning < min(
|
|
103
|
+
trough_offset_samples, spike_length_samples - trough_offset_samples
|
|
104
|
+
)
|
|
105
|
+
self.thinning = thinning
|
|
106
|
+
self.time_jitter = time_jitter
|
|
107
|
+
if spatial_jitter_channel_index is not None:
|
|
108
|
+
self.register_buffer(
|
|
109
|
+
"spatial_jitter_channel_index", spatial_jitter_channel_index
|
|
110
|
+
)
|
|
111
|
+
else:
|
|
112
|
+
self.spatial_jitter_channel_index = None
|
|
113
|
+
self.is_random = (
|
|
114
|
+
thinning or time_jitter or spatial_jitter_channel_index is not None
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
@classmethod
|
|
118
|
+
def from_config(
|
|
119
|
+
cls,
|
|
120
|
+
*,
|
|
121
|
+
recording: BaseRecording,
|
|
122
|
+
waveform_cfg: WaveformConfig,
|
|
123
|
+
thresholding_cfg: ThresholdingConfig,
|
|
124
|
+
featurization_cfg: FeaturizationConfig,
|
|
125
|
+
sampling_cfg: FitSamplingConfig,
|
|
126
|
+
):
|
|
127
|
+
geom = torch.tensor(recording.get_channel_locations())
|
|
128
|
+
channel_index = make_channel_index(
|
|
129
|
+
geom, featurization_cfg.extract_radius, to_torch=True
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
featurization_pipeline = WaveformPipeline.from_config(
|
|
133
|
+
geom=geom,
|
|
134
|
+
channel_index=channel_index,
|
|
135
|
+
featurization_cfg=featurization_cfg,
|
|
136
|
+
waveform_cfg=waveform_cfg,
|
|
137
|
+
sampling_frequency=recording.sampling_frequency,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
if thresholding_cfg.relative_peak_radius_um:
|
|
141
|
+
relative_peak_channel_index = make_channel_index(
|
|
142
|
+
geom, thresholding_cfg.relative_peak_radius_um, to_torch=True
|
|
143
|
+
)
|
|
144
|
+
else:
|
|
145
|
+
relative_peak_channel_index = None
|
|
146
|
+
|
|
147
|
+
if thresholding_cfg.spatial_dedup_radius:
|
|
148
|
+
spatial_dedup_channel_index = make_channel_index(
|
|
149
|
+
geom, thresholding_cfg.spatial_dedup_radius, to_torch=True
|
|
150
|
+
)
|
|
151
|
+
else:
|
|
152
|
+
spatial_dedup_channel_index = None
|
|
153
|
+
|
|
154
|
+
if thresholding_cfg.spatial_jitter_radius:
|
|
155
|
+
spatial_jitter_channel_index = make_channel_index(
|
|
156
|
+
geom, thresholding_cfg.spatial_jitter_radius, to_torch=True
|
|
157
|
+
)
|
|
158
|
+
else:
|
|
159
|
+
spatial_jitter_channel_index = None
|
|
160
|
+
|
|
161
|
+
# waveform logic
|
|
162
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(
|
|
163
|
+
recording.sampling_frequency
|
|
164
|
+
)
|
|
165
|
+
spike_length_samples = waveform_cfg.spike_length_samples(
|
|
166
|
+
recording.sampling_frequency
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
return cls(
|
|
170
|
+
recording,
|
|
171
|
+
channel_index,
|
|
172
|
+
featurization_pipeline=featurization_pipeline,
|
|
173
|
+
trough_offset_samples=trough_offset_samples,
|
|
174
|
+
spike_length_samples=spike_length_samples,
|
|
175
|
+
detection_threshold=thresholding_cfg.detection_threshold,
|
|
176
|
+
chunk_length_samples=thresholding_cfg.chunk_length_samples,
|
|
177
|
+
max_spikes_per_chunk=thresholding_cfg.max_spikes_per_chunk,
|
|
178
|
+
peak_sign=thresholding_cfg.peak_sign,
|
|
179
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
180
|
+
spatial_dedup_channel_index=spatial_dedup_channel_index,
|
|
181
|
+
relative_peak_radius_samples=thresholding_cfg.relative_peak_radius_samples,
|
|
182
|
+
temporal_dedup_radius_samples=thresholding_cfg.temporal_dedup_radius_samples,
|
|
183
|
+
remove_exact_duplicates=thresholding_cfg.remove_exact_duplicates,
|
|
184
|
+
cumulant_order=thresholding_cfg.cumulant_order,
|
|
185
|
+
convexity_threshold=thresholding_cfg.convexity_threshold,
|
|
186
|
+
convexity_radius=thresholding_cfg.convexity_radius,
|
|
187
|
+
n_seconds_fit=thresholding_cfg.n_seconds_fit,
|
|
188
|
+
fit_sampling=sampling_cfg.fit_sampling,
|
|
189
|
+
fit_max_reweighting=sampling_cfg.fit_max_reweighting,
|
|
190
|
+
n_waveforms_fit=sampling_cfg.n_waveforms_fit,
|
|
191
|
+
max_waveforms_fit=sampling_cfg.max_waveforms_fit,
|
|
192
|
+
fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
|
|
193
|
+
thinning=thresholding_cfg.thinning,
|
|
194
|
+
time_jitter=thresholding_cfg.time_jitter,
|
|
195
|
+
spatial_jitter_channel_index=spatial_jitter_channel_index,
|
|
196
|
+
trough_priority=thresholding_cfg.trough_priority,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def out_datasets(self):
|
|
200
|
+
datasets = super().out_datasets()
|
|
201
|
+
if self.is_random:
|
|
202
|
+
datasets.append(
|
|
203
|
+
SpikeDataset(name="orig_times_samples", shape_per_spike=(), dtype=float)
|
|
204
|
+
)
|
|
205
|
+
datasets.append(
|
|
206
|
+
SpikeDataset(name="orig_channels", shape_per_spike=(), dtype=float)
|
|
207
|
+
)
|
|
208
|
+
datasets.append(SpikeDataset(name="voltages", shape_per_spike=(), dtype=float))
|
|
209
|
+
return datasets
|
|
210
|
+
|
|
211
|
+
def peel_chunk(
|
|
212
|
+
self,
|
|
213
|
+
traces,
|
|
214
|
+
*,
|
|
215
|
+
chunk_start_samples=0,
|
|
216
|
+
left_margin=0,
|
|
217
|
+
right_margin=0,
|
|
218
|
+
return_residual=False,
|
|
219
|
+
return_waveforms=True,
|
|
220
|
+
) -> PeelingBatchResult:
|
|
221
|
+
threshold_res = threshold_chunk(
|
|
222
|
+
traces,
|
|
223
|
+
self.channel_index,
|
|
224
|
+
detection_threshold=self.detection_threshold,
|
|
225
|
+
peak_sign=self.peak_sign,
|
|
226
|
+
spatial_dedup_channel_index=self.spatial_dedup_channel_index,
|
|
227
|
+
spatial_dedup_rel_inds=self.spatial_dedup_rel_inds,
|
|
228
|
+
dedup_batch_size=self.dedup_batch_size,
|
|
229
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
230
|
+
spike_length_samples=self.spike_length_samples,
|
|
231
|
+
left_margin=left_margin,
|
|
232
|
+
right_margin=right_margin,
|
|
233
|
+
relative_peak_channel_index=self.relative_peak_channel_index,
|
|
234
|
+
temporal_dedup_radius_samples=self.temporal_dedup_radius_samples,
|
|
235
|
+
remove_exact_duplicates=self.remove_exact_duplicates,
|
|
236
|
+
cumulant_order=self.cumulant_order,
|
|
237
|
+
convexity_threshold=self.convexity_threshold,
|
|
238
|
+
convexity_radius=self.convexity_radius,
|
|
239
|
+
thinning=self.thinning,
|
|
240
|
+
time_jitter=self.time_jitter,
|
|
241
|
+
spatial_jitter_channel_index=self.spatial_jitter_channel_index,
|
|
242
|
+
rg=self.rg if self.is_random else None,
|
|
243
|
+
max_spikes_per_chunk=None,
|
|
244
|
+
return_waveforms=return_waveforms,
|
|
245
|
+
trough_priority=self.trough_priority,
|
|
246
|
+
quiet=False,
|
|
247
|
+
)
|
|
248
|
+
if not threshold_res["n_spikes"]:
|
|
249
|
+
return peeling_empty_result
|
|
250
|
+
|
|
251
|
+
# get absolute times
|
|
252
|
+
times_rel = threshold_res["times_rel"]
|
|
253
|
+
assert times_rel is not None
|
|
254
|
+
times_samples = times_rel + chunk_start_samples
|
|
255
|
+
|
|
256
|
+
peel_result = PeelingBatchResult(
|
|
257
|
+
n_spikes=threshold_res["n_spikes"],
|
|
258
|
+
times_samples=times_samples,
|
|
259
|
+
channels=threshold_res["channels"],
|
|
260
|
+
voltages=threshold_res["voltages"],
|
|
261
|
+
)
|
|
262
|
+
if return_waveforms:
|
|
263
|
+
assert "waveforms" in threshold_res
|
|
264
|
+
peel_result["collisioncleaned_waveforms"] = threshold_res["waveforms"]
|
|
265
|
+
if self.is_random:
|
|
266
|
+
orig_times_rel = threshold_res["orig_times_rel"]
|
|
267
|
+
assert orig_times_rel is not None
|
|
268
|
+
peel_result["orig_times_samples"] = orig_times_rel + chunk_start_samples
|
|
269
|
+
peel_result["orig_channels"] = threshold_res["orig_channels"]
|
|
270
|
+
if return_residual:
|
|
271
|
+
# note, this is same as the input.
|
|
272
|
+
peel_result["residual"] = traces[
|
|
273
|
+
left_margin : traces.shape[0] - right_margin
|
|
274
|
+
]
|
|
275
|
+
return peel_result
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def threshold_chunk(
|
|
279
|
+
traces,
|
|
280
|
+
channel_index,
|
|
281
|
+
detection_threshold=4.0,
|
|
282
|
+
peak_sign="both",
|
|
283
|
+
relative_peak_channel_index=None,
|
|
284
|
+
spatial_dedup_channel_index=None,
|
|
285
|
+
spatial_dedup_rel_inds=None,
|
|
286
|
+
dedup_batch_size=512,
|
|
287
|
+
trough_offset_samples=42,
|
|
288
|
+
spike_length_samples=121,
|
|
289
|
+
left_margin=0,
|
|
290
|
+
right_margin=0,
|
|
291
|
+
relative_peak_radius=5,
|
|
292
|
+
temporal_dedup_radius_samples=7,
|
|
293
|
+
remove_exact_duplicates=True,
|
|
294
|
+
max_spikes_per_chunk=None,
|
|
295
|
+
thinning=0.0,
|
|
296
|
+
time_jitter=0,
|
|
297
|
+
trough_priority=None,
|
|
298
|
+
spatial_jitter_channel_index=None,
|
|
299
|
+
cumulant_order=None,
|
|
300
|
+
convexity_threshold=None,
|
|
301
|
+
convexity_radius=3,
|
|
302
|
+
return_waveforms=True,
|
|
303
|
+
rg=None,
|
|
304
|
+
quiet=False,
|
|
305
|
+
) -> PeelingBatchResult:
|
|
306
|
+
n_index = channel_index.shape[1]
|
|
307
|
+
times_rel, channels, energies = detect_and_deduplicate( # type: ignore
|
|
308
|
+
traces,
|
|
309
|
+
detection_threshold,
|
|
310
|
+
relative_peak_channel_index=relative_peak_channel_index,
|
|
311
|
+
dedup_channel_index=spatial_dedup_channel_index,
|
|
312
|
+
dedup_index_inds=spatial_dedup_rel_inds,
|
|
313
|
+
spatial_dedup_batch_size=dedup_batch_size,
|
|
314
|
+
peak_sign=peak_sign,
|
|
315
|
+
dedup_temporal_radius=temporal_dedup_radius_samples,
|
|
316
|
+
remove_exact_duplicates=remove_exact_duplicates,
|
|
317
|
+
relative_peak_radius=relative_peak_radius,
|
|
318
|
+
return_energies=True,
|
|
319
|
+
trough_priority=trough_priority,
|
|
320
|
+
cumulant_order=cumulant_order,
|
|
321
|
+
)
|
|
322
|
+
if not times_rel.numel():
|
|
323
|
+
return PeelingBatchResult(
|
|
324
|
+
n_spikes=0,
|
|
325
|
+
orig_times_rel=times_rel,
|
|
326
|
+
times_rel=times_rel,
|
|
327
|
+
orig_channels=channels,
|
|
328
|
+
channels=channels,
|
|
329
|
+
voltages=energies,
|
|
330
|
+
waveforms=energies.view(-1, spike_length_samples, n_index),
|
|
331
|
+
)
|
|
332
|
+
keep = convexity_filter(
|
|
333
|
+
traces,
|
|
334
|
+
times_rel,
|
|
335
|
+
channels,
|
|
336
|
+
threshold=convexity_threshold,
|
|
337
|
+
radius=convexity_radius,
|
|
338
|
+
)
|
|
339
|
+
times_rel = times_rel[keep]
|
|
340
|
+
channels = channels[keep]
|
|
341
|
+
energies = energies[keep]
|
|
342
|
+
|
|
343
|
+
orig_times_rel = times_rel
|
|
344
|
+
orig_channels = channels
|
|
345
|
+
keep, times_rel, channels = perturb_detections(
|
|
346
|
+
times_rel,
|
|
347
|
+
channels,
|
|
348
|
+
thinning=thinning,
|
|
349
|
+
time_jitter=time_jitter,
|
|
350
|
+
spatial_jitter_channel_index=spatial_jitter_channel_index,
|
|
351
|
+
rg=rg,
|
|
352
|
+
)
|
|
353
|
+
orig_times_rel = orig_times_rel[keep]
|
|
354
|
+
orig_channels = orig_channels[keep]
|
|
355
|
+
energies = energies[keep]
|
|
356
|
+
|
|
357
|
+
# want only peaks in the chunk
|
|
358
|
+
min_time = left_margin + trough_offset_samples
|
|
359
|
+
tail_samples = spike_length_samples - trough_offset_samples
|
|
360
|
+
max_time = traces.shape[0] - right_margin - tail_samples - 1
|
|
361
|
+
valid = times_rel == times_rel.clamp(min_time, max_time)
|
|
362
|
+
(valid,) = valid.nonzero(as_tuple=True)
|
|
363
|
+
orig_times_rel = orig_times_rel[valid]
|
|
364
|
+
times_rel = times_rel[valid]
|
|
365
|
+
channels = channels[valid]
|
|
366
|
+
orig_channels = orig_channels[valid]
|
|
367
|
+
voltages = traces[orig_times_rel, orig_channels]
|
|
368
|
+
n_detect = times_rel.numel()
|
|
369
|
+
if not n_detect:
|
|
370
|
+
return PeelingBatchResult(
|
|
371
|
+
n_spikes=0,
|
|
372
|
+
times_rel=times_rel,
|
|
373
|
+
channels=channels,
|
|
374
|
+
voltages=energies,
|
|
375
|
+
orig_times_rel=orig_times_rel,
|
|
376
|
+
orig_channels=orig_channels,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
if max_spikes_per_chunk is not None:
|
|
380
|
+
if n_detect > max_spikes_per_chunk and not quiet:
|
|
381
|
+
warnings.warn(
|
|
382
|
+
f"{n_detect} spikes in chunk was larger than "
|
|
383
|
+
f"{max_spikes_per_chunk=}. Keeping the top ones."
|
|
384
|
+
)
|
|
385
|
+
energies = energies[valid]
|
|
386
|
+
best = torch.argsort(energies)[-max_spikes_per_chunk:]
|
|
387
|
+
best = best.sort().values
|
|
388
|
+
del energies
|
|
389
|
+
|
|
390
|
+
times_rel = times_rel[best]
|
|
391
|
+
channels = channels[best]
|
|
392
|
+
voltages = voltages[best]
|
|
393
|
+
orig_channels = orig_channels[best]
|
|
394
|
+
orig_times_rel = orig_times_rel[best]
|
|
395
|
+
|
|
396
|
+
# load up the waveforms for this chunk
|
|
397
|
+
if return_waveforms:
|
|
398
|
+
waveforms = spiketorch.grab_spikes(
|
|
399
|
+
traces,
|
|
400
|
+
times_rel,
|
|
401
|
+
channels,
|
|
402
|
+
channel_index,
|
|
403
|
+
trough_offset=trough_offset_samples,
|
|
404
|
+
spike_length_samples=spike_length_samples,
|
|
405
|
+
already_padded=False,
|
|
406
|
+
pad_value=torch.nan,
|
|
407
|
+
)
|
|
408
|
+
else:
|
|
409
|
+
waveforms = None
|
|
410
|
+
|
|
411
|
+
# offset times for caller
|
|
412
|
+
orig_times_rel -= left_margin
|
|
413
|
+
times_rel -= left_margin
|
|
414
|
+
|
|
415
|
+
res = PeelingBatchResult(
|
|
416
|
+
n_spikes=times_rel.numel(),
|
|
417
|
+
orig_times_rel=orig_times_rel,
|
|
418
|
+
orig_channels=orig_channels,
|
|
419
|
+
times_rel=times_rel,
|
|
420
|
+
channels=channels,
|
|
421
|
+
voltages=voltages,
|
|
422
|
+
)
|
|
423
|
+
if waveforms is not None:
|
|
424
|
+
res["waveforms"] = waveforms
|
|
425
|
+
return res
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def perturb_detections(
|
|
429
|
+
times_rel,
|
|
430
|
+
channels,
|
|
431
|
+
thinning: float = 0,
|
|
432
|
+
time_jitter=0,
|
|
433
|
+
spatial_jitter_channel_index=None,
|
|
434
|
+
rg: np.random.Generator | None = None,
|
|
435
|
+
):
|
|
436
|
+
keep = slice(None)
|
|
437
|
+
if not (thinning or time_jitter or spatial_jitter_channel_index is not None):
|
|
438
|
+
return keep, times_rel, channels
|
|
439
|
+
|
|
440
|
+
n = len(times_rel)
|
|
441
|
+
if not n:
|
|
442
|
+
return keep, times_rel, channels
|
|
443
|
+
|
|
444
|
+
if thinning:
|
|
445
|
+
assert 0 <= thinning <= 1
|
|
446
|
+
assert rg is not None
|
|
447
|
+
keep = rg.binomial(n=1, p=1.0 - thinning, size=n)
|
|
448
|
+
keep = torch.from_numpy(np.flatnonzero(keep))
|
|
449
|
+
keep = keep.to(times_rel)
|
|
450
|
+
|
|
451
|
+
times_rel = times_rel[keep]
|
|
452
|
+
channels = channels[keep]
|
|
453
|
+
|
|
454
|
+
n = len(times_rel)
|
|
455
|
+
if time_jitter:
|
|
456
|
+
assert rg is not None
|
|
457
|
+
jitter = rg.integers(low=-time_jitter, high=time_jitter + 1)
|
|
458
|
+
times_rel = times_rel + torch.asarray(
|
|
459
|
+
jitter, dtype=times_rel.dtype, device=times_rel.device
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
if spatial_jitter_channel_index is not None:
|
|
463
|
+
assert rg is not None
|
|
464
|
+
n_channels = len(spatial_jitter_channel_index)
|
|
465
|
+
n_valid = (spatial_jitter_channel_index < n_channels).sum(1)
|
|
466
|
+
n_valid = n_valid[channels].cpu()
|
|
467
|
+
rel_ix = rg.integers(0, high=n_valid)
|
|
468
|
+
rel_ix = torch.from_numpy(rel_ix).to(channels)
|
|
469
|
+
channels = spatial_jitter_channel_index[channels, rel_ix]
|
|
470
|
+
|
|
471
|
+
return keep, times_rel, channels
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
import numpy as np
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from ..templates.template_util import LowRankTemplates, compressed_upsampled_templates
|
|
6
|
+
from ..transform import WaveformPipeline
|
|
7
|
+
from ..util import universal_util, waveform_util
|
|
8
|
+
from ..util.internal_config import (
|
|
9
|
+
UniversalMatchingConfig,
|
|
10
|
+
FitSamplingConfig,
|
|
11
|
+
FeaturizationConfig,
|
|
12
|
+
WaveformConfig,
|
|
13
|
+
)
|
|
14
|
+
from .matching import ObjectiveUpdateTemplateMatchingPeeler
|
|
15
|
+
from .matching_util.compressed_upsampled import CompressedUpsampledMatchingTemplates
|
|
16
|
+
from .matching_util.pairwise import SeparablePairwiseConv
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class UniversalTemplatesMatchingPeeler(ObjectiveUpdateTemplateMatchingPeeler):
|
|
20
|
+
"""KS-style universal-templates-from-data detection
|
|
21
|
+
|
|
22
|
+
This tries to rephrase their algorithm as faithfully as possible
|
|
23
|
+
using dartsort tools, for comparison purposes with our algorithms.
|
|
24
|
+
|
|
25
|
+
The idea is to estimate some (they use 6, it turns out) single-channel
|
|
26
|
+
shapes via K means applied to single-channel waveforms. These
|
|
27
|
+
are then expanded out into a full template library by spatial
|
|
28
|
+
convs with various Gaussians. Then, throw them into the matcher.
|
|
29
|
+
Since KS' matcher has scale_std --> infty, we can put a large
|
|
30
|
+
scale prior variance to match the spirit of the thing.
|
|
31
|
+
|
|
32
|
+
TODO maybe I should implement scale_prior->infty in our matcher?
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
recording,
|
|
38
|
+
channel_index,
|
|
39
|
+
featurization_pipeline,
|
|
40
|
+
threshold=100.0,
|
|
41
|
+
trough_offset_samples=42,
|
|
42
|
+
spike_length_samples=121,
|
|
43
|
+
amplitude_scaling_variance=100.0,
|
|
44
|
+
amplitude_scaling_boundary=500.0,
|
|
45
|
+
detection_threshold=6.0,
|
|
46
|
+
alignment_padding=20,
|
|
47
|
+
n_centroids=6,
|
|
48
|
+
pca_rank=8,
|
|
49
|
+
taper=True,
|
|
50
|
+
n_sigmas=5,
|
|
51
|
+
min_template_size=10.0,
|
|
52
|
+
max_distance=32.0,
|
|
53
|
+
dx=32.0,
|
|
54
|
+
chunk_length_samples=1000,
|
|
55
|
+
n_seconds_fit=40,
|
|
56
|
+
max_waveforms_fit=50_000,
|
|
57
|
+
n_waveforms_fit=20_000,
|
|
58
|
+
fit_subsampling_random_state=0,
|
|
59
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
60
|
+
fit_max_reweighting=4.0,
|
|
61
|
+
refractory_radius_frames=10,
|
|
62
|
+
max_iter=1000,
|
|
63
|
+
dtype=torch.float,
|
|
64
|
+
):
|
|
65
|
+
shapes, footprints, template_data = (
|
|
66
|
+
universal_util.universal_templates_from_data(
|
|
67
|
+
rec=recording,
|
|
68
|
+
detection_threshold=detection_threshold,
|
|
69
|
+
trough_offset_samples=trough_offset_samples,
|
|
70
|
+
spike_length_samples=spike_length_samples,
|
|
71
|
+
alignment_padding=alignment_padding,
|
|
72
|
+
n_centroids=n_centroids,
|
|
73
|
+
pca_rank=pca_rank,
|
|
74
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
75
|
+
taper=taper,
|
|
76
|
+
taper_start=alignment_padding // 2,
|
|
77
|
+
taper_end=alignment_padding // 2,
|
|
78
|
+
random_seed=fit_subsampling_random_state,
|
|
79
|
+
n_sigmas=n_sigmas,
|
|
80
|
+
min_template_size=min_template_size,
|
|
81
|
+
max_distance=max_distance,
|
|
82
|
+
dx=dx,
|
|
83
|
+
# let's not worry about exposing these
|
|
84
|
+
deduplication_radius=150.0,
|
|
85
|
+
kmeanspp_initial="random",
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# TODO: this should all be put in a matchingtemplate+builder
|
|
90
|
+
# so that it is lazy and resuming works better.
|
|
91
|
+
Nf = len(footprints)
|
|
92
|
+
Ns = len(shapes)
|
|
93
|
+
shapes_ixd = torch.asarray(shapes)[None]
|
|
94
|
+
shapes_ixd = shapes_ixd.broadcast_to((Nf, Ns, *shapes.shape[1:]))
|
|
95
|
+
shapes_ixd = shapes_ixd.reshape(Nf * Ns, *shapes.shape[1:], 1)
|
|
96
|
+
footprints_ixd = torch.asarray(footprints)[:, None]
|
|
97
|
+
footprints_ixd = footprints_ixd.broadcast_to((Nf, Ns, *footprints.shape[1:]))
|
|
98
|
+
footprints_ixd = footprints_ixd.reshape(Nf * Ns, 1, *footprints.shape[1:])
|
|
99
|
+
low_rank_templates = LowRankTemplates(
|
|
100
|
+
unit_ids=np.arange(Nf * Ns),
|
|
101
|
+
temporal_components=shapes_ixd.numpy(),
|
|
102
|
+
singular_values=shapes_ixd.new_ones(Nf * Ns, 1).numpy(),
|
|
103
|
+
spatial_components=footprints_ixd.numpy(),
|
|
104
|
+
spike_counts_by_channel=np.broadcast_to(
|
|
105
|
+
np.atleast_2d([100]), (Nf * Ns, footprints.shape[1])
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
pairwise_conv_db = SeparablePairwiseConv(footprints, shapes)
|
|
109
|
+
|
|
110
|
+
cupt = compressed_upsampled_templates(
|
|
111
|
+
low_rank_templates.temporal_components, max_upsample=1
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# TODO: re-using the individually compressed basis algorithm here
|
|
115
|
+
# which is general enough that it works, but for this separable
|
|
116
|
+
# basis there is a faster algorithm.
|
|
117
|
+
matching_templates = CompressedUpsampledMatchingTemplates(
|
|
118
|
+
trough_offset_samples=trough_offset_samples,
|
|
119
|
+
low_rank_templates=low_rank_templates,
|
|
120
|
+
pconv_db=pairwise_conv_db,
|
|
121
|
+
compressed_upsampled_temporal=cupt,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
super().__init__(
|
|
125
|
+
recording=recording,
|
|
126
|
+
matching_templates=matching_templates,
|
|
127
|
+
channel_index=channel_index,
|
|
128
|
+
featurization_pipeline=featurization_pipeline,
|
|
129
|
+
threshold=threshold,
|
|
130
|
+
amplitude_scaling_variance=amplitude_scaling_variance,
|
|
131
|
+
amplitude_scaling_boundary=amplitude_scaling_boundary,
|
|
132
|
+
trough_offset_samples=trough_offset_samples,
|
|
133
|
+
chunk_length_samples=chunk_length_samples,
|
|
134
|
+
n_seconds_fit=n_seconds_fit,
|
|
135
|
+
max_waveforms_fit=max_waveforms_fit,
|
|
136
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
137
|
+
fit_subsampling_random_state=fit_subsampling_random_state,
|
|
138
|
+
fit_sampling=fit_sampling,
|
|
139
|
+
fit_max_reweighting=fit_max_reweighting,
|
|
140
|
+
dtype=dtype,
|
|
141
|
+
refractory_radius_frames=refractory_radius_frames,
|
|
142
|
+
max_iter=max_iter,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def from_config( # type: ignore[reportIncompatibleOverride]
|
|
147
|
+
cls,
|
|
148
|
+
recording,
|
|
149
|
+
*,
|
|
150
|
+
universal_cfg: UniversalMatchingConfig,
|
|
151
|
+
featurization_cfg: FeaturizationConfig,
|
|
152
|
+
waveform_cfg: WaveformConfig,
|
|
153
|
+
sampling_cfg: FitSamplingConfig,
|
|
154
|
+
):
|
|
155
|
+
geom = torch.tensor(recording.get_channel_locations())
|
|
156
|
+
channel_index = waveform_util.make_channel_index(
|
|
157
|
+
geom, featurization_cfg.extract_radius, to_torch=True
|
|
158
|
+
)
|
|
159
|
+
featurization_pipeline = WaveformPipeline.from_config(
|
|
160
|
+
geom=geom,
|
|
161
|
+
channel_index=channel_index,
|
|
162
|
+
featurization_cfg=featurization_cfg,
|
|
163
|
+
waveform_cfg=waveform_cfg,
|
|
164
|
+
sampling_frequency=recording.sampling_frequency,
|
|
165
|
+
)
|
|
166
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(
|
|
167
|
+
recording.sampling_frequency
|
|
168
|
+
)
|
|
169
|
+
spike_length_samples = waveform_cfg.spike_length_samples(
|
|
170
|
+
recording.sampling_frequency
|
|
171
|
+
)
|
|
172
|
+
fs_ms = recording.sampling_frequency / 1000
|
|
173
|
+
alignment_padding = int(universal_cfg.alignment_padding_ms * fs_ms)
|
|
174
|
+
return cls(
|
|
175
|
+
recording,
|
|
176
|
+
chunk_length_samples=universal_cfg.chunk_length_samples,
|
|
177
|
+
n_seconds_fit=universal_cfg.n_seconds_fit,
|
|
178
|
+
n_waveforms_fit=sampling_cfg.n_waveforms_fit,
|
|
179
|
+
max_waveforms_fit=sampling_cfg.max_waveforms_fit,
|
|
180
|
+
fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
|
|
181
|
+
fit_sampling=sampling_cfg.fit_sampling,
|
|
182
|
+
fit_max_reweighting=sampling_cfg.fit_max_reweighting,
|
|
183
|
+
threshold=universal_cfg.threshold,
|
|
184
|
+
detection_threshold=universal_cfg.detection_threshold,
|
|
185
|
+
n_sigmas=universal_cfg.n_sigmas,
|
|
186
|
+
channel_index=channel_index,
|
|
187
|
+
alignment_padding=alignment_padding,
|
|
188
|
+
featurization_pipeline=featurization_pipeline,
|
|
189
|
+
trough_offset_samples=trough_offset_samples,
|
|
190
|
+
spike_length_samples=spike_length_samples,
|
|
191
|
+
)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from .templates import TemplateData
|
|
2
|
+
from .template_util import (
|
|
3
|
+
svd_compress_templates,
|
|
4
|
+
compressed_upsampled_templates,
|
|
5
|
+
CompressedUpsampledTemplates,
|
|
6
|
+
LowRankTemplates,
|
|
7
|
+
templates_at_time,
|
|
8
|
+
)
|
|
9
|
+
from ..peel.matching_util.pairwise import CompressedPairwiseConv, SeparablePairwiseConv
|
|
10
|
+
from .realignment import realign, estimate_offset
|
|
11
|
+
from .postprocess_util import estimate_template_library
|