DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
from dataclasses import replace
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
import torch.nn.functional as F
|
|
8
|
+
from spikeinterface.core import BaseRecording
|
|
9
|
+
from tqdm.auto import trange
|
|
10
|
+
|
|
11
|
+
from ..util.data_util import DARTsortSorting
|
|
12
|
+
from ..util.internal_config import (
|
|
13
|
+
ComputationConfig,
|
|
14
|
+
RealignStrategy,
|
|
15
|
+
TemplateRealignmentConfig,
|
|
16
|
+
WaveformConfig,
|
|
17
|
+
default_waveform_cfg,
|
|
18
|
+
)
|
|
19
|
+
from ..util.job_util import ensure_computation_config
|
|
20
|
+
from ..util.logging_util import get_logger
|
|
21
|
+
from ..util.spiketorch import ptp
|
|
22
|
+
from .templates import TemplateData
|
|
23
|
+
|
|
24
|
+
logger = get_logger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def realign(
|
|
28
|
+
*,
|
|
29
|
+
recording: BaseRecording,
|
|
30
|
+
sorting: DARTsortSorting,
|
|
31
|
+
realign_cfg: TemplateRealignmentConfig | None = None,
|
|
32
|
+
waveform_cfg: WaveformConfig = default_waveform_cfg,
|
|
33
|
+
computation_cfg: ComputationConfig | None = None,
|
|
34
|
+
motion_est=None,
|
|
35
|
+
) -> tuple[DARTsortSorting, TemplateData | None]:
|
|
36
|
+
if realign_cfg is None:
|
|
37
|
+
return sorting, None
|
|
38
|
+
if not realign_cfg.realign_peaks:
|
|
39
|
+
return sorting, None
|
|
40
|
+
if not realign_cfg.realign_shift_ms:
|
|
41
|
+
return sorting, None
|
|
42
|
+
|
|
43
|
+
realignment_waveform_cfg = WaveformConfig(
|
|
44
|
+
ms_before=waveform_cfg.ms_before + realign_cfg.realign_shift_ms,
|
|
45
|
+
ms_after=waveform_cfg.ms_after + realign_cfg.realign_shift_ms,
|
|
46
|
+
)
|
|
47
|
+
templates = TemplateData.from_config(
|
|
48
|
+
recording=recording,
|
|
49
|
+
sorting=sorting,
|
|
50
|
+
template_cfg=realign_cfg.template_cfg,
|
|
51
|
+
waveform_cfg=realignment_waveform_cfg,
|
|
52
|
+
computation_cfg=computation_cfg,
|
|
53
|
+
motion_est=motion_est,
|
|
54
|
+
)
|
|
55
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(
|
|
56
|
+
sampling_frequency=recording.sampling_frequency
|
|
57
|
+
)
|
|
58
|
+
spike_length_samples = waveform_cfg.spike_length_samples(
|
|
59
|
+
sampling_frequency=recording.sampling_frequency
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
template_shifts, aligned_templates = realign_templates(
|
|
63
|
+
templates=templates.templates,
|
|
64
|
+
snrs_by_channel=templates.snrs_by_channel(),
|
|
65
|
+
unit_ids=templates.unit_ids,
|
|
66
|
+
padded_trough_offset_samples=realignment_waveform_cfg.trough_offset_samples(
|
|
67
|
+
sampling_frequency=recording.sampling_frequency
|
|
68
|
+
),
|
|
69
|
+
trough_offset_samples=trough_offset_samples,
|
|
70
|
+
realign_strategy=realign_cfg.realign_strategy,
|
|
71
|
+
trough_factor=realign_cfg.trough_factor,
|
|
72
|
+
min_pair_corr=realign_cfg.min_pair_corr,
|
|
73
|
+
computation_cfg=computation_cfg,
|
|
74
|
+
)
|
|
75
|
+
templates = replace(
|
|
76
|
+
templates,
|
|
77
|
+
templates=aligned_templates,
|
|
78
|
+
trough_offset_samples=trough_offset_samples,
|
|
79
|
+
)
|
|
80
|
+
aligned_sorting = apply_time_shifts(
|
|
81
|
+
sorting,
|
|
82
|
+
template_shifts=template_shifts,
|
|
83
|
+
trough_offset_samples=trough_offset_samples,
|
|
84
|
+
spike_length_samples=spike_length_samples,
|
|
85
|
+
recording_length_samples=recording.get_num_samples(),
|
|
86
|
+
)
|
|
87
|
+
return aligned_sorting, templates
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@torch.jit.script
|
|
91
|
+
def singlechan_alignments(
|
|
92
|
+
traces: torch.Tensor, trough_factor: float = 3.0, dim: int = 1
|
|
93
|
+
) -> torch.Tensor:
|
|
94
|
+
aligner_traces = torch.where(traces < 0, trough_factor * traces, -traces)
|
|
95
|
+
offsets = aligner_traces.argmin(dim=dim)
|
|
96
|
+
return offsets
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def get_main_channels_and_alignments(
|
|
100
|
+
template_data=None, trough_factor=3.0, templates=None, main_channels=None
|
|
101
|
+
):
|
|
102
|
+
if templates is None:
|
|
103
|
+
assert template_data is not None
|
|
104
|
+
templates = template_data.templates
|
|
105
|
+
if main_channels is None:
|
|
106
|
+
main_channels = np.ptp(templates, axis=1).argmax(1)
|
|
107
|
+
mc_traces = np.take_along_axis(templates, main_channels[:, None, None], axis=2)[
|
|
108
|
+
:, :, 0
|
|
109
|
+
]
|
|
110
|
+
aligner_traces = np.where(mc_traces < 0, trough_factor * mc_traces, -mc_traces)
|
|
111
|
+
aligner_traces = np.abs(aligner_traces, out=aligner_traces)
|
|
112
|
+
# break ties late
|
|
113
|
+
tmax = aligner_traces.shape[1] - 1
|
|
114
|
+
offsets = tmax - aligner_traces[:, ::-1].argmax(1)
|
|
115
|
+
return main_channels, mc_traces, offsets
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def estimate_offset(
|
|
119
|
+
*,
|
|
120
|
+
templates,
|
|
121
|
+
computation_cfg: ComputationConfig,
|
|
122
|
+
snrs_by_channel=None,
|
|
123
|
+
strategy="mainchan_trough_factor",
|
|
124
|
+
trough_factor=3.0,
|
|
125
|
+
min_weight=0.75,
|
|
126
|
+
min_pair_corr=0.75,
|
|
127
|
+
main_channels=None,
|
|
128
|
+
padded_trough_offset_samples=42,
|
|
129
|
+
trough_offset_samples=42,
|
|
130
|
+
) -> torch.Tensor:
|
|
131
|
+
templates = torch.asarray(templates)
|
|
132
|
+
|
|
133
|
+
if strategy == "mainchan_trough_factor":
|
|
134
|
+
_, _, offsets = get_main_channels_and_alignments(
|
|
135
|
+
None,
|
|
136
|
+
trough_factor=trough_factor,
|
|
137
|
+
templates=templates.numpy(force=True),
|
|
138
|
+
main_channels=main_channels,
|
|
139
|
+
)
|
|
140
|
+
return offsets - padded_trough_offset_samples
|
|
141
|
+
|
|
142
|
+
if strategy == "dredge":
|
|
143
|
+
offsets = dredge_realign(
|
|
144
|
+
templates=templates,
|
|
145
|
+
main_channels=main_channels,
|
|
146
|
+
snrs_by_channel=snrs_by_channel,
|
|
147
|
+
trough_factor=trough_factor,
|
|
148
|
+
padded_trough_offset_samples=padded_trough_offset_samples,
|
|
149
|
+
trough_offset_samples=trough_offset_samples,
|
|
150
|
+
min_corr=min_pair_corr,
|
|
151
|
+
computation_cfg=computation_cfg,
|
|
152
|
+
)
|
|
153
|
+
gc.collect()
|
|
154
|
+
torch.cuda.empty_cache()
|
|
155
|
+
return offsets
|
|
156
|
+
|
|
157
|
+
if strategy == "snr_weighted_trough_factor":
|
|
158
|
+
assert snrs_by_channel is not None
|
|
159
|
+
weights = torch.asarray(snrs_by_channel).clone()
|
|
160
|
+
weights /= weights.amax(dim=1, keepdim=True)
|
|
161
|
+
weights[weights < min_weight] = 0.0
|
|
162
|
+
weights /= weights.sum(dim=1, keepdim=True)
|
|
163
|
+
tmp = templates.clone()
|
|
164
|
+
tmp[tmp < 0] *= trough_factor
|
|
165
|
+
tmp.abs_()
|
|
166
|
+
offsets = tmp.argmax(dim=1).double()
|
|
167
|
+
offsets = torch.sum(offsets * weights, dim=1)
|
|
168
|
+
offsets = torch.round(offsets).long()
|
|
169
|
+
return offsets - padded_trough_offset_samples
|
|
170
|
+
|
|
171
|
+
if strategy == "normsq_weighted_trough_factor":
|
|
172
|
+
tmp = templates.square()
|
|
173
|
+
weights = tmp.sum(dim=1)
|
|
174
|
+
weights /= weights.amax(dim=1, keepdim=True)
|
|
175
|
+
weights[weights < min_weight] = 0.0
|
|
176
|
+
weights /= weights.sum(dim=1, keepdim=True)
|
|
177
|
+
tmp[:] = templates
|
|
178
|
+
tmp[tmp < 0] *= trough_factor
|
|
179
|
+
tmp.abs_()
|
|
180
|
+
offsets = tmp.argmax(dim=1).double()
|
|
181
|
+
offsets = torch.sum(offsets * weights, dim=1)
|
|
182
|
+
offsets = torch.round(offsets).long()
|
|
183
|
+
return offsets - padded_trough_offset_samples
|
|
184
|
+
|
|
185
|
+
if strategy == "ampsq_weighted_trough_factor":
|
|
186
|
+
weights = ptp(templates).square()
|
|
187
|
+
weights /= weights.amax(dim=1, keepdim=True)
|
|
188
|
+
weights[weights < min_weight] = 0.0
|
|
189
|
+
weights /= weights.sum(dim=1, keepdim=True)
|
|
190
|
+
tmp = templates.clone()
|
|
191
|
+
tmp[tmp < 0] *= trough_factor
|
|
192
|
+
tmp.abs_()
|
|
193
|
+
offsets = tmp.argmax(dim=1).double()
|
|
194
|
+
offsets = torch.sum(offsets * weights, dim=1)
|
|
195
|
+
offsets = torch.round(offsets).long()
|
|
196
|
+
return offsets - padded_trough_offset_samples
|
|
197
|
+
|
|
198
|
+
assert False
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def realign_templates(
|
|
202
|
+
*,
|
|
203
|
+
templates,
|
|
204
|
+
snrs_by_channel=None,
|
|
205
|
+
unit_ids=None,
|
|
206
|
+
main_channels=None,
|
|
207
|
+
padded_trough_offset_samples=42,
|
|
208
|
+
trough_offset_samples=42,
|
|
209
|
+
realign_strategy: RealignStrategy = "mainchan_trough_factor",
|
|
210
|
+
min_pair_corr=0.75,
|
|
211
|
+
trough_factor=3.0,
|
|
212
|
+
computation_cfg: ComputationConfig | None = None,
|
|
213
|
+
):
|
|
214
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
215
|
+
if main_channels is None:
|
|
216
|
+
if realign_strategy in ("mainchan_trough_factor", "dredge"):
|
|
217
|
+
if snrs_by_channel is not None:
|
|
218
|
+
main_channels = snrs_by_channel.argmax(1)
|
|
219
|
+
else:
|
|
220
|
+
main_channels = np.ptp(templates, axis=1).argmax(1)
|
|
221
|
+
elif realign_strategy.startswith("snr_weighted"):
|
|
222
|
+
assert snrs_by_channel is not None
|
|
223
|
+
main_channels = snrs_by_channel.argmax(1)
|
|
224
|
+
elif realign_strategy.startswith("normsq_weighted"):
|
|
225
|
+
main_channels = np.square(templates).sum(1).argmax(1)
|
|
226
|
+
elif realign_strategy.startswith("ampsq_weighted"):
|
|
227
|
+
main_channels = np.ptp(templates).max(1).argmax(1)
|
|
228
|
+
else:
|
|
229
|
+
assert False
|
|
230
|
+
assert main_channels is not None
|
|
231
|
+
|
|
232
|
+
# find template peak time
|
|
233
|
+
max_shift = padded_trough_offset_samples - trough_offset_samples
|
|
234
|
+
template_shifts__ = estimate_offset(
|
|
235
|
+
computation_cfg=computation_cfg,
|
|
236
|
+
templates=templates,
|
|
237
|
+
snrs_by_channel=snrs_by_channel,
|
|
238
|
+
strategy=realign_strategy,
|
|
239
|
+
trough_factor=trough_factor,
|
|
240
|
+
main_channels=main_channels,
|
|
241
|
+
min_pair_corr=min_pair_corr,
|
|
242
|
+
padded_trough_offset_samples=padded_trough_offset_samples,
|
|
243
|
+
trough_offset_samples=trough_offset_samples,
|
|
244
|
+
)
|
|
245
|
+
if torch.is_tensor(template_shifts__):
|
|
246
|
+
template_shifts_ = template_shifts__.numpy(force=True)
|
|
247
|
+
else:
|
|
248
|
+
template_shifts_ = template_shifts__
|
|
249
|
+
|
|
250
|
+
# clip if needed
|
|
251
|
+
template_shifts_[np.abs(template_shifts_) > max_shift] = 0
|
|
252
|
+
|
|
253
|
+
# find unit sample time shifts
|
|
254
|
+
if unit_ids is None:
|
|
255
|
+
template_shifts = template_shifts_
|
|
256
|
+
else:
|
|
257
|
+
template_shifts = np.zeros(unit_ids.max() + 1, dtype=np.int64)
|
|
258
|
+
template_shifts[unit_ids] = template_shifts_
|
|
259
|
+
|
|
260
|
+
aligned_templates = trim_templates_to_shift(
|
|
261
|
+
templates=templates,
|
|
262
|
+
max_shift=max_shift,
|
|
263
|
+
template_shifts=template_shifts,
|
|
264
|
+
unit_ids=unit_ids,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
return template_shifts, aligned_templates
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def trim_templates_to_shift(
|
|
271
|
+
templates, max_shift=0, template_shifts=None, unit_ids=None
|
|
272
|
+
):
|
|
273
|
+
if not max_shift or template_shifts is None:
|
|
274
|
+
return templates
|
|
275
|
+
|
|
276
|
+
n, t, c = templates.shape
|
|
277
|
+
|
|
278
|
+
# trim templates
|
|
279
|
+
aligned_spike_len = t - 2 * max_shift
|
|
280
|
+
aligned_templates = np.empty((n, aligned_spike_len, c), dtype=templates.dtype)
|
|
281
|
+
if unit_ids is None:
|
|
282
|
+
assert len(templates) == len(template_shifts) == n
|
|
283
|
+
else:
|
|
284
|
+
template_shifts = template_shifts[unit_ids]
|
|
285
|
+
for i, dt in enumerate(template_shifts):
|
|
286
|
+
aligned_templates[i] = templates[
|
|
287
|
+
i, max_shift + dt : max_shift + dt + aligned_spike_len
|
|
288
|
+
]
|
|
289
|
+
|
|
290
|
+
return aligned_templates
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def apply_time_shifts(
|
|
294
|
+
sorting,
|
|
295
|
+
template_shifts=None,
|
|
296
|
+
template_data=None,
|
|
297
|
+
trough_offset_samples=None,
|
|
298
|
+
spike_length_samples=None,
|
|
299
|
+
recording_length_samples=None,
|
|
300
|
+
) -> DARTsortSorting:
|
|
301
|
+
if template_shifts is not None:
|
|
302
|
+
unit_ids = None
|
|
303
|
+
elif template_data is not None and template_data.properties:
|
|
304
|
+
if "template_time_shifts" in template_data.properties:
|
|
305
|
+
template_shifts = template_data.properties["template_time_shifts"]
|
|
306
|
+
elif "time_shifts" in template_data.properties:
|
|
307
|
+
template_shifts = template_data.properties["time_shifts"]
|
|
308
|
+
unit_ids = template_data.unit_ids
|
|
309
|
+
else:
|
|
310
|
+
unit_ids = None
|
|
311
|
+
|
|
312
|
+
if template_shifts is None:
|
|
313
|
+
return sorting
|
|
314
|
+
|
|
315
|
+
ixs = sorting.labels
|
|
316
|
+
valid = np.flatnonzero(ixs >= 0)
|
|
317
|
+
ixsv = ixs[valid]
|
|
318
|
+
if unit_ids is not None and not np.array_equal(unit_ids, np.arange(len(unit_ids))):
|
|
319
|
+
ixsv = np.searchsorted(unit_ids, ixsv)
|
|
320
|
+
assert np.array_equal(unit_ids[ixsv], sorting.labels[valid])
|
|
321
|
+
|
|
322
|
+
new_times = sorting.times_samples.copy()
|
|
323
|
+
new_times[valid] += template_shifts[ixsv]
|
|
324
|
+
|
|
325
|
+
if recording_length_samples is not None:
|
|
326
|
+
assert spike_length_samples is not None
|
|
327
|
+
assert trough_offset_samples is not None
|
|
328
|
+
labels = sorting.labels.copy()
|
|
329
|
+
tail_samples = spike_length_samples - trough_offset_samples
|
|
330
|
+
highlim = recording_length_samples - tail_samples
|
|
331
|
+
labels[new_times < trough_offset_samples] = -1
|
|
332
|
+
labels[new_times >= highlim] = -1
|
|
333
|
+
else:
|
|
334
|
+
labels = sorting.labels
|
|
335
|
+
|
|
336
|
+
return sorting.ephemeral_replace(labels=labels, times_samples=new_times)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def dredge_realign(
|
|
340
|
+
*,
|
|
341
|
+
templates: torch.Tensor,
|
|
342
|
+
main_channels: torch.Tensor | None,
|
|
343
|
+
snrs_by_channel: torch.Tensor | None,
|
|
344
|
+
min_spatial_cosine: float = 0.75,
|
|
345
|
+
min_corr: float = 0.75,
|
|
346
|
+
trough_factor: float = 3.0,
|
|
347
|
+
padded_trough_offset_samples: int,
|
|
348
|
+
trough_offset_samples: int,
|
|
349
|
+
computation_cfg: ComputationConfig,
|
|
350
|
+
dredge_window_fraction: float = 0.5,
|
|
351
|
+
eps=1e-3,
|
|
352
|
+
) -> torch.Tensor:
|
|
353
|
+
from dredge.dredgelib import newton_solve_rigid
|
|
354
|
+
|
|
355
|
+
max_shift = padded_trough_offset_samples - trough_offset_samples
|
|
356
|
+
|
|
357
|
+
if main_channels is None and snrs_by_channel is not None:
|
|
358
|
+
main_channels = snrs_by_channel.argmax(1)
|
|
359
|
+
|
|
360
|
+
# start with trough alignment
|
|
361
|
+
main_channels, main_channel_traces, offsets = get_main_channels_and_alignments(
|
|
362
|
+
None,
|
|
363
|
+
trough_factor=trough_factor,
|
|
364
|
+
templates=templates.numpy(force=True),
|
|
365
|
+
main_channels=main_channels,
|
|
366
|
+
)
|
|
367
|
+
offsets = offsets - padded_trough_offset_samples
|
|
368
|
+
offsets[np.abs(offsets) > max_shift] = 0
|
|
369
|
+
|
|
370
|
+
# trim to alignment
|
|
371
|
+
templates_trim0 = trim_templates_to_shift(
|
|
372
|
+
templates=templates.numpy(force=True),
|
|
373
|
+
max_shift=max_shift,
|
|
374
|
+
template_shifts=offsets,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
# which pairs to convolve?
|
|
378
|
+
sp = snrs_by_channel if snrs_by_channel is not None else ptp(templates_trim0)
|
|
379
|
+
sp = torch.asarray(sp)
|
|
380
|
+
sp = sp / torch.linalg.norm(sp, dim=1)[:, None]
|
|
381
|
+
spcos = sp @ sp.T
|
|
382
|
+
pair_mask = spcos >= min_spatial_cosine
|
|
383
|
+
ii, jj = pair_mask.nonzero(as_tuple=True)
|
|
384
|
+
triu = ii < jj
|
|
385
|
+
ii = ii[triu]
|
|
386
|
+
jj = jj[triu]
|
|
387
|
+
logger.dartsortdebug(
|
|
388
|
+
f"DREDge align: convolving {ii.numel()} pairs (mincos={min_spatial_cosine:.2f})."
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# get best lags and correlations
|
|
392
|
+
templates_trim0 = torch.asarray(
|
|
393
|
+
templates_trim0, dtype=torch.float, device=computation_cfg.actual_device()
|
|
394
|
+
)
|
|
395
|
+
dredge_max_shift = math.floor(max_shift * dredge_window_fraction)
|
|
396
|
+
lags, corrs = _pairwise_correlate_templates(
|
|
397
|
+
templates=templates_trim0, ii=ii, jj=jj, max_shift=dredge_max_shift
|
|
398
|
+
)
|
|
399
|
+
lags = lags.numpy(force=True)
|
|
400
|
+
corrs = corrs.numpy(force=True)
|
|
401
|
+
logger.dartsortdebug(
|
|
402
|
+
f"DREDge align: {100 * (corrs >= min_corr).mean():.1f}% "
|
|
403
|
+
"of pairs were correlated enough."
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
# densify
|
|
407
|
+
k = len(templates)
|
|
408
|
+
D = np.zeros((k, k))
|
|
409
|
+
C = D.copy()
|
|
410
|
+
D[ii, jj] = lags
|
|
411
|
+
D[jj, ii] = -lags
|
|
412
|
+
C[ii, jj] = corrs
|
|
413
|
+
C[jj, ii] = corrs
|
|
414
|
+
np.fill_diagonal(C, 1.0)
|
|
415
|
+
|
|
416
|
+
# corrs->weights
|
|
417
|
+
C[C < min_corr] = 0.0
|
|
418
|
+
|
|
419
|
+
# call out to dredge
|
|
420
|
+
Sigma0inv = np.eye(C.shape[0]) * eps
|
|
421
|
+
p, *_ = newton_solve_rigid(D, C, Sigma0inv)
|
|
422
|
+
p = np.rint(p).astype(offsets.dtype)
|
|
423
|
+
|
|
424
|
+
# adjust offsets
|
|
425
|
+
offsets = torch.asarray(offsets) - torch.asarray(p)
|
|
426
|
+
|
|
427
|
+
return offsets
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _pairwise_correlate_templates(templates, ii, jj, max_shift: int, batch_size=256):
|
|
431
|
+
lags = torch.arange(-max_shift, max_shift + 1).to(templates.device)
|
|
432
|
+
k = len(templates)
|
|
433
|
+
npair = len(ii)
|
|
434
|
+
|
|
435
|
+
# compute norms...
|
|
436
|
+
normsa = templates.new_zeros((k, lags.shape[0]))
|
|
437
|
+
normsb = templates.new_zeros((k, lags.shape[0]))
|
|
438
|
+
for i0 in range(0, k, batch_size):
|
|
439
|
+
i1 = min(k, i0 + batch_size)
|
|
440
|
+
|
|
441
|
+
filt = torch.square(templates[i0:i1])[:, None]
|
|
442
|
+
ones = torch.ones_like(templates[i0:i1])[None, :]
|
|
443
|
+
conv = F.conv2d(ones, filt, padding=(max_shift, 0), groups=i1 - i0)
|
|
444
|
+
assert conv.shape == (1, i1 - i0, lags.shape[0], 1)
|
|
445
|
+
conv = conv[0, :, :, 0].sqrt_()
|
|
446
|
+
normsa[i0:i1] = conv
|
|
447
|
+
normsb[i0:i1] = torch.flip(conv, dims=(1,))
|
|
448
|
+
|
|
449
|
+
best_lag = templates.new_zeros(ii.shape)
|
|
450
|
+
best_corr = templates.new_zeros(ii.shape)
|
|
451
|
+
|
|
452
|
+
for i0 in trange(0, npair, batch_size, desc="DREDge template alignment"):
|
|
453
|
+
i1 = min(npair, i0 + batch_size)
|
|
454
|
+
|
|
455
|
+
iib = ii[i0:i1]
|
|
456
|
+
jjb = jj[i0:i1]
|
|
457
|
+
|
|
458
|
+
filt = templates[iib][:, None]
|
|
459
|
+
inpt = templates[jjb][None, :]
|
|
460
|
+
conv = F.conv2d(inpt, filt, padding=(max_shift, 0), groups=i1 - i0)
|
|
461
|
+
assert conv.shape == (1, i1 - i0, lags.shape[0], 1)
|
|
462
|
+
conv = conv[0, :, :, 0].div_(normsa[iib] * normsb[jjb])
|
|
463
|
+
best_corr[i0:i1], lag_ix = conv.max(dim=1)
|
|
464
|
+
best_lag[i0:i1] = lags[lag_ix]
|
|
465
|
+
|
|
466
|
+
return best_lag, best_corr
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
from dataclasses import replace
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from dartsort.util import drift_util
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
logger = getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def superres_sorting(
|
|
12
|
+
sorting,
|
|
13
|
+
geom,
|
|
14
|
+
motion_est=None,
|
|
15
|
+
strategy="motion_estimate",
|
|
16
|
+
localizations_dataset_name="point_source_localizations",
|
|
17
|
+
superres_bin_size_um=10.0,
|
|
18
|
+
min_spikes_per_bin=5,
|
|
19
|
+
probe_margin_um=200.0,
|
|
20
|
+
):
|
|
21
|
+
"""Construct the spatially superresolved spike train
|
|
22
|
+
|
|
23
|
+
The superres spike train assigns new labels to each unit
|
|
24
|
+
to capture shape variation due to vertical drift. Each
|
|
25
|
+
unit is broken into several sub-units, and spikes are
|
|
26
|
+
assigned into sub-units (superres units) depending on
|
|
27
|
+
their position and estimated displacement.
|
|
28
|
+
|
|
29
|
+
This function just needs to compute superres bin labels;
|
|
30
|
+
the pitch shifting is taken care of during template
|
|
31
|
+
computation.
|
|
32
|
+
|
|
33
|
+
There is a strategy flag here to allow experiments
|
|
34
|
+
with how this behaves.
|
|
35
|
+
|
|
36
|
+
Strategies:
|
|
37
|
+
- motion_estimate : bin spikes purely based on the drift
|
|
38
|
+
estimate. if the motion estimate is entirely correct,
|
|
39
|
+
then that means we should bin by the displacement estimate
|
|
40
|
+
mod the pitch.
|
|
41
|
+
- drift_pitch_loc_bin : spikes are coarsely motion corrected
|
|
42
|
+
by integer multiples of the pitch using the motion estimate,
|
|
43
|
+
and then these coarsely corrected positions are used to bin
|
|
44
|
+
for superres
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
dict with keys : values
|
|
50
|
+
group_ids : np.array
|
|
51
|
+
Int array such that group_ids[superres_id] is the original id of
|
|
52
|
+
superres unit superres_id
|
|
53
|
+
sorting : DARTsortSorting
|
|
54
|
+
properties : dict of np.array
|
|
55
|
+
Each array in here is of shape (n_superres_units,) and contains
|
|
56
|
+
bookkeeping information about the superres process.
|
|
57
|
+
For instance, for motion_estimate binning, we store the centers
|
|
58
|
+
of the bins (which are modulo pitch).
|
|
59
|
+
"""
|
|
60
|
+
pitch = drift_util.get_pitch(geom)
|
|
61
|
+
full_labels = sorting.labels.copy()
|
|
62
|
+
|
|
63
|
+
# load spike depths
|
|
64
|
+
# TODO: relying on this index feels wrong
|
|
65
|
+
spike_times_s = sorting.times_seconds
|
|
66
|
+
spike_depths_um = getattr(sorting, localizations_dataset_name)[:, 2]
|
|
67
|
+
|
|
68
|
+
# remove spikes far away from the probe
|
|
69
|
+
if probe_margin_um is not None:
|
|
70
|
+
valid = spike_depths_um == np.clip(
|
|
71
|
+
spike_depths_um,
|
|
72
|
+
geom[:, 1].min() - probe_margin_um,
|
|
73
|
+
geom[:, 1].max() + probe_margin_um,
|
|
74
|
+
)
|
|
75
|
+
full_labels[~valid] = -1
|
|
76
|
+
|
|
77
|
+
# handle triaging
|
|
78
|
+
kept = np.flatnonzero(full_labels >= 0)
|
|
79
|
+
labels = full_labels[kept]
|
|
80
|
+
spike_times_s = spike_times_s[kept]
|
|
81
|
+
spike_depths_um = spike_depths_um[kept]
|
|
82
|
+
|
|
83
|
+
# make superres spike train
|
|
84
|
+
properties = {}
|
|
85
|
+
if strategy in ("none", None):
|
|
86
|
+
superres_to_original = np.arange(labels.max() + 1)
|
|
87
|
+
superres_sorting = sorting
|
|
88
|
+
elif strategy == "motion_estimate":
|
|
89
|
+
superres_labels, superres_to_original, bin_centers = motion_estimate_strategy(
|
|
90
|
+
labels,
|
|
91
|
+
spike_times_s,
|
|
92
|
+
spike_depths_um,
|
|
93
|
+
pitch,
|
|
94
|
+
motion_est,
|
|
95
|
+
superres_bin_size_um=superres_bin_size_um,
|
|
96
|
+
)
|
|
97
|
+
properties["motion_estimate_bin_centers"] = bin_centers
|
|
98
|
+
elif strategy == "drift_pitch_loc_bin":
|
|
99
|
+
superres_labels, superres_to_original = drift_pitch_loc_bin_strategy(
|
|
100
|
+
labels,
|
|
101
|
+
spike_times_s,
|
|
102
|
+
spike_depths_um,
|
|
103
|
+
pitch,
|
|
104
|
+
motion_est,
|
|
105
|
+
superres_bin_size_um=superres_bin_size_um,
|
|
106
|
+
)
|
|
107
|
+
else:
|
|
108
|
+
raise ValueError(f"Unknown superres {strategy=}")
|
|
109
|
+
|
|
110
|
+
# handle too-small units
|
|
111
|
+
superres_labels, superres_to_original = remove_small_superres_units(
|
|
112
|
+
superres_labels, superres_to_original, min_spikes_per_bin=min_spikes_per_bin # type: ignore
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# back to un-triaged label space
|
|
116
|
+
full_labels[kept] = superres_labels
|
|
117
|
+
superres_sorting = sorting.ephemeral_replace(labels=full_labels)
|
|
118
|
+
return dict(
|
|
119
|
+
group_ids=superres_to_original,
|
|
120
|
+
sorting=superres_sorting,
|
|
121
|
+
properties=properties,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def motion_estimate_strategy(
|
|
126
|
+
original_labels,
|
|
127
|
+
spike_times_s,
|
|
128
|
+
spike_depths_um,
|
|
129
|
+
pitch,
|
|
130
|
+
motion_est,
|
|
131
|
+
superres_bin_size_um=10.0,
|
|
132
|
+
bin_round_atol=0.1,
|
|
133
|
+
):
|
|
134
|
+
""" """
|
|
135
|
+
# reg_pos = pos - disp, pos = reg_pos + disp
|
|
136
|
+
# so, disp is the motion of spikes relative to fixed probe
|
|
137
|
+
if motion_est is None:
|
|
138
|
+
displacements = np.zeros_like(spike_depths_um)
|
|
139
|
+
else:
|
|
140
|
+
displacements = motion_est.disp_at_s(spike_times_s, spike_depths_um)
|
|
141
|
+
|
|
142
|
+
n_bins = pitch // superres_bin_size_um
|
|
143
|
+
remainder = pitch - n_bins * superres_bin_size_um
|
|
144
|
+
if not np.isclose(remainder, 0.0):
|
|
145
|
+
n_bins = n_bins + (remainder > bin_round_atol)
|
|
146
|
+
logger.info(
|
|
147
|
+
f"Superres bin size didn't divide the pitch. Rounding it down from "
|
|
148
|
+
f"{superres_bin_size_um} to {pitch / n_bins}, for {n_bins} bins."
|
|
149
|
+
)
|
|
150
|
+
superres_bin_size_um = pitch / n_bins
|
|
151
|
+
|
|
152
|
+
mod_positions = displacements % pitch
|
|
153
|
+
bin_centers = np.arange(n_bins) * superres_bin_size_um + superres_bin_size_um / 2
|
|
154
|
+
bin_ids = mod_positions // superres_bin_size_um
|
|
155
|
+
bin_ids = bin_ids.astype(original_labels.dtype)
|
|
156
|
+
orig_label_and_bin, superres_labels = np.unique(
|
|
157
|
+
np.c_[original_labels, bin_ids], axis=0, return_inverse=True
|
|
158
|
+
)
|
|
159
|
+
superres_to_original = orig_label_and_bin[:, 0]
|
|
160
|
+
bin_centers = bin_centers[orig_label_and_bin[:, 1]]
|
|
161
|
+
return superres_labels, superres_to_original, bin_centers
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def drift_pitch_loc_bin_strategy(
|
|
165
|
+
original_labels,
|
|
166
|
+
spike_times_s,
|
|
167
|
+
spike_depths_um,
|
|
168
|
+
pitch,
|
|
169
|
+
motion_est,
|
|
170
|
+
superres_bin_size_um=10.0,
|
|
171
|
+
):
|
|
172
|
+
n_pitches_shift = drift_util.get_spike_pitch_shifts(
|
|
173
|
+
spike_depths_um, pitch=pitch, times_s=spike_times_s, motion_est=motion_est
|
|
174
|
+
)
|
|
175
|
+
coarse_reg_depths = spike_depths_um + n_pitches_shift * pitch
|
|
176
|
+
|
|
177
|
+
bin_ids = coarse_reg_depths // superres_bin_size_um
|
|
178
|
+
bin_ids = bin_ids.astype(int)
|
|
179
|
+
orig_label_and_bin, superres_labels = np.unique(
|
|
180
|
+
np.c_[original_labels, bin_ids], axis=0, return_inverse=True
|
|
181
|
+
)
|
|
182
|
+
superres_to_original = orig_label_and_bin[:, 0]
|
|
183
|
+
return superres_labels, superres_to_original
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def remove_small_superres_units(
|
|
187
|
+
superres_labels, superres_to_original, min_spikes_per_bin
|
|
188
|
+
):
|
|
189
|
+
if not min_spikes_per_bin:
|
|
190
|
+
return superres_labels, superres_to_original
|
|
191
|
+
|
|
192
|
+
slabels, scounts = np.unique(superres_labels, return_counts=True)
|
|
193
|
+
|
|
194
|
+
# new labels
|
|
195
|
+
kept = scounts >= min_spikes_per_bin
|
|
196
|
+
kept_labels = slabels[kept]
|
|
197
|
+
relabeling = np.full_like(slabels, -1)
|
|
198
|
+
relabeling[kept] = np.arange(kept_labels.size)
|
|
199
|
+
|
|
200
|
+
# relabel
|
|
201
|
+
superres_labels = relabeling[superres_labels]
|
|
202
|
+
superres_to_original = superres_to_original[kept_labels]
|
|
203
|
+
|
|
204
|
+
return superres_labels, superres_to_original
|