DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Literal, Self, Sequence, cast
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as F
|
|
7
|
+
from dredge.motion_util import MotionEstimate
|
|
8
|
+
from spikeinterface.core import BaseRecording
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
from ...templates import TemplateData
|
|
12
|
+
from ...util.internal_config import ComputationConfig, MatchingConfig
|
|
13
|
+
from ...util.logging_util import DARTSORTVERBOSE, get_logger
|
|
14
|
+
from ...util.py_util import databag
|
|
15
|
+
from ...util.spiketorch import argrelmax, grab_spikes, ptp
|
|
16
|
+
from ...util.torch_util import BModule
|
|
17
|
+
|
|
18
|
+
logger = get_logger(__name__)
|
|
19
|
+
_extra_checks = logger.isEnabledFor(DARTSORTVERBOSE)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MatchingTemplates(BModule):
|
|
23
|
+
# subclasses should have their own template_type
|
|
24
|
+
template_type = "base"
|
|
25
|
+
# shared subclass registry for from_config()
|
|
26
|
+
_registry = {}
|
|
27
|
+
|
|
28
|
+
# these should be assigned in __init__. adding here for typing.
|
|
29
|
+
spike_length_samples: int
|
|
30
|
+
|
|
31
|
+
def __init_subclass__(cls):
|
|
32
|
+
logger.dartsortverbose("Register templates type: %s", cls.template_type)
|
|
33
|
+
cls._registry[cls.template_type] = cls
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_config(
|
|
37
|
+
cls,
|
|
38
|
+
save_folder: Path,
|
|
39
|
+
recording: BaseRecording,
|
|
40
|
+
template_data: TemplateData,
|
|
41
|
+
matching_cfg: MatchingConfig,
|
|
42
|
+
computation_cfg: ComputationConfig | None = None,
|
|
43
|
+
motion_est=None,
|
|
44
|
+
whitener: Tensor | None = None,
|
|
45
|
+
overwrite: bool = False,
|
|
46
|
+
dtype=torch.float,
|
|
47
|
+
) -> Self:
|
|
48
|
+
global _extra_checks
|
|
49
|
+
_extra_checks = logger.isEnabledFor(DARTSORTVERBOSE)
|
|
50
|
+
if _extra_checks:
|
|
51
|
+
logger.dartsortverbose(f"Extra checks enabled in matching.")
|
|
52
|
+
return cls._registry[matching_cfg.template_type]._from_config(
|
|
53
|
+
save_folder=save_folder,
|
|
54
|
+
recording=recording,
|
|
55
|
+
template_data=template_data,
|
|
56
|
+
matching_cfg=matching_cfg,
|
|
57
|
+
computation_cfg=computation_cfg,
|
|
58
|
+
motion_est=motion_est,
|
|
59
|
+
overwrite=overwrite,
|
|
60
|
+
whitener=whitener,
|
|
61
|
+
dtype=dtype,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def _from_config(
|
|
66
|
+
cls,
|
|
67
|
+
save_folder: Path,
|
|
68
|
+
recording: BaseRecording,
|
|
69
|
+
template_data: TemplateData,
|
|
70
|
+
matching_cfg: MatchingConfig,
|
|
71
|
+
computation_cfg: ComputationConfig | None = None,
|
|
72
|
+
motion_est=None,
|
|
73
|
+
whitener=None,
|
|
74
|
+
overwrite: bool = False,
|
|
75
|
+
dtype=torch.float,
|
|
76
|
+
) -> Self:
|
|
77
|
+
raise NotImplementedError
|
|
78
|
+
|
|
79
|
+
def data_at_time(
|
|
80
|
+
self,
|
|
81
|
+
t_s: float,
|
|
82
|
+
scaling: bool,
|
|
83
|
+
inv_lambda: float,
|
|
84
|
+
scale_min: float,
|
|
85
|
+
scale_max: float,
|
|
86
|
+
) -> "ChunkTemplateData":
|
|
87
|
+
raise NotImplementedError
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(kw_only=True, frozen=True)
|
|
91
|
+
class MatchingTemplatesBuilder:
|
|
92
|
+
"""Helper so that the matching peeler can be a little lazy
|
|
93
|
+
|
|
94
|
+
The matcher's from_config passes the builder, and build() is called in
|
|
95
|
+
its precompute_peeling_data(). This exists so that resuming the sorter
|
|
96
|
+
using run_peeler() logic after matching doesn't require doing a million
|
|
97
|
+
SVDs and whatnot -- the peeler has to be constructed, but we can keep
|
|
98
|
+
this stuff lazy.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
recording: BaseRecording
|
|
102
|
+
template_data: TemplateData
|
|
103
|
+
matching_cfg: MatchingConfig
|
|
104
|
+
motion_est: MotionEstimate | None = None
|
|
105
|
+
whitener: Tensor | None = None
|
|
106
|
+
dtype: torch.dtype = torch.float
|
|
107
|
+
|
|
108
|
+
def build(
|
|
109
|
+
self,
|
|
110
|
+
save_folder: Path,
|
|
111
|
+
computation_cfg: ComputationConfig | None,
|
|
112
|
+
overwrite: bool = False,
|
|
113
|
+
) -> MatchingTemplates:
|
|
114
|
+
return MatchingTemplates.from_config(
|
|
115
|
+
save_folder=save_folder,
|
|
116
|
+
recording=self.recording,
|
|
117
|
+
template_data=self.template_data,
|
|
118
|
+
matching_cfg=self.matching_cfg,
|
|
119
|
+
computation_cfg=computation_cfg,
|
|
120
|
+
motion_est=self.motion_est,
|
|
121
|
+
whitener=self.whitener,
|
|
122
|
+
dtype=self.dtype,
|
|
123
|
+
overwrite=overwrite,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def spike_length_samples(self) -> int:
|
|
128
|
+
return self.template_data.spike_length_samples
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class ChunkTemplateData:
|
|
132
|
+
# -- subclasses must assign the following properties that the matcher uses.
|
|
133
|
+
spike_length_samples: int
|
|
134
|
+
# for the full templates
|
|
135
|
+
unit_ids: Tensor
|
|
136
|
+
main_channels: Tensor
|
|
137
|
+
# for obj templates
|
|
138
|
+
obj_normsq: Tensor
|
|
139
|
+
obj_n_templates: int
|
|
140
|
+
coarse_objective: bool
|
|
141
|
+
upsampling: bool
|
|
142
|
+
scaling: bool
|
|
143
|
+
needs_fine_pass: bool
|
|
144
|
+
up_factor: int
|
|
145
|
+
inv_lambda: Tensor
|
|
146
|
+
scale_min: Tensor
|
|
147
|
+
scale_max: Tensor
|
|
148
|
+
|
|
149
|
+
# -- subclasses must implement
|
|
150
|
+
|
|
151
|
+
def convolve(self, traces: Tensor, padding: int = 0, out: Tensor | None = None):
|
|
152
|
+
raise NotImplementedError
|
|
153
|
+
|
|
154
|
+
def subtract(self, traces: Tensor, peaks: "MatchingPeaks", sign: int = -1):
|
|
155
|
+
raise NotImplementedError
|
|
156
|
+
|
|
157
|
+
def subtract_conv(
|
|
158
|
+
self, conv: Tensor, peaks: "MatchingPeaks", padding=0, batch_size=256, sign=-1
|
|
159
|
+
):
|
|
160
|
+
raise NotImplementedError
|
|
161
|
+
|
|
162
|
+
def get_clean_waveforms(
|
|
163
|
+
self,
|
|
164
|
+
peaks: "MatchingPeaks",
|
|
165
|
+
channels: Tensor,
|
|
166
|
+
channel_index: Tensor,
|
|
167
|
+
add_into: Tensor | None = None,
|
|
168
|
+
):
|
|
169
|
+
raise NotImplementedError
|
|
170
|
+
|
|
171
|
+
def _enforce_refractory(
|
|
172
|
+
self, mask: Tensor, peaks: "MatchingPeaks", offset: int = 0, value=-torch.inf
|
|
173
|
+
):
|
|
174
|
+
raise NotImplementedError
|
|
175
|
+
|
|
176
|
+
def fine_match(
|
|
177
|
+
self,
|
|
178
|
+
*,
|
|
179
|
+
peaks: "MatchingPeaks",
|
|
180
|
+
residual: Tensor,
|
|
181
|
+
conv: Tensor,
|
|
182
|
+
padding: int = 0,
|
|
183
|
+
) -> "MatchingPeaks":
|
|
184
|
+
raise NotImplementedError
|
|
185
|
+
|
|
186
|
+
# this one is just for debugging / unit testing
|
|
187
|
+
def reconstruct_up_templates(self):
|
|
188
|
+
raise NotImplementedError
|
|
189
|
+
|
|
190
|
+
# -- super handles below
|
|
191
|
+
|
|
192
|
+
def enforce_refractory(self, mask, peaks, offset=0):
|
|
193
|
+
if not peaks.n_spikes:
|
|
194
|
+
return
|
|
195
|
+
self._enforce_refractory(mask, peaks, offset=offset, value=-torch.inf)
|
|
196
|
+
|
|
197
|
+
def forget_refractory(self, mask, peaks, offset=0):
|
|
198
|
+
if not peaks.n_spikes:
|
|
199
|
+
return
|
|
200
|
+
self._enforce_refractory(mask, peaks, offset=offset, value=0.0)
|
|
201
|
+
|
|
202
|
+
def unsubtract(self, traces: Tensor, peaks: "MatchingPeaks"):
|
|
203
|
+
return self.subtract(traces, peaks, sign=1)
|
|
204
|
+
|
|
205
|
+
def unsubtract_conv(
|
|
206
|
+
self, conv: Tensor, peaks: "MatchingPeaks", padding=0, batch_size=256
|
|
207
|
+
):
|
|
208
|
+
return self.subtract_conv(
|
|
209
|
+
conv=conv, peaks=peaks, padding=padding, batch_size=batch_size, sign=1
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
def obj_from_conv(
|
|
213
|
+
self,
|
|
214
|
+
*,
|
|
215
|
+
conv: Tensor,
|
|
216
|
+
out: Tensor,
|
|
217
|
+
scalings_out: Tensor | None = None,
|
|
218
|
+
) -> Tensor:
|
|
219
|
+
if self.scaling and self.inv_lambda == 0.0:
|
|
220
|
+
assert scalings_out is not None
|
|
221
|
+
return _free_coarse_objective(
|
|
222
|
+
conv=conv, normsq=self.obj_normsq, out=out, scalings=scalings_out
|
|
223
|
+
)
|
|
224
|
+
elif self.scaling:
|
|
225
|
+
assert scalings_out is not None
|
|
226
|
+
return _scaled_coarse_objective(
|
|
227
|
+
conv=conv,
|
|
228
|
+
normsq=self.obj_normsq,
|
|
229
|
+
out=out,
|
|
230
|
+
scalings=scalings_out,
|
|
231
|
+
inv_lambda=self.inv_lambda,
|
|
232
|
+
scale_min=self.scale_min,
|
|
233
|
+
scale_max=self.scale_max,
|
|
234
|
+
)
|
|
235
|
+
else:
|
|
236
|
+
return torch.add(
|
|
237
|
+
self.obj_normsq[:, None]._neg_view(), conv, alpha=2.0, out=out
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
def coarse_match(
|
|
241
|
+
self,
|
|
242
|
+
objective: Tensor,
|
|
243
|
+
scalings: Tensor | None,
|
|
244
|
+
thresholdsq: float,
|
|
245
|
+
obj_arange: Tensor,
|
|
246
|
+
) -> "MatchingPeaks":
|
|
247
|
+
objective_max, max_obj_template = objective.max(dim=0)
|
|
248
|
+
times = argrelmax(
|
|
249
|
+
x=objective_max,
|
|
250
|
+
radius=self.spike_length_samples,
|
|
251
|
+
threshold=thresholdsq,
|
|
252
|
+
arange=obj_arange[: objective_max.numel()],
|
|
253
|
+
)
|
|
254
|
+
n_spikes = times.numel()
|
|
255
|
+
if not n_spikes:
|
|
256
|
+
return MatchingPeaks()
|
|
257
|
+
|
|
258
|
+
objs = objective_max[times]
|
|
259
|
+
template_inds = max_obj_template[times]
|
|
260
|
+
if _extra_checks:
|
|
261
|
+
assert (objective_max[times] >= thresholdsq).all()
|
|
262
|
+
if self.scaling:
|
|
263
|
+
assert scalings is not None
|
|
264
|
+
scalings = scalings[template_inds, times]
|
|
265
|
+
else:
|
|
266
|
+
scalings = None
|
|
267
|
+
if _extra_checks:
|
|
268
|
+
assert (objs >= thresholdsq).all()
|
|
269
|
+
if _extra_checks and scalings is not None:
|
|
270
|
+
assert (scalings >= self.scale_min).all()
|
|
271
|
+
assert (scalings <= self.scale_max).all()
|
|
272
|
+
return MatchingPeaks(
|
|
273
|
+
times=times,
|
|
274
|
+
obj_template_inds=template_inds,
|
|
275
|
+
template_inds=template_inds,
|
|
276
|
+
scalings=scalings,
|
|
277
|
+
scores=objs,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
def get_collisioncleaned_waveforms(
|
|
281
|
+
self,
|
|
282
|
+
residual_padded: Tensor,
|
|
283
|
+
peaks: "MatchingPeaks",
|
|
284
|
+
channels: Tensor | Literal["template", "amplitude"],
|
|
285
|
+
channel_index: Tensor,
|
|
286
|
+
channel_selection_index: Tensor | None = None,
|
|
287
|
+
) -> tuple[Tensor, Tensor]:
|
|
288
|
+
if not peaks.n_spikes:
|
|
289
|
+
empty_channels = residual_padded.new_zeros(size=(0,), dtype=torch.long)
|
|
290
|
+
empty_waveforms = residual_padded.new_zeros(size=())
|
|
291
|
+
return empty_channels, empty_waveforms
|
|
292
|
+
|
|
293
|
+
if channels == "template":
|
|
294
|
+
channels = self.main_channels[peaks.template_inds]
|
|
295
|
+
selecting_channels = False
|
|
296
|
+
active_channel_index = channel_index
|
|
297
|
+
elif torch.is_tensor(channels):
|
|
298
|
+
selecting_channels = False
|
|
299
|
+
active_channel_index = channel_index
|
|
300
|
+
elif channels == "amplitude":
|
|
301
|
+
selecting_channels = True
|
|
302
|
+
assert channel_selection_index is not None
|
|
303
|
+
active_channel_index = channel_selection_index
|
|
304
|
+
channels = self.main_channels[peaks.template_inds]
|
|
305
|
+
else:
|
|
306
|
+
assert False
|
|
307
|
+
|
|
308
|
+
times = peaks.times
|
|
309
|
+
assert times is not None
|
|
310
|
+
|
|
311
|
+
# get noise
|
|
312
|
+
waveforms = grab_spikes(
|
|
313
|
+
residual_padded,
|
|
314
|
+
times,
|
|
315
|
+
channels,
|
|
316
|
+
active_channel_index,
|
|
317
|
+
trough_offset=0,
|
|
318
|
+
spike_length_samples=self.spike_length_samples,
|
|
319
|
+
buffer=0,
|
|
320
|
+
already_padded=True,
|
|
321
|
+
)
|
|
322
|
+
waveforms = self.get_clean_waveforms(
|
|
323
|
+
peaks=peaks,
|
|
324
|
+
channel_index=active_channel_index,
|
|
325
|
+
channels=channels,
|
|
326
|
+
add_into=waveforms,
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
if not selecting_channels:
|
|
330
|
+
return channels, waveforms
|
|
331
|
+
|
|
332
|
+
cix = ptp(waveforms).nan_to_num_(nan=-torch.inf).argmax(dim=1)
|
|
333
|
+
assert channel_selection_index is not None
|
|
334
|
+
sel = channel_selection_index[channels]
|
|
335
|
+
channels = sel.take_along_dim(cix[:, None], dim=1)[:, 0]
|
|
336
|
+
return self.get_collisioncleaned_waveforms(
|
|
337
|
+
residual_padded=residual_padded,
|
|
338
|
+
peaks=peaks,
|
|
339
|
+
channel_index=channel_index,
|
|
340
|
+
channels=channels,
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class PconvBase(BModule):
|
|
345
|
+
def query(
|
|
346
|
+
self,
|
|
347
|
+
template_indices_a,
|
|
348
|
+
template_indices_b,
|
|
349
|
+
upsampling_indices_b=None,
|
|
350
|
+
shifts_a=None,
|
|
351
|
+
shifts_b=None,
|
|
352
|
+
scalings_b=None,
|
|
353
|
+
):
|
|
354
|
+
raise NotImplementedError
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
@databag
|
|
358
|
+
class MatchingPeaks:
|
|
359
|
+
times: Tensor | None = None
|
|
360
|
+
obj_template_inds: Tensor | None = None
|
|
361
|
+
template_inds: Tensor | None = None
|
|
362
|
+
up_inds: Tensor | None = None
|
|
363
|
+
scalings: Tensor | None = None
|
|
364
|
+
scores: Tensor | None = None
|
|
365
|
+
time_shifts: Tensor | None = None
|
|
366
|
+
|
|
367
|
+
if _extra_checks:
|
|
368
|
+
|
|
369
|
+
def __post_init__(self):
|
|
370
|
+
if self.times is None:
|
|
371
|
+
assert self.obj_template_inds is None
|
|
372
|
+
assert self.template_inds is None
|
|
373
|
+
assert self.up_inds is None
|
|
374
|
+
assert self.scalings is None
|
|
375
|
+
assert self.scores is None
|
|
376
|
+
assert self.time_shifts is None
|
|
377
|
+
else:
|
|
378
|
+
assert self.times.ndim == 1
|
|
379
|
+
assert self.obj_template_inds is not None
|
|
380
|
+
assert self.times.shape == self.obj_template_inds.shape
|
|
381
|
+
assert self.template_inds is not None
|
|
382
|
+
assert self.times.shape == self.template_inds.shape
|
|
383
|
+
assert (self.up_inds is None) or (
|
|
384
|
+
self.times.shape == self.up_inds.shape
|
|
385
|
+
)
|
|
386
|
+
assert (self.scalings is None) or (
|
|
387
|
+
self.times.shape == self.scalings.shape
|
|
388
|
+
)
|
|
389
|
+
assert self.scores is not None
|
|
390
|
+
assert self.times.shape == self.scores.shape
|
|
391
|
+
assert (self.time_shifts is None) or (
|
|
392
|
+
self.times.shape == self.time_shifts.shape
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
@property
|
|
396
|
+
def n_spikes(self):
|
|
397
|
+
if self.times is None:
|
|
398
|
+
return 0
|
|
399
|
+
else:
|
|
400
|
+
return self.times.numel()
|
|
401
|
+
|
|
402
|
+
def subset_by_time(
|
|
403
|
+
self, min_time: int, max_time: int, offset: int, sort: bool = True
|
|
404
|
+
) -> Self:
|
|
405
|
+
if not self.n_spikes:
|
|
406
|
+
return self
|
|
407
|
+
assert self.times is not None
|
|
408
|
+
t = self.times + offset
|
|
409
|
+
mask = t == t.clamp(min_time, max_time)
|
|
410
|
+
return self.subset(mask=mask, sort=sort)
|
|
411
|
+
|
|
412
|
+
def subset(self, mask: Tensor, sort: bool = True) -> Self:
|
|
413
|
+
if not self.n_spikes:
|
|
414
|
+
assert not mask.numel()
|
|
415
|
+
return self
|
|
416
|
+
if mask.dtype == torch.bool:
|
|
417
|
+
(mask,) = mask.nonzero(as_tuple=True)
|
|
418
|
+
if sort:
|
|
419
|
+
assert self.times is not None
|
|
420
|
+
times = self.times[mask]
|
|
421
|
+
times, order = torch.sort(times)
|
|
422
|
+
mask = mask[order]
|
|
423
|
+
else:
|
|
424
|
+
times = _mask_or_none(self.times, mask)
|
|
425
|
+
return self.__class__(
|
|
426
|
+
times=times,
|
|
427
|
+
obj_template_inds=_mask_or_none(self.obj_template_inds, mask),
|
|
428
|
+
template_inds=_mask_or_none(self.template_inds, mask),
|
|
429
|
+
up_inds=_mask_or_none(self.up_inds, mask),
|
|
430
|
+
scalings=_mask_or_none(self.scalings, mask),
|
|
431
|
+
scores=_mask_or_none(self.scores, mask),
|
|
432
|
+
time_shifts=_mask_or_none(self.time_shifts, mask),
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
@classmethod
|
|
436
|
+
def concatenate(cls, peaks: list[Self]) -> Self:
|
|
437
|
+
if len(peaks) == 0:
|
|
438
|
+
return cls()
|
|
439
|
+
elif len(peaks) == 1:
|
|
440
|
+
return peaks[0]
|
|
441
|
+
return cls(
|
|
442
|
+
times=_cat_or_none([p.times for p in peaks]),
|
|
443
|
+
obj_template_inds=_cat_or_none([p.obj_template_inds for p in peaks]),
|
|
444
|
+
template_inds=_cat_or_none([p.template_inds for p in peaks]),
|
|
445
|
+
up_inds=_cat_or_none([p.up_inds for p in peaks]),
|
|
446
|
+
scalings=_cat_or_none([p.scalings for p in peaks]),
|
|
447
|
+
scores=_cat_or_none([p.scores for p in peaks]),
|
|
448
|
+
time_shifts=_cat_or_none([p.time_shifts for p in peaks]),
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _mask_or_none(x: Tensor | None, mask: Tensor) -> Tensor | None:
|
|
453
|
+
if x is None:
|
|
454
|
+
return None
|
|
455
|
+
else:
|
|
456
|
+
return x[mask]
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _cat_or_none(xs: list[Tensor | None]) -> Tensor | None:
|
|
460
|
+
if xs[0] is None:
|
|
461
|
+
return None
|
|
462
|
+
else:
|
|
463
|
+
return torch.concatenate(cast(list[Tensor], xs))
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
# -- matching helper fn library
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def subtract_precomputed_pconv(
|
|
470
|
+
*,
|
|
471
|
+
conv: Tensor,
|
|
472
|
+
pconv: Tensor,
|
|
473
|
+
peaks: MatchingPeaks,
|
|
474
|
+
conv_lags: Tensor,
|
|
475
|
+
sign: int,
|
|
476
|
+
padding: int,
|
|
477
|
+
batch_size: int = 128,
|
|
478
|
+
):
|
|
479
|
+
assert conv.shape[0] == pconv.shape[0]
|
|
480
|
+
assert sign in (-1, 1)
|
|
481
|
+
if not peaks.n_spikes:
|
|
482
|
+
return
|
|
483
|
+
padded_lags = padding + conv_lags
|
|
484
|
+
times = peaks.times
|
|
485
|
+
assert times is not None
|
|
486
|
+
assert peaks.template_inds is not None
|
|
487
|
+
up_inds = peaks.up_inds
|
|
488
|
+
if up_inds is None:
|
|
489
|
+
up_inds = torch.zeros_like(peaks.template_inds)
|
|
490
|
+
if peaks.scalings is None:
|
|
491
|
+
_subtract_precomputed_pconv_unscaled(
|
|
492
|
+
conv=conv,
|
|
493
|
+
pconv=pconv,
|
|
494
|
+
template_indices=peaks.template_inds,
|
|
495
|
+
upsampling_indices=up_inds,
|
|
496
|
+
times=times,
|
|
497
|
+
padded_conv_lags=padded_lags,
|
|
498
|
+
neg=sign == -1,
|
|
499
|
+
batch_size=batch_size,
|
|
500
|
+
)
|
|
501
|
+
else:
|
|
502
|
+
_subtract_precomputed_pconv_scaled(
|
|
503
|
+
conv=conv,
|
|
504
|
+
pconv=pconv,
|
|
505
|
+
template_indices=peaks.template_inds,
|
|
506
|
+
upsampling_indices=up_inds,
|
|
507
|
+
scalings=peaks.scalings,
|
|
508
|
+
times=times,
|
|
509
|
+
padded_conv_lags=padded_lags,
|
|
510
|
+
neg=sign == -1,
|
|
511
|
+
batch_size=batch_size,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
@torch.jit.script
|
|
516
|
+
def _subtract_precomputed_pconv_unscaled(
|
|
517
|
+
conv: Tensor,
|
|
518
|
+
pconv: Tensor,
|
|
519
|
+
template_indices: Tensor,
|
|
520
|
+
upsampling_indices: Tensor,
|
|
521
|
+
times: Tensor,
|
|
522
|
+
padded_conv_lags: Tensor,
|
|
523
|
+
neg: bool,
|
|
524
|
+
batch_size: int = 128,
|
|
525
|
+
):
|
|
526
|
+
ix_time = times[:, None] + padded_conv_lags[None, :]
|
|
527
|
+
for i0 in range(0, conv.shape[0], batch_size):
|
|
528
|
+
i1 = min(conv.shape[0], i0 + batch_size)
|
|
529
|
+
batch = pconv[i0:i1, template_indices, upsampling_indices]
|
|
530
|
+
ix = ix_time.broadcast_to(batch.shape)
|
|
531
|
+
batch = batch.reshape(i1 - i0, -1)
|
|
532
|
+
ix = ix.reshape(i1 - i0, batch.shape[1])
|
|
533
|
+
if neg:
|
|
534
|
+
batch = batch._neg_view()
|
|
535
|
+
conv[i0:i1].scatter_add_(dim=1, src=batch, index=ix)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
@torch.jit.script
|
|
539
|
+
def _subtract_precomputed_pconv_scaled(
|
|
540
|
+
conv: Tensor,
|
|
541
|
+
pconv: Tensor,
|
|
542
|
+
template_indices: Tensor,
|
|
543
|
+
upsampling_indices: Tensor,
|
|
544
|
+
scalings: Tensor,
|
|
545
|
+
times: Tensor,
|
|
546
|
+
padded_conv_lags: Tensor,
|
|
547
|
+
neg: bool,
|
|
548
|
+
batch_size: int = 128,
|
|
549
|
+
):
|
|
550
|
+
ix_time = times[:, None] + padded_conv_lags[None, :]
|
|
551
|
+
scalings = scalings[None, :, None]
|
|
552
|
+
for i0 in range(0, conv.shape[0], batch_size):
|
|
553
|
+
i1 = min(conv.shape[0], i0 + batch_size)
|
|
554
|
+
batch = pconv[i0:i1, template_indices, upsampling_indices]
|
|
555
|
+
batch.mul_(scalings)
|
|
556
|
+
ix = ix_time.broadcast_to(batch.shape)
|
|
557
|
+
batch = batch.reshape(i1 - i0, -1)
|
|
558
|
+
ix = ix.reshape(i1 - i0, batch.shape[1])
|
|
559
|
+
if neg:
|
|
560
|
+
batch = batch._neg_view()
|
|
561
|
+
conv[i0:i1].scatter_add_(dim=1, src=batch, index=ix)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
@torch.jit.script
|
|
565
|
+
def _free_coarse_objective(
|
|
566
|
+
conv: Tensor,
|
|
567
|
+
normsq: Tensor,
|
|
568
|
+
out: Tensor,
|
|
569
|
+
scalings: Tensor,
|
|
570
|
+
) -> Tensor:
|
|
571
|
+
out.copy_(conv)
|
|
572
|
+
F.relu(out, inplace=True)
|
|
573
|
+
torch.divide(out, normsq[:, None], out=scalings)
|
|
574
|
+
obj = out.mul_(scalings)
|
|
575
|
+
return obj
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
@torch.jit.script
|
|
579
|
+
def _scaled_coarse_objective(
|
|
580
|
+
conv: Tensor,
|
|
581
|
+
normsq: Tensor,
|
|
582
|
+
out: Tensor,
|
|
583
|
+
scalings: Tensor,
|
|
584
|
+
inv_lambda: Tensor,
|
|
585
|
+
scale_min: Tensor,
|
|
586
|
+
scale_max: Tensor,
|
|
587
|
+
) -> Tensor:
|
|
588
|
+
neg = conv < 0
|
|
589
|
+
b = conv + inv_lambda
|
|
590
|
+
a = normsq[:, None] + inv_lambda
|
|
591
|
+
torch.divide(b, a, out=scalings)
|
|
592
|
+
scalings.clamp_(min=scale_min, max=scale_max)
|
|
593
|
+
scalings.masked_fill_(neg, 0.0)
|
|
594
|
+
# this is 2 * sc * b - sc**2 * a - inv_lambda
|
|
595
|
+
torch.square(scalings, out=out)
|
|
596
|
+
torch.addcmul(-inv_lambda, -a, out, out=out)
|
|
597
|
+
out.addcmul_(scalings, b, value=2.0)
|
|
598
|
+
return out
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
@torch.jit.script
|
|
602
|
+
def _coarse_match_scaled(
|
|
603
|
+
conv: Tensor,
|
|
604
|
+
template_inds: Tensor,
|
|
605
|
+
times: Tensor,
|
|
606
|
+
obj_normsq: Tensor,
|
|
607
|
+
inv_lambda: Tensor,
|
|
608
|
+
scale_min: Tensor,
|
|
609
|
+
scale_max: Tensor,
|
|
610
|
+
):
|
|
611
|
+
b = conv[template_inds, times] + inv_lambda
|
|
612
|
+
a = obj_normsq[template_inds] + inv_lambda
|
|
613
|
+
scalings = b.div(a).clamp_(min=scale_min, max=scale_max)
|
|
614
|
+
objs = scalings.square().mul_(-a)
|
|
615
|
+
objs.addcmul_(scalings, b, value=2.0).sub_(inv_lambda)
|
|
616
|
+
return scalings, objs
|