DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Literal, Self, cast
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as F
|
|
7
|
+
from scipy.signal import correlate
|
|
8
|
+
from spikeinterface.core import BaseRecording
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
from ...peel.matching import (
|
|
12
|
+
ChunkTemplateData,
|
|
13
|
+
MatchingPeaks,
|
|
14
|
+
MatchingTemplates,
|
|
15
|
+
ObjectiveUpdateTemplateMatchingPeeler,
|
|
16
|
+
)
|
|
17
|
+
from ...peel.matching_util.matching_base import subtract_precomputed_pconv
|
|
18
|
+
from ...templates.templates import TemplateData
|
|
19
|
+
from ..data_util import DARTsortSorting
|
|
20
|
+
from ..internal_config import ComputationConfig, MatchingConfig
|
|
21
|
+
from ..job_util import ensure_computation_config
|
|
22
|
+
from ..py_util import databag
|
|
23
|
+
from ..waveform_util import upsample_multichan
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def reference_shared_temporal_convolution(
|
|
27
|
+
temporal: np.ndarray, temporal_up: np.ndarray
|
|
28
|
+
):
|
|
29
|
+
"""Compute the same convolution of temporal basis two ways."""
|
|
30
|
+
rank, up, t = temporal_up.shape
|
|
31
|
+
assert temporal.shape == (rank, t)
|
|
32
|
+
conv_len = 2 * t - 1
|
|
33
|
+
|
|
34
|
+
tconv0 = np.zeros((rank, rank, up, conv_len), dtype=np.float32)
|
|
35
|
+
tconv00 = np.zeros((rank, rank, up, conv_len), dtype=np.float32)
|
|
36
|
+
for p in range(rank):
|
|
37
|
+
for q in range(rank):
|
|
38
|
+
for u in range(up):
|
|
39
|
+
tconv0[p, q, u] = correlate(
|
|
40
|
+
temporal[p], temporal_up[q, u], mode="full", method="direct"
|
|
41
|
+
)[::-1]
|
|
42
|
+
tconv00[p, q, u] = correlate(
|
|
43
|
+
temporal[p][::-1],
|
|
44
|
+
temporal_up[q, u][::-1],
|
|
45
|
+
mode="full",
|
|
46
|
+
method="direct",
|
|
47
|
+
)
|
|
48
|
+
return tconv0, tconv00
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def reference_pairwise_convolution(
|
|
52
|
+
templates: np.ndarray,
|
|
53
|
+
templates_up: np.ndarray,
|
|
54
|
+
accum_dtype: np.typing.DTypeLike = np.float32,
|
|
55
|
+
):
|
|
56
|
+
K, up, t, nc = templates_up.shape
|
|
57
|
+
assert templates.shape == (K, t, nc)
|
|
58
|
+
conv_len = 2 * t - 1
|
|
59
|
+
pconv = np.zeros((K, K, up, conv_len), dtype=np.float32)
|
|
60
|
+
tmp = np.zeros(conv_len, dtype=accum_dtype)
|
|
61
|
+
for i in range(K):
|
|
62
|
+
for j in range(K):
|
|
63
|
+
for u in range(up):
|
|
64
|
+
tmp[:] = 0.0
|
|
65
|
+
for c in range(nc):
|
|
66
|
+
tmp += correlate(
|
|
67
|
+
templates[i, :, c].astype(accum_dtype)[::-1],
|
|
68
|
+
templates_up[j, u, :, c].astype(accum_dtype)[::-1],
|
|
69
|
+
mode="full",
|
|
70
|
+
method="direct",
|
|
71
|
+
)
|
|
72
|
+
pconv[i, j, u] = tmp.astype(np.float32)
|
|
73
|
+
return pconv
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def yield_step_results(
|
|
77
|
+
matcher: ObjectiveUpdateTemplateMatchingPeeler,
|
|
78
|
+
chunk,
|
|
79
|
+
t_s: float,
|
|
80
|
+
max_iter: int = 5,
|
|
81
|
+
obj_mode=False,
|
|
82
|
+
):
|
|
83
|
+
device = matcher.b.channel_index.device
|
|
84
|
+
chunk = torch.asarray(chunk, device=device)
|
|
85
|
+
assert matcher.matching_templates is not None
|
|
86
|
+
chunk_data = matcher.matching_templates.data_at_time(
|
|
87
|
+
t_s,
|
|
88
|
+
scaling=matcher.is_scaling,
|
|
89
|
+
inv_lambda=matcher.inv_lambda,
|
|
90
|
+
scale_min=matcher.amp_scale_min,
|
|
91
|
+
scale_max=matcher.amp_scale_max,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
cur_residual = chunk.clone()
|
|
95
|
+
for _ in range(max_iter):
|
|
96
|
+
pre_conv = chunk_data.convolve(cur_residual.T, padding=matcher.obj_pad_len)
|
|
97
|
+
if obj_mode:
|
|
98
|
+
pre_conv = chunk_data.obj_from_conv(
|
|
99
|
+
conv=pre_conv,
|
|
100
|
+
out=torch.zeros_like(pre_conv),
|
|
101
|
+
scalings_out=torch.zeros_like(pre_conv) if matcher.is_scaling else None,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
chk = matcher.match_chunk(
|
|
105
|
+
cur_residual,
|
|
106
|
+
chunk_data,
|
|
107
|
+
return_conv=True,
|
|
108
|
+
return_residual=True,
|
|
109
|
+
max_iter=1,
|
|
110
|
+
)
|
|
111
|
+
cur_residual = chk["residual"].clone()
|
|
112
|
+
|
|
113
|
+
pre_conv = pre_conv.numpy(force=True)
|
|
114
|
+
resid = chk["residual"].numpy(force=True)
|
|
115
|
+
if obj_mode:
|
|
116
|
+
conv = chunk_data.obj_from_conv(
|
|
117
|
+
conv=chk["conv"],
|
|
118
|
+
out=torch.zeros_like(chk["conv"]),
|
|
119
|
+
scalings_out=torch.zeros_like(chk["conv"])
|
|
120
|
+
if matcher.is_scaling
|
|
121
|
+
else None,
|
|
122
|
+
).numpy(force=True)
|
|
123
|
+
else:
|
|
124
|
+
conv = chk["conv"].numpy(force=True)
|
|
125
|
+
if not chk["n_spikes"]:
|
|
126
|
+
break
|
|
127
|
+
times_samples = chk["times_samples"].numpy(force=True)
|
|
128
|
+
labels = chk["labels"].numpy(force=True)
|
|
129
|
+
channels = chk["channels"].numpy(force=True)
|
|
130
|
+
|
|
131
|
+
yield resid, pre_conv, conv, times_samples, labels, channels
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def visualize_step_results(
|
|
135
|
+
matcher: ObjectiveUpdateTemplateMatchingPeeler,
|
|
136
|
+
chunk: np.ndarray,
|
|
137
|
+
t_s: float,
|
|
138
|
+
chunk_start_samples: int = 0,
|
|
139
|
+
max_iter: int = 5,
|
|
140
|
+
cmap="berlin",
|
|
141
|
+
figsize=(10, 10),
|
|
142
|
+
s=10,
|
|
143
|
+
vis_start=None,
|
|
144
|
+
vis_end=None,
|
|
145
|
+
obj_mode=False,
|
|
146
|
+
chunk_vis_style: Literal["im", "trace"] = "im",
|
|
147
|
+
gt_sorting: DARTsortSorting | None = None,
|
|
148
|
+
):
|
|
149
|
+
import matplotlib.pyplot as plt
|
|
150
|
+
|
|
151
|
+
from ...vis import glasbey1024
|
|
152
|
+
|
|
153
|
+
if vis_start is None:
|
|
154
|
+
vis_start = 0
|
|
155
|
+
if vis_end is None:
|
|
156
|
+
vis_end = chunk.shape[0]
|
|
157
|
+
vis_len = vis_end - vis_start
|
|
158
|
+
chunk_sl = slice(vis_start, vis_end)
|
|
159
|
+
|
|
160
|
+
if gt_sorting is not None:
|
|
161
|
+
gt_t = gt_sorting.times_samples - chunk_start_samples
|
|
162
|
+
gtvalid = np.flatnonzero(gt_t == gt_t.clip(vis_start, vis_end - 1))
|
|
163
|
+
gt_t = gt_t[gtvalid] - vis_start
|
|
164
|
+
gt_chan = gt_sorting.channels[gtvalid]
|
|
165
|
+
gt_l = cast(np.ndarray, gt_sorting.labels)[gtvalid]
|
|
166
|
+
gt_c = glasbey1024[gt_l]
|
|
167
|
+
else:
|
|
168
|
+
gt_t = gt_chan = gt_c = None
|
|
169
|
+
|
|
170
|
+
obj_sl = slice(
|
|
171
|
+
max(vis_start + matcher.obj_pad_len, matcher.obj_pad_len),
|
|
172
|
+
min(vis_end, chunk.shape[0]),
|
|
173
|
+
)
|
|
174
|
+
obj_domain = matcher.trough_offset_samples + np.arange(obj_sl.stop - obj_sl.start)
|
|
175
|
+
|
|
176
|
+
t_full = np.zeros(chunk.size, dtype=np.int64)
|
|
177
|
+
c_full = np.zeros(chunk.size, dtype=np.int64)
|
|
178
|
+
l_full = np.zeros(chunk.size, dtype=np.int64)
|
|
179
|
+
n = 0
|
|
180
|
+
|
|
181
|
+
it = 0
|
|
182
|
+
for resid, pre_conv, conv, times_samples, labels, channels in yield_step_results(
|
|
183
|
+
matcher=matcher,
|
|
184
|
+
chunk=chunk,
|
|
185
|
+
t_s=t_s,
|
|
186
|
+
max_iter=max_iter,
|
|
187
|
+
obj_mode=obj_mode,
|
|
188
|
+
):
|
|
189
|
+
it += 1
|
|
190
|
+
|
|
191
|
+
v = np.flatnonzero(times_samples == times_samples.clip(vis_start, vis_end - 1))
|
|
192
|
+
times_samples = times_samples[v] - vis_start
|
|
193
|
+
labels = labels[v]
|
|
194
|
+
channels = channels[v]
|
|
195
|
+
|
|
196
|
+
nnew = times_samples.shape[0]
|
|
197
|
+
if not nnew:
|
|
198
|
+
continue
|
|
199
|
+
t_full[n : n + nnew] = times_samples
|
|
200
|
+
c_full[n : n + nnew] = channels
|
|
201
|
+
l_full[n : n + nnew] = labels
|
|
202
|
+
|
|
203
|
+
panel = plt.figure(figsize=figsize, layout="constrained")
|
|
204
|
+
axes = panel.subplots(nrows=6, sharex=True)
|
|
205
|
+
|
|
206
|
+
for x, ax, name in zip(
|
|
207
|
+
(chunk, resid, chunk - resid), axes, ("chunk", "resid", "signal")
|
|
208
|
+
):
|
|
209
|
+
if chunk_vis_style == "im":
|
|
210
|
+
ax.imshow(
|
|
211
|
+
x[chunk_sl].T,
|
|
212
|
+
vmin=-5,
|
|
213
|
+
vmax=5,
|
|
214
|
+
aspect="auto",
|
|
215
|
+
cmap=cmap,
|
|
216
|
+
origin="lower",
|
|
217
|
+
interpolation="none",
|
|
218
|
+
)
|
|
219
|
+
elif chunk_vis_style == "trace":
|
|
220
|
+
for j, trace in enumerate(x[chunk_sl].T):
|
|
221
|
+
ax.plot(j, color="k", zorder=1)
|
|
222
|
+
ax.plot(trace + j, color="k", zorder=2)
|
|
223
|
+
ax.scatter(
|
|
224
|
+
times_samples, channels, c=glasbey1024[labels], s=s, ec="w", lw=1
|
|
225
|
+
)
|
|
226
|
+
ax.set_ylabel(name)
|
|
227
|
+
|
|
228
|
+
if gt_t is not None:
|
|
229
|
+
axes[-3].scatter(gt_t, gt_chan, c=gt_c, s=4 * s, lw=0, marker="o")
|
|
230
|
+
|
|
231
|
+
axes[-3].scatter(
|
|
232
|
+
t_full[:n], c_full[:n], c=glasbey1024[l_full[:n]], s=s, lw=1, ec="k"
|
|
233
|
+
)
|
|
234
|
+
axes[-3].set_ylabel("channel scatter")
|
|
235
|
+
axes[-3].scatter(
|
|
236
|
+
times_samples, channels, c=glasbey1024[labels], s=s, ec="w", lw=1
|
|
237
|
+
)
|
|
238
|
+
n += nnew
|
|
239
|
+
|
|
240
|
+
for j, c in enumerate(pre_conv):
|
|
241
|
+
axes[-2].plot(obj_domain, c[obj_sl], color=glasbey1024[j], lw=0.5)
|
|
242
|
+
axes[-2].set_ylabel("pre-step " + ("obj" if obj_mode else "conv"))
|
|
243
|
+
for t, l in zip(times_samples, labels):
|
|
244
|
+
axes[-2].axvline(t, color=glasbey1024[l], lw=1, ls=":")
|
|
245
|
+
for j, c in enumerate(conv):
|
|
246
|
+
axes[-1].plot(obj_domain, c[obj_sl], color=glasbey1024[j], lw=0.5)
|
|
247
|
+
axes[-1].set_ylabel("post-step " + ("obj" if obj_mode else "conv"))
|
|
248
|
+
for ax in axes[-2:]:
|
|
249
|
+
ax.grid()
|
|
250
|
+
if obj_mode:
|
|
251
|
+
for ax in axes[-2:]:
|
|
252
|
+
ax.set_ylim([-100, pre_conv[:, obj_sl].max() * 1.05])
|
|
253
|
+
|
|
254
|
+
panel.suptitle(f"iteration {it}", fontsize=12)
|
|
255
|
+
|
|
256
|
+
plt.show()
|
|
257
|
+
plt.close(panel)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# -- reference implementation for upsampled matching
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class DebugMatchingTemplates(MatchingTemplates):
|
|
264
|
+
"""The goal with this one is to be obviously correct."""
|
|
265
|
+
|
|
266
|
+
template_type = "debug"
|
|
267
|
+
|
|
268
|
+
def __init__(self, templates_up: Tensor, refrac_radius: int):
|
|
269
|
+
super().__init__()
|
|
270
|
+
self.register_buffer("templates_up", templates_up)
|
|
271
|
+
pconv = reference_pairwise_convolution(
|
|
272
|
+
templates=templates_up[:, 0].numpy(force=True),
|
|
273
|
+
templates_up=templates_up.numpy(force=True),
|
|
274
|
+
)
|
|
275
|
+
pconv = torch.asarray(pconv, device=templates_up.device)
|
|
276
|
+
self.register_buffer("pconv", pconv)
|
|
277
|
+
refrac_ix = torch.arange(-refrac_radius, refrac_radius + 1, device=pconv.device)
|
|
278
|
+
conv_lags = torch.arange(
|
|
279
|
+
-templates_up.shape[2] + 1, templates_up.shape[2], device=pconv.device
|
|
280
|
+
)
|
|
281
|
+
self.register_buffer("refrac_ix", refrac_ix)
|
|
282
|
+
self.register_buffer("conv_lags", conv_lags)
|
|
283
|
+
main_channels = templates_up[:, 0].square().sum(dim=1).argmax(dim=1)
|
|
284
|
+
self.register_buffer("main_channels", main_channels)
|
|
285
|
+
|
|
286
|
+
@classmethod
|
|
287
|
+
def _from_config(
|
|
288
|
+
cls,
|
|
289
|
+
save_folder: Path,
|
|
290
|
+
recording: BaseRecording,
|
|
291
|
+
template_data: TemplateData,
|
|
292
|
+
matching_cfg: MatchingConfig,
|
|
293
|
+
computation_cfg: ComputationConfig | None = None,
|
|
294
|
+
motion_est=None,
|
|
295
|
+
whitener: Tensor | None = None,
|
|
296
|
+
overwrite: bool = False,
|
|
297
|
+
dtype=torch.float,
|
|
298
|
+
) -> Self:
|
|
299
|
+
assert motion_est is None
|
|
300
|
+
assert whitener is None
|
|
301
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
302
|
+
device = computation_cfg.actual_device()
|
|
303
|
+
templates_up = upsample_multichan(
|
|
304
|
+
template_data.templates, matching_cfg.template_temporal_upsampling_factor
|
|
305
|
+
)
|
|
306
|
+
assert templates_up.shape[0] == template_data.templates.shape[0]
|
|
307
|
+
assert templates_up.shape[1] == matching_cfg.template_temporal_upsampling_factor
|
|
308
|
+
assert templates_up.shape[3] == recording.get_num_channels()
|
|
309
|
+
assert template_data.templates.shape[2] == recording.get_num_channels()
|
|
310
|
+
templates_up = torch.asarray(templates_up, device=device, dtype=dtype)
|
|
311
|
+
return cls(
|
|
312
|
+
templates_up=templates_up,
|
|
313
|
+
refrac_radius=matching_cfg.refractory_radius_frames,
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
def data_at_time(
|
|
317
|
+
self,
|
|
318
|
+
t_s: float,
|
|
319
|
+
scaling: bool,
|
|
320
|
+
inv_lambda: float,
|
|
321
|
+
scale_min: float,
|
|
322
|
+
scale_max: float,
|
|
323
|
+
) -> ChunkTemplateData:
|
|
324
|
+
return DebugChunkTemplateData(
|
|
325
|
+
spike_length_samples=self.b.templates_up.shape[2],
|
|
326
|
+
unit_ids=torch.arange(
|
|
327
|
+
self.b.templates_up.shape[0], device=self.b.pconv.device
|
|
328
|
+
),
|
|
329
|
+
main_channels=self.b.main_channels,
|
|
330
|
+
templates_up=self.b.templates_up,
|
|
331
|
+
obj_normsq=self.b.templates_up[:, 0].square().sum(dim=(1, 2)),
|
|
332
|
+
normsq_up=self.b.templates_up.square().sum(dim=(2, 3)),
|
|
333
|
+
obj_n_templates=self.b.templates_up.shape[0],
|
|
334
|
+
pconv=self.b.pconv,
|
|
335
|
+
refrac_ix=self.b.refrac_ix,
|
|
336
|
+
conv_lags=self.b.conv_lags,
|
|
337
|
+
inv_lambda=torch.asarray(inv_lambda, device=self.b.pconv.device),
|
|
338
|
+
scale_min=torch.asarray(scale_min, device=self.b.pconv.device),
|
|
339
|
+
scale_max=torch.asarray(scale_max, device=self.b.pconv.device),
|
|
340
|
+
scaling=scaling,
|
|
341
|
+
needs_fine_pass=self.b.templates_up.shape[1] > 1 or scale_max > 3.0,
|
|
342
|
+
upsampling=self.b.templates_up.shape[1] > 1,
|
|
343
|
+
up_factor=self.b.templates_up.shape[1],
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@databag
|
|
348
|
+
class DebugChunkTemplateData(ChunkTemplateData):
|
|
349
|
+
spike_length_samples: int
|
|
350
|
+
unit_ids: Tensor
|
|
351
|
+
main_channels: Tensor
|
|
352
|
+
obj_normsq: Tensor
|
|
353
|
+
normsq_up: Tensor
|
|
354
|
+
obj_n_templates: int
|
|
355
|
+
templates_up: Tensor
|
|
356
|
+
pconv: Tensor
|
|
357
|
+
conv_lags: Tensor
|
|
358
|
+
refrac_ix: Tensor
|
|
359
|
+
inv_lambda: Tensor
|
|
360
|
+
scale_min: Tensor
|
|
361
|
+
scale_max: Tensor
|
|
362
|
+
scaling: bool
|
|
363
|
+
upsampling: bool
|
|
364
|
+
up_factor: int
|
|
365
|
+
needs_fine_pass: bool
|
|
366
|
+
coarse_objective: bool = False
|
|
367
|
+
|
|
368
|
+
def convolve(self, traces: Tensor, padding: int = 0, out: Tensor | None = None):
|
|
369
|
+
assert traces.shape[0] == self.templates_up.shape[3]
|
|
370
|
+
T = traces.shape[1]
|
|
371
|
+
t = self.templates_up.shape[2]
|
|
372
|
+
|
|
373
|
+
out_len = T - (t - 1) + 2 * padding
|
|
374
|
+
if out is None:
|
|
375
|
+
out = traces.new_zeros((self.templates_up.shape[0], out_len))
|
|
376
|
+
else:
|
|
377
|
+
assert out.shape == (self.templates_up.shape[0], out_len)
|
|
378
|
+
|
|
379
|
+
traces = traces.double()
|
|
380
|
+
for i, tplt in enumerate(self.templates_up[:, 0]):
|
|
381
|
+
res_i = F.conv1d(traces[None], tplt.T[None].double(), padding=padding)
|
|
382
|
+
assert res_i.shape == (1, 1, out_len)
|
|
383
|
+
out[i] = res_i[0, 0].to(dtype=out.dtype)
|
|
384
|
+
|
|
385
|
+
return out
|
|
386
|
+
|
|
387
|
+
def subtract(self, traces: Tensor, peaks: "MatchingPeaks", sign: int = -1):
|
|
388
|
+
if not peaks.n_spikes:
|
|
389
|
+
return
|
|
390
|
+
|
|
391
|
+
assert peaks.template_inds is not None
|
|
392
|
+
assert peaks.up_inds is not None
|
|
393
|
+
wfs = self.templates_up[peaks.template_inds, peaks.up_inds]
|
|
394
|
+
if peaks.scalings is not None:
|
|
395
|
+
wfs = wfs * peaks.scalings[:, None, None]
|
|
396
|
+
|
|
397
|
+
time_ix = torch.arange(wfs.shape[1], device=wfs.device)
|
|
398
|
+
times = peaks.times
|
|
399
|
+
assert times is not None
|
|
400
|
+
for t, wf in zip(times, wfs):
|
|
401
|
+
if sign == -1:
|
|
402
|
+
traces[t + time_ix, :-1] -= wf
|
|
403
|
+
elif sign == 1:
|
|
404
|
+
traces[t + time_ix, :-1] += wf
|
|
405
|
+
else:
|
|
406
|
+
assert False
|
|
407
|
+
|
|
408
|
+
def subtract_conv(
|
|
409
|
+
self, conv: Tensor, peaks: "MatchingPeaks", padding=0, batch_size=256, sign=-1
|
|
410
|
+
):
|
|
411
|
+
subtract_precomputed_pconv(
|
|
412
|
+
conv=conv,
|
|
413
|
+
pconv=self.pconv,
|
|
414
|
+
peaks=peaks,
|
|
415
|
+
padding=padding,
|
|
416
|
+
conv_lags=self.conv_lags,
|
|
417
|
+
sign=sign,
|
|
418
|
+
batch_size=batch_size,
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
def get_clean_waveforms(
|
|
422
|
+
self,
|
|
423
|
+
peaks: MatchingPeaks,
|
|
424
|
+
channels: Tensor,
|
|
425
|
+
channel_index: Tensor,
|
|
426
|
+
add_into: Tensor | None = None,
|
|
427
|
+
):
|
|
428
|
+
if not peaks.n_spikes:
|
|
429
|
+
return add_into
|
|
430
|
+
assert peaks.template_inds is not None
|
|
431
|
+
assert peaks.up_inds is not None
|
|
432
|
+
wfs = self.templates_up[peaks.template_inds, peaks.up_inds]
|
|
433
|
+
wfs = F.pad(wfs, (0, 1))
|
|
434
|
+
chan_ix = channel_index[channels][:, None, :]
|
|
435
|
+
wfs = wfs.take_along_dim(dim=2, indices=chan_ix)
|
|
436
|
+
if peaks.scalings is not None:
|
|
437
|
+
wfs = wfs * peaks.scalings[:, None, None]
|
|
438
|
+
if add_into is None:
|
|
439
|
+
return wfs
|
|
440
|
+
else:
|
|
441
|
+
return add_into.add_(wfs)
|
|
442
|
+
|
|
443
|
+
def _enforce_refractory(
|
|
444
|
+
self, mask: Tensor, peaks: MatchingPeaks, offset: int = 0, value=-torch.inf
|
|
445
|
+
):
|
|
446
|
+
if not peaks.n_spikes:
|
|
447
|
+
return
|
|
448
|
+
assert peaks.times is not None
|
|
449
|
+
assert peaks.obj_template_inds is not None
|
|
450
|
+
time_ix = peaks.times[:, None] + (self.refrac_ix[None, :] + offset)
|
|
451
|
+
row_ix = peaks.obj_template_inds[:, None]
|
|
452
|
+
mask[row_ix, time_ix] = value
|
|
453
|
+
|
|
454
|
+
def fine_match(
|
|
455
|
+
self, *, peaks: MatchingPeaks, residual: Tensor, conv: Tensor, padding: int = 0
|
|
456
|
+
) -> MatchingPeaks:
|
|
457
|
+
nt = self.templates_up.shape[2]
|
|
458
|
+
if not peaks.n_spikes:
|
|
459
|
+
return peaks
|
|
460
|
+
|
|
461
|
+
assert peaks.times is not None
|
|
462
|
+
times = peaks.times.clone()
|
|
463
|
+
template_inds = peaks.template_inds
|
|
464
|
+
assert times is not None
|
|
465
|
+
assert template_inds is not None
|
|
466
|
+
if self.scaling:
|
|
467
|
+
scalings = conv.new_ones(times.shape)
|
|
468
|
+
else:
|
|
469
|
+
scalings = None
|
|
470
|
+
up_inds = torch.zeros_like(template_inds)
|
|
471
|
+
scores = conv.new_zeros(times.shape)
|
|
472
|
+
|
|
473
|
+
for n, (t, l) in enumerate(zip(times, template_inds)):
|
|
474
|
+
bank = self.templates_up[l]
|
|
475
|
+
resid_chunk = residual[t : t + nt + 1]
|
|
476
|
+
T = resid_chunk.shape[0]
|
|
477
|
+
snips = [resid_chunk[t0 : t0 + nt] for t0 in range(T - nt + 1)]
|
|
478
|
+
snips = torch.stack(snips, dim=0)
|
|
479
|
+
assert snips.shape[1] == nt
|
|
480
|
+
dots = torch.einsum("utc,stc->us", bank, snips)
|
|
481
|
+
if self.scaling:
|
|
482
|
+
b = dots + self.inv_lambda
|
|
483
|
+
a = self.normsq_up[l, :, None] + self.inv_lambda
|
|
484
|
+
sc = (b / a).clamp(min=self.scale_min, max=self.scale_max)
|
|
485
|
+
objs = -a * sc * sc + 2.0 * b * sc - self.inv_lambda
|
|
486
|
+
else:
|
|
487
|
+
sc = None
|
|
488
|
+
objs = 2.0 * dots - self.normsq_up[l, :, None]
|
|
489
|
+
objs[0].add_(1e-5)
|
|
490
|
+
best_val, best_flat = objs.view(-1).max(dim=0)
|
|
491
|
+
best_u = best_flat // dots.shape[1]
|
|
492
|
+
best_s = best_flat % dots.shape[1]
|
|
493
|
+
assert objs[best_u, best_s] == best_val
|
|
494
|
+
|
|
495
|
+
if self.scaling:
|
|
496
|
+
assert scalings is not None
|
|
497
|
+
assert sc is not None
|
|
498
|
+
scalings[n] = sc[best_u, best_s]
|
|
499
|
+
up_inds[n] = best_u
|
|
500
|
+
times[n] += best_s
|
|
501
|
+
scores[n] = best_val
|
|
502
|
+
|
|
503
|
+
up_half = self.up_factor // 2
|
|
504
|
+
time_shifts = (up_inds > up_half).long().neg_()
|
|
505
|
+
|
|
506
|
+
return MatchingPeaks(
|
|
507
|
+
times=times,
|
|
508
|
+
obj_template_inds=peaks.obj_template_inds,
|
|
509
|
+
template_inds=template_inds,
|
|
510
|
+
scalings=scalings,
|
|
511
|
+
up_inds=up_inds,
|
|
512
|
+
scores=scores,
|
|
513
|
+
time_shifts=time_shifts,
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
def reconstruct_up_templates(self):
|
|
517
|
+
return self.templates_up
|