DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import torch
|
|
3
|
+
import torch.nn.functional as F
|
|
4
|
+
from dartsort.util import spiketorch
|
|
5
|
+
from scipy.spatial.distance import cdist
|
|
6
|
+
|
|
7
|
+
from .transform_base import BaseWaveformDenoiser
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class EnforceDecrease(BaseWaveformDenoiser):
|
|
11
|
+
"""A torch module for enforcing spatial decrease of amplitudes
|
|
12
|
+
|
|
13
|
+
Calling an instance of this class on a N,T,C batch of waveforms
|
|
14
|
+
will return a result whose peak-to-peak amplitudes decrease
|
|
15
|
+
as you move away from the detection channel.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
default_name = "enforce_decrease"
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
channel_index,
|
|
23
|
+
geom,
|
|
24
|
+
batch_size=1024,
|
|
25
|
+
name=None,
|
|
26
|
+
name_prefix="",
|
|
27
|
+
):
|
|
28
|
+
super().__init__(
|
|
29
|
+
name=name,
|
|
30
|
+
name_prefix=name_prefix,
|
|
31
|
+
geom=geom,
|
|
32
|
+
channel_index=channel_index,
|
|
33
|
+
)
|
|
34
|
+
self.batch_size = batch_size
|
|
35
|
+
self.n_channels = self.b.geom.shape[0]
|
|
36
|
+
|
|
37
|
+
def needs_precompute(self):
|
|
38
|
+
return self.n_channels > 1 and not hasattr(self, "parents_index")
|
|
39
|
+
|
|
40
|
+
def precompute(self):
|
|
41
|
+
self.register_buffer(
|
|
42
|
+
"parents_index",
|
|
43
|
+
torch.tensor(
|
|
44
|
+
make_parents_index(
|
|
45
|
+
self.b.geom.numpy(force=True),
|
|
46
|
+
self.b.channel_index.numpy(force=True),
|
|
47
|
+
)
|
|
48
|
+
),
|
|
49
|
+
persistent=False,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def forward(self, waveforms, *, channels, **unused):
|
|
53
|
+
"""
|
|
54
|
+
enfdec = EnforceDecrease(geom, channel_index)
|
|
55
|
+
...
|
|
56
|
+
dec_wfs, dec_ptps = enfdec(waveforms, maxchans)
|
|
57
|
+
"""
|
|
58
|
+
if self.n_channels == 1:
|
|
59
|
+
return waveforms
|
|
60
|
+
n = waveforms.shape[0]
|
|
61
|
+
assert (n,) == channels.shape
|
|
62
|
+
assert waveforms.shape[2] == self.b.parents_index.shape[1]
|
|
63
|
+
|
|
64
|
+
# get peak to peak amplitudes -- (N, c) shaped
|
|
65
|
+
ptps = spiketorch.ptp(waveforms)
|
|
66
|
+
ptps = ptps.to(self.b.parents_index.device)
|
|
67
|
+
|
|
68
|
+
# pad with an extra channel to support indexing tricks
|
|
69
|
+
pad_ptps = F.pad(ptps, (0, 1), value=torch.inf)
|
|
70
|
+
parent_min_ptps = torch.zeros_like(ptps)
|
|
71
|
+
# get amplitudes of all parents for all channels -- (N, c, <=c-1)
|
|
72
|
+
# this is a gather_reduce, and it would be nice if torch had one.
|
|
73
|
+
# batching the following:
|
|
74
|
+
# parent_ptps = pad_ptps[
|
|
75
|
+
# torch.arange(n)[:, None, None],
|
|
76
|
+
# self.parents_index[channels],
|
|
77
|
+
# ]
|
|
78
|
+
# parent_min_ptps = parent_ptps.min(dim=2).values
|
|
79
|
+
for bs in range(0, n, self.batch_size):
|
|
80
|
+
be = min(n, bs + self.batch_size)
|
|
81
|
+
parent_ptps = pad_ptps[
|
|
82
|
+
torch.arange(bs, be)[:, None, None],
|
|
83
|
+
self.b.parents_index[channels[bs:be]],
|
|
84
|
+
]
|
|
85
|
+
torch.amin(parent_ptps, dim=2, out=parent_min_ptps[bs:be])
|
|
86
|
+
|
|
87
|
+
# what would we need to multiply by to ensure my amp is <= all parents?
|
|
88
|
+
rescaling = parent_min_ptps.div_(ptps).clamp_(max=1.0)
|
|
89
|
+
rescaling = rescaling.to(waveforms.device)
|
|
90
|
+
decreasing_waveforms = waveforms.mul_(rescaling[:, None, :])
|
|
91
|
+
return decreasing_waveforms
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def make_parents_index(geom, channel_index):
|
|
95
|
+
"""Indexing helper for EnforceDecrease
|
|
96
|
+
|
|
97
|
+
Waveforms extracted according to channel_index enter into the enforce
|
|
98
|
+
decrease routine, and for each channel A in the waveform we want to find
|
|
99
|
+
the maximum amplitude of any channel which is A's ancestor in the tree
|
|
100
|
+
rooted at the detection channel. This function computes an indexing
|
|
101
|
+
structure to help with that.
|
|
102
|
+
|
|
103
|
+
If channel_index is (C, c)-shaped, where C is the full number of channels
|
|
104
|
+
and c is the number of channels extracted per waverform, then this function
|
|
105
|
+
returns a (C, c, <=c-1)-shaped array parents_index, where parents_index[a, b]
|
|
106
|
+
contains the (at most) c-1 parents (relatively indexed) of channel b
|
|
107
|
+
(relatively indexed) in the tree rooted at detection channel a.
|
|
108
|
+
|
|
109
|
+
For each site A, another site B is considered a parent relative to detection
|
|
110
|
+
channel C if it is inside the square whose diagonal is AC, which is tested
|
|
111
|
+
by <ACB < 45 deg and <CBA > 90 deg.
|
|
112
|
+
|
|
113
|
+
Here, relatively indexed means in the 0:c index space rather than the 0:C
|
|
114
|
+
index space.
|
|
115
|
+
"""
|
|
116
|
+
C, c = channel_index.shape
|
|
117
|
+
assert geom.ndim == 2 and geom.shape[0] == C
|
|
118
|
+
|
|
119
|
+
parents_index = np.full((C, c, c - 1), c)
|
|
120
|
+
geom_pad = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
|
|
121
|
+
for i in range(len(geom)):
|
|
122
|
+
g = geom_pad[channel_index[i]]
|
|
123
|
+
i_rel = np.flatnonzero(channel_index[i] == i)
|
|
124
|
+
assert i_rel.size == 1
|
|
125
|
+
i_rel = i_rel[0]
|
|
126
|
+
|
|
127
|
+
# detection channel = C
|
|
128
|
+
# candidate parent = B
|
|
129
|
+
# leaf = A
|
|
130
|
+
g_detect = geom[i][None]
|
|
131
|
+
g_rel = g - g_detect
|
|
132
|
+
|
|
133
|
+
# compute all angles ACB
|
|
134
|
+
cos_across_detect = 1.0 - cdist(g_rel, g_rel, metric="cosine")
|
|
135
|
+
cos_across_detect[i_rel, :] = cos_across_detect[:, i_rel] = 1
|
|
136
|
+
|
|
137
|
+
# compute all angles CBA
|
|
138
|
+
# these are segments BA (parent is on the second axis)
|
|
139
|
+
# child,parent
|
|
140
|
+
g_par_to_child = g[:, None] - g[None, :]
|
|
141
|
+
# segments BC (parent on first+only axis)
|
|
142
|
+
g_par_to_detect = g_detect - g
|
|
143
|
+
cos_across_parent = np.sum(g_par_to_detect[None] * g_par_to_child, axis=2)
|
|
144
|
+
cos_across_parent[:, i_rel] = -1
|
|
145
|
+
# don't need to normalize this one since we just care about sign of cosine
|
|
146
|
+
|
|
147
|
+
# check valid -- child on first axis, candidate parent on second
|
|
148
|
+
is_parent = cos_across_detect >= np.cos(np.pi / 4)
|
|
149
|
+
is_parent &= cos_across_parent < 0
|
|
150
|
+
for j in range(c):
|
|
151
|
+
my_parents = np.setdiff1d(np.flatnonzero(is_parent[j]), [j])
|
|
152
|
+
parents_index[i, j, : my_parents.size] = my_parents
|
|
153
|
+
|
|
154
|
+
# trim to largest number of parents
|
|
155
|
+
biggest = (parents_index < c).sum(axis=2).max()
|
|
156
|
+
parents_index = parents_index[:, :, :biggest]
|
|
157
|
+
|
|
158
|
+
return parents_index
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from dartsort.localize.localize_torch import localize_amplitude_vectors
|
|
6
|
+
from dartsort.util.spiketorch import ptp
|
|
7
|
+
|
|
8
|
+
from .transform_base import BaseWaveformFeaturizer
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Localization(BaseWaveformFeaturizer):
|
|
12
|
+
"""Order of output columns: x, y, z_abs, alpha"""
|
|
13
|
+
|
|
14
|
+
default_name = "point_source_localizations"
|
|
15
|
+
shape = (4,)
|
|
16
|
+
dtype = torch.double
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
channel_index,
|
|
21
|
+
geom,
|
|
22
|
+
radius=None,
|
|
23
|
+
n_channels_subset=None,
|
|
24
|
+
logbarrier=True,
|
|
25
|
+
amplitude_kind="peak",
|
|
26
|
+
localization_model="pointsource",
|
|
27
|
+
name=None,
|
|
28
|
+
name_prefix="",
|
|
29
|
+
batch_size=1024,
|
|
30
|
+
):
|
|
31
|
+
assert amplitude_kind in ("peak", "ptp")
|
|
32
|
+
super().__init__(
|
|
33
|
+
geom=geom,
|
|
34
|
+
channel_index=channel_index,
|
|
35
|
+
name=name,
|
|
36
|
+
name_prefix=name_prefix,
|
|
37
|
+
)
|
|
38
|
+
self.amplitude_kind = amplitude_kind
|
|
39
|
+
self.radius = radius
|
|
40
|
+
self.n_channels_subset = n_channels_subset
|
|
41
|
+
self.logbarrier = logbarrier
|
|
42
|
+
self.localization_model = localization_model
|
|
43
|
+
self.batch_size = batch_size
|
|
44
|
+
|
|
45
|
+
def transform(self, waveforms, *, channels, **unused):
|
|
46
|
+
# get amplitude vectors
|
|
47
|
+
if self.amplitude_kind == "peak":
|
|
48
|
+
ampvecs = waveforms.abs().max(dim=1).values
|
|
49
|
+
elif self.amplitude_kind == "ptp":
|
|
50
|
+
ampvecs = ptp(waveforms, dim=1)
|
|
51
|
+
else:
|
|
52
|
+
assert False
|
|
53
|
+
|
|
54
|
+
n = len(ampvecs)
|
|
55
|
+
localizations = ampvecs.new_empty((n, 4), dtype=self.dtype)
|
|
56
|
+
|
|
57
|
+
with torch.enable_grad():
|
|
58
|
+
for batch_start in range(0, n, self.batch_size):
|
|
59
|
+
batch_end = min(n, batch_start + self.batch_size)
|
|
60
|
+
sl = slice(batch_start, batch_end)
|
|
61
|
+
loc_result = localize_amplitude_vectors(
|
|
62
|
+
ampvecs[sl],
|
|
63
|
+
self.geom,
|
|
64
|
+
channels[sl],
|
|
65
|
+
channel_index=self.channel_index,
|
|
66
|
+
radius=self.radius,
|
|
67
|
+
n_channels_subset=self.n_channels_subset,
|
|
68
|
+
logbarrier=self.logbarrier,
|
|
69
|
+
dtype=cast(torch.dtype, self.dtype),
|
|
70
|
+
model=self.localization_model,
|
|
71
|
+
)
|
|
72
|
+
localizations[sl, 0] = loc_result["x"]
|
|
73
|
+
if "y" in loc_result:
|
|
74
|
+
localizations[sl, 1] = loc_result["y"]
|
|
75
|
+
else:
|
|
76
|
+
localizations[sl, 1] = 0.0
|
|
77
|
+
localizations[sl, 2] = loc_result["z_abs"]
|
|
78
|
+
if "alpha" in loc_result:
|
|
79
|
+
localizations[sl, 3] = loc_result["alpha"]
|
|
80
|
+
else:
|
|
81
|
+
localizations[sl, 3] = 0.0
|
|
82
|
+
|
|
83
|
+
return {self.name: localizations}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
PointSourceLocalization = Localization
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
from ..util.waveform_util import get_channels_in_probe, set_channels_in_probe
|
|
4
|
+
from .transform_base import BaseWaveformDenoiser
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DebugMatchingPursuitDenoiser(BaseWaveformDenoiser):
|
|
8
|
+
"""This denoiser is used for testing purposes only."""
|
|
9
|
+
|
|
10
|
+
default_name = "pursuit_basis"
|
|
11
|
+
|
|
12
|
+
def __init__(self, channel_index, geom=None, basis=None, name=None, name_prefix=""):
|
|
13
|
+
super().__init__(
|
|
14
|
+
channel_index=channel_index, name=name, name_prefix=name_prefix
|
|
15
|
+
)
|
|
16
|
+
if basis is not None:
|
|
17
|
+
assert basis.ndim == 2
|
|
18
|
+
self.register_buffer("basis", basis)
|
|
19
|
+
|
|
20
|
+
def initialize_spike_length_dependent_params(self):
|
|
21
|
+
if not hasattr(self, "basis"):
|
|
22
|
+
# this path is only hit by test_transform
|
|
23
|
+
zbuf = torch.zeros((1, self.spike_length_samples)) # type: ignore
|
|
24
|
+
self.register_buffer("basis", zbuf)
|
|
25
|
+
|
|
26
|
+
def needs_fit(self) -> bool:
|
|
27
|
+
return not hasattr(self, "basis")
|
|
28
|
+
|
|
29
|
+
def _project_in_probe(self, waveforms_in_probe):
|
|
30
|
+
dev = waveforms_in_probe.device
|
|
31
|
+
assert self.basis is not None
|
|
32
|
+
dots = waveforms_in_probe.to(device=self.b.basis.device) @ self.basis.T
|
|
33
|
+
best = dots.abs().argmax(dim=1)
|
|
34
|
+
dots = dots.take_along_dim(best[:, None], dim=1)[:, 0]
|
|
35
|
+
denoised = self.b.basis.take_along_dim(best[:, None], dim=0)
|
|
36
|
+
denoised = denoised.mul_(dots.sign()[:, None])
|
|
37
|
+
return denoised.to(device=dev)
|
|
38
|
+
|
|
39
|
+
def forward(self, waveforms, *, channels, **unused):
|
|
40
|
+
channels_in_probe, waveforms_in_probe = get_channels_in_probe(
|
|
41
|
+
waveforms, channels, self.channel_index
|
|
42
|
+
)
|
|
43
|
+
waveforms_in_probe = self._project_in_probe(waveforms_in_probe)
|
|
44
|
+
return set_channels_in_probe(waveforms_in_probe, waveforms, channels_in_probe)
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"""A class which manages pipelines of denoisers and featurizers"""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from .transform_base import BaseWaveformModule, BaseWaveformFeaturizer
|
|
6
|
+
from ..util.data_util import SpikeDataset
|
|
7
|
+
from ..util.internal_config import WaveformConfig, FeaturizationConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class WaveformPipeline(torch.nn.Module):
|
|
11
|
+
def __init__(self, transformers: list[BaseWaveformModule], kwargs_to_store=None):
|
|
12
|
+
super().__init__()
|
|
13
|
+
check_unique_feature_names(transformers)
|
|
14
|
+
self.transformers: list[BaseWaveformModule] = torch.nn.ModuleList(transformers) # type: ignore
|
|
15
|
+
self.kwargs_to_store = kwargs_to_store
|
|
16
|
+
|
|
17
|
+
def __len__(self):
|
|
18
|
+
return len(self.transformers)
|
|
19
|
+
|
|
20
|
+
def __bool__(self):
|
|
21
|
+
# if I am False, I return empty dicts always.
|
|
22
|
+
return bool(len(self.transformers))
|
|
23
|
+
|
|
24
|
+
def get_extra_state(self):
|
|
25
|
+
if self.kwargs_to_store is None:
|
|
26
|
+
return {}
|
|
27
|
+
return dict(class_names_and_kwargs=self.kwargs_to_store)
|
|
28
|
+
|
|
29
|
+
def set_extra_state(self, state):
|
|
30
|
+
# needed so that load_state_dict doesn't complain if there's
|
|
31
|
+
# extra state in there.
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
def spike_datasets(self) -> list[SpikeDataset]:
|
|
35
|
+
datasets = []
|
|
36
|
+
for transformer in self.transformers:
|
|
37
|
+
if transformer.is_featurizer:
|
|
38
|
+
assert isinstance(transformer, BaseWaveformFeaturizer)
|
|
39
|
+
datasets.extend(transformer.spike_datasets)
|
|
40
|
+
return datasets
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def from_state_dict_pt(cls, geom, channel_index, state_dict_pt):
|
|
44
|
+
state_dict = torch.load(state_dict_pt)
|
|
45
|
+
extra_state = state_dict.get("_extra_state", {})
|
|
46
|
+
class_names_and_kwargs = extra_state.get("class_names_and_kwargs")
|
|
47
|
+
if class_names_and_kwargs is None:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
"Can't load a featurization pipeline from state dict if it "
|
|
50
|
+
"wasn't initially created with from_config() or "
|
|
51
|
+
"from_class_names_and_kwargs(). Instead, you can instantiate "
|
|
52
|
+
"it the same way you originally did and use .load_state_dict() "
|
|
53
|
+
"directly. (You may want to just use torch.save() and load()!)"
|
|
54
|
+
)
|
|
55
|
+
self = cls.from_class_names_and_kwargs(
|
|
56
|
+
geom, channel_index, class_names_and_kwargs
|
|
57
|
+
)
|
|
58
|
+
self.precompute()
|
|
59
|
+
# strict=False is needed here, because some transformers don't have
|
|
60
|
+
# parameters initialized until their pre-load hooks are called, and
|
|
61
|
+
# the strict check happens before that...
|
|
62
|
+
self.load_state_dict(state_dict, strict=False)
|
|
63
|
+
return self
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def from_class_names_and_kwargs(cls, geom, channel_index, class_names_and_kwargs):
|
|
67
|
+
from .all_transformers import transformers_by_class_name
|
|
68
|
+
|
|
69
|
+
channel_index = torch.as_tensor(channel_index)
|
|
70
|
+
geom = torch.as_tensor(geom)
|
|
71
|
+
probe_kw = dict(channel_index=channel_index, geom=geom)
|
|
72
|
+
|
|
73
|
+
transformers = []
|
|
74
|
+
for name, kwargs in class_names_and_kwargs:
|
|
75
|
+
transformer_cls = transformers_by_class_name[name]
|
|
76
|
+
if kwargs.get("pretrained_path") is not None:
|
|
77
|
+
transformer = transformer_cls.load_from_pt(**probe_kw, **kwargs)
|
|
78
|
+
else:
|
|
79
|
+
assert kwargs.pop("pretrained_path", None) is None
|
|
80
|
+
transformer = transformer_cls(**probe_kw, **kwargs)
|
|
81
|
+
transformers.append(transformer)
|
|
82
|
+
|
|
83
|
+
return cls(transformers, kwargs_to_store=class_names_and_kwargs)
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def from_config(
|
|
87
|
+
cls,
|
|
88
|
+
featurization_cfg,
|
|
89
|
+
waveform_cfg,
|
|
90
|
+
recording=None,
|
|
91
|
+
geom=None,
|
|
92
|
+
channel_index=None,
|
|
93
|
+
sampling_frequency: int | float = 30_000,
|
|
94
|
+
):
|
|
95
|
+
if geom is None:
|
|
96
|
+
from dartsort.util.waveform_util import make_channel_index
|
|
97
|
+
|
|
98
|
+
assert recording is not None
|
|
99
|
+
sampling_frequency = recording.sampling_frequency
|
|
100
|
+
geom = torch.tensor(recording.get_channel_locations())
|
|
101
|
+
channel_index = make_channel_index(
|
|
102
|
+
geom, featurization_cfg.extract_radius, to_torch=True
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
assert recording is None
|
|
106
|
+
assert channel_index is not None
|
|
107
|
+
args = featurization_config_to_class_names_and_kwargs(
|
|
108
|
+
featurization_cfg, waveform_cfg, sampling_frequency=sampling_frequency
|
|
109
|
+
)
|
|
110
|
+
return cls.from_class_names_and_kwargs(geom, channel_index, args)
|
|
111
|
+
|
|
112
|
+
def needs_precompute(self):
|
|
113
|
+
return any(t.needs_precompute() for t in self.transformers)
|
|
114
|
+
|
|
115
|
+
def needs_fit(self):
|
|
116
|
+
return any(t.needs_fit() for t in self.transformers)
|
|
117
|
+
|
|
118
|
+
def forward(self, waveforms, **fixed_properties):
|
|
119
|
+
"""
|
|
120
|
+
fixed_properties usually contains max_channels, and may contain other relevant
|
|
121
|
+
unchanging aspects of spikes like weights (used in fit()) or temporal shifts
|
|
122
|
+
(used in pca slice logic).
|
|
123
|
+
"""
|
|
124
|
+
waveforms = torch.asarray(waveforms)
|
|
125
|
+
fixed_properties = {k: torch.asarray(v) for k, v in fixed_properties.items()}
|
|
126
|
+
assert waveforms.ndim == 3
|
|
127
|
+
for v in fixed_properties.values():
|
|
128
|
+
assert v.shape[0] == waveforms.shape[0]
|
|
129
|
+
|
|
130
|
+
features = {}
|
|
131
|
+
|
|
132
|
+
if not waveforms.shape[0]:
|
|
133
|
+
return waveforms, features
|
|
134
|
+
|
|
135
|
+
for transformer in self.transformers:
|
|
136
|
+
if transformer.is_featurizer and transformer.is_denoiser:
|
|
137
|
+
waveforms, new_features = transformer(waveforms, **fixed_properties)
|
|
138
|
+
features.update(new_features)
|
|
139
|
+
elif transformer.is_featurizer:
|
|
140
|
+
assert isinstance(transformer, BaseWaveformFeaturizer)
|
|
141
|
+
features.update(transformer.transform(waveforms, **fixed_properties))
|
|
142
|
+
elif transformer.is_denoiser:
|
|
143
|
+
waveforms = transformer(waveforms, **fixed_properties)
|
|
144
|
+
|
|
145
|
+
return waveforms, features
|
|
146
|
+
|
|
147
|
+
def fit(self, recording, waveforms, **fixed_properties):
|
|
148
|
+
waveforms = torch.asarray(waveforms)
|
|
149
|
+
fixed_properties = {k: torch.asarray(v) for k, v in fixed_properties.items()}
|
|
150
|
+
assert waveforms.ndim == 3
|
|
151
|
+
for v in fixed_properties.values():
|
|
152
|
+
assert v.shape[0] == waveforms.shape[0]
|
|
153
|
+
|
|
154
|
+
if not self.needs_fit():
|
|
155
|
+
return
|
|
156
|
+
|
|
157
|
+
for transformer in self.transformers:
|
|
158
|
+
if transformer.needs_fit():
|
|
159
|
+
transformer.train()
|
|
160
|
+
transformer.fit(
|
|
161
|
+
recording=recording, waveforms=waveforms, **fixed_properties
|
|
162
|
+
)
|
|
163
|
+
transformer.eval()
|
|
164
|
+
transformer.requires_grad_(False)
|
|
165
|
+
|
|
166
|
+
# if we're done already, stop before denoising
|
|
167
|
+
if not self.needs_fit():
|
|
168
|
+
break
|
|
169
|
+
|
|
170
|
+
if transformer.is_denoiser:
|
|
171
|
+
waveforms = transformer(waveforms, **fixed_properties)
|
|
172
|
+
if transformer.is_featurizer:
|
|
173
|
+
# result is tuple wfs, feats
|
|
174
|
+
waveforms = waveforms[0]
|
|
175
|
+
assert not waveforms.requires_grad
|
|
176
|
+
|
|
177
|
+
def precompute(self):
|
|
178
|
+
for transformer in self.transformers:
|
|
179
|
+
transformer.precompute()
|
|
180
|
+
|
|
181
|
+
def __iter__(self):
|
|
182
|
+
return iter(self.transformers)
|
|
183
|
+
|
|
184
|
+
def get_transformer(self, transformer_name):
|
|
185
|
+
for t in self.transformers:
|
|
186
|
+
if t.name == transformer_name:
|
|
187
|
+
return t
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def check_unique_feature_names(transformers):
|
|
192
|
+
fnames = []
|
|
193
|
+
for f in transformers:
|
|
194
|
+
if f.is_featurizer:
|
|
195
|
+
if f.is_multi:
|
|
196
|
+
fnames.extend(f.name)
|
|
197
|
+
else:
|
|
198
|
+
fnames.append(f.name)
|
|
199
|
+
if not len(fnames) == len(set(fnames)):
|
|
200
|
+
raise ValueError("Featurizer name collision in a WaveformPipeline")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def featurization_config_to_class_names_and_kwargs(
|
|
204
|
+
featurization_cfg: FeaturizationConfig,
|
|
205
|
+
waveform_cfg: WaveformConfig,
|
|
206
|
+
sampling_frequency: float = 30_000.0,
|
|
207
|
+
):
|
|
208
|
+
"""Convert this config into a list of waveform transformer classes and arguments
|
|
209
|
+
|
|
210
|
+
Used by WaveformPipeline.from_config(...) to construct WaveformPipelines
|
|
211
|
+
from FeaturizationConfig objects.
|
|
212
|
+
"""
|
|
213
|
+
fc = featurization_cfg
|
|
214
|
+
if fc.skip:
|
|
215
|
+
return []
|
|
216
|
+
|
|
217
|
+
class_names_and_kwargs = []
|
|
218
|
+
do_feats = not fc.denoise_only
|
|
219
|
+
|
|
220
|
+
tos = waveform_cfg.trough_offset_samples(sampling_frequency)
|
|
221
|
+
sls = waveform_cfg.spike_length_samples(sampling_frequency)
|
|
222
|
+
tos_kw = dict(trough_offset_samples=tos)
|
|
223
|
+
sls_kw = dict(spike_length_samples=sls)
|
|
224
|
+
|
|
225
|
+
if do_feats and fc.save_input_voltages:
|
|
226
|
+
class_names_and_kwargs.append(
|
|
227
|
+
("Voltage", {"name_prefix": fc.input_waveforms_name, **tos_kw})
|
|
228
|
+
)
|
|
229
|
+
if fc.save_input_waveforms:
|
|
230
|
+
class_names_and_kwargs.append(
|
|
231
|
+
("Waveform", {"name_prefix": fc.input_waveforms_name, **sls_kw})
|
|
232
|
+
)
|
|
233
|
+
if do_feats and fc.learn_cleaned_tpca_basis:
|
|
234
|
+
class_names_and_kwargs.append(
|
|
235
|
+
(
|
|
236
|
+
"BaseTemporalPCA",
|
|
237
|
+
{
|
|
238
|
+
"rank": fc.tpca_rank,
|
|
239
|
+
"name_prefix": fc.input_waveforms_name,
|
|
240
|
+
"centered": False,
|
|
241
|
+
"max_waveforms": fc.tpca_max_waveforms,
|
|
242
|
+
"fit_radius": fc.tpca_fit_radius,
|
|
243
|
+
},
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
# logic for picking an efficient combo of tpcas and nn denoisers
|
|
248
|
+
class_names_and_kwargs.extend(_add_tpca_and_nn(featurization_cfg, waveform_cfg))
|
|
249
|
+
|
|
250
|
+
if fc.do_enforce_decrease:
|
|
251
|
+
class_names_and_kwargs.append(("EnforceDecrease", {}))
|
|
252
|
+
if fc.save_output_waveforms:
|
|
253
|
+
class_names_and_kwargs.append(
|
|
254
|
+
("Waveform", {"name_prefix": fc.output_waveforms_name, **sls_kw})
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
if fc.save_output_tpca_projs:
|
|
258
|
+
class_names_and_kwargs.append(
|
|
259
|
+
(
|
|
260
|
+
"TemporalPCAFeaturizer",
|
|
261
|
+
{
|
|
262
|
+
"rank": fc.tpca_rank,
|
|
263
|
+
"name_prefix": fc.output_waveforms_name,
|
|
264
|
+
"centered": fc.tpca_centered,
|
|
265
|
+
"fit_radius": fc.tpca_fit_radius,
|
|
266
|
+
},
|
|
267
|
+
)
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# logic for grabbing localizations and amplitude vectors
|
|
271
|
+
class_names_and_kwargs.extend(_add_localization_and_ampvec(featurization_cfg))
|
|
272
|
+
|
|
273
|
+
return class_names_and_kwargs
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _add_tpca_and_nn(fc, wc):
|
|
277
|
+
do_feats = not fc.denoise_only
|
|
278
|
+
more = []
|
|
279
|
+
|
|
280
|
+
# now, if there's no downstream waveform feature or output tpca feature,
|
|
281
|
+
# we can combine the tpca featurization and denoising into one step
|
|
282
|
+
# it's not exactly equivalent, since the denoiser would have run on the full
|
|
283
|
+
# time length, but it's close enough
|
|
284
|
+
combine = (
|
|
285
|
+
do_feats
|
|
286
|
+
and fc.save_input_tpca_projs
|
|
287
|
+
and not (
|
|
288
|
+
fc.do_nn_denoise or fc.save_output_waveforms or fc.save_output_tpca_projs
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
if combine or (do_feats and fc.save_input_tpca_projs):
|
|
292
|
+
tslice = fc.input_tpca_waveform_cfg.relative_slice(wc)
|
|
293
|
+
more.append(
|
|
294
|
+
(
|
|
295
|
+
"TemporalPCA",
|
|
296
|
+
{
|
|
297
|
+
"rank": fc.tpca_rank,
|
|
298
|
+
"name_prefix": fc.input_waveforms_name,
|
|
299
|
+
"centered": fc.tpca_centered,
|
|
300
|
+
"temporal_slice": tslice,
|
|
301
|
+
"max_waveforms": fc.tpca_max_waveforms,
|
|
302
|
+
"fit_radius": fc.tpca_fit_radius,
|
|
303
|
+
},
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
if combine:
|
|
308
|
+
# that was it, all in one as discussed above.
|
|
309
|
+
return more
|
|
310
|
+
|
|
311
|
+
if fc.do_nn_denoise:
|
|
312
|
+
more.append(
|
|
313
|
+
(
|
|
314
|
+
fc.nn_denoiser_class_name,
|
|
315
|
+
{
|
|
316
|
+
"pretrained_path": fc.nn_denoiser_pretrained_path,
|
|
317
|
+
"n_epochs": fc.nn_denoiser_train_epochs,
|
|
318
|
+
"epoch_size": fc.nn_denoiser_epoch_size,
|
|
319
|
+
**(fc.nn_denoiser_extra_kwargs or {}),
|
|
320
|
+
},
|
|
321
|
+
)
|
|
322
|
+
)
|
|
323
|
+
if fc.do_tpca_denoise:
|
|
324
|
+
more.append(
|
|
325
|
+
(
|
|
326
|
+
"TemporalPCADenoiser",
|
|
327
|
+
{
|
|
328
|
+
"rank": fc.tpca_rank,
|
|
329
|
+
"fit_radius": fc.tpca_fit_radius,
|
|
330
|
+
"centered": fc.tpca_centered,
|
|
331
|
+
"max_waveforms": fc.tpca_max_waveforms,
|
|
332
|
+
},
|
|
333
|
+
)
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
return more
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _add_localization_and_ampvec(fc):
|
|
340
|
+
do_feats = not fc.denoise_only
|
|
341
|
+
more = []
|
|
342
|
+
|
|
343
|
+
if do_feats and fc.do_localization and fc.nn_localization:
|
|
344
|
+
more.append(
|
|
345
|
+
(
|
|
346
|
+
"AmortizedLocalization",
|
|
347
|
+
{
|
|
348
|
+
"amplitude_kind": fc.localization_amplitude_type,
|
|
349
|
+
"localization_model": fc.localization_model,
|
|
350
|
+
"radius": fc.localization_radius,
|
|
351
|
+
"softmax_noise_floor": fc.localization_noise_floor,
|
|
352
|
+
},
|
|
353
|
+
)
|
|
354
|
+
)
|
|
355
|
+
if do_feats and fc.additional_com_localization:
|
|
356
|
+
more.append(
|
|
357
|
+
(
|
|
358
|
+
"Localization",
|
|
359
|
+
{
|
|
360
|
+
"amplitude_kind": fc.localization_amplitude_type,
|
|
361
|
+
"localization_model": "com",
|
|
362
|
+
"radius": fc.localization_radius,
|
|
363
|
+
"name": "com_localizations",
|
|
364
|
+
},
|
|
365
|
+
)
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
do_ptp_amp = do_feats and fc.save_amplitudes
|
|
369
|
+
do_peak_vec = do_feats and (
|
|
370
|
+
fc.do_localization
|
|
371
|
+
and fc.localization_amplitude_type == "peak"
|
|
372
|
+
and not fc.nn_localization
|
|
373
|
+
)
|
|
374
|
+
do_ptp_vec = do_feats and fc.save_amplitudes
|
|
375
|
+
do_logptt = do_feats and fc.save_amplitudes
|
|
376
|
+
do_any_amp = do_peak_vec or do_ptp_vec or do_ptp_amp or do_logptt
|
|
377
|
+
if do_any_amp or (do_feats and fc.save_all_amplitudes):
|
|
378
|
+
more.append(
|
|
379
|
+
(
|
|
380
|
+
"AmplitudeFeatures",
|
|
381
|
+
{
|
|
382
|
+
"name_prefix": fc.output_waveforms_name,
|
|
383
|
+
"ptp_max_amplitude": do_ptp_amp or fc.save_all_amplitudes,
|
|
384
|
+
"peak_amplitude_vectors": do_peak_vec or fc.save_all_amplitudes,
|
|
385
|
+
"ptp_amplitude_vectors": do_ptp_vec or fc.save_all_amplitudes,
|
|
386
|
+
"log_peak_to_trough": do_logptt or fc.save_all_amplitudes,
|
|
387
|
+
},
|
|
388
|
+
)
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
return more
|