DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
from dataclasses import dataclass, replace
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from sys import getrefcount
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import torch
|
|
8
|
+
from spikeinterface.core import BaseRecording
|
|
9
|
+
from tqdm.auto import trange
|
|
10
|
+
from sklearn.decomposition import TruncatedSVD, PCA
|
|
11
|
+
|
|
12
|
+
from ..localize.localize_util import localize_waveforms
|
|
13
|
+
from ..util import data_util, drift_util, job_util
|
|
14
|
+
from ..util.data_util import DARTsortSorting
|
|
15
|
+
from ..util.internal_config import (
|
|
16
|
+
ComputationConfig,
|
|
17
|
+
TemplateConfig,
|
|
18
|
+
WaveformConfig,
|
|
19
|
+
default_waveform_cfg,
|
|
20
|
+
)
|
|
21
|
+
from ..util.logging_util import get_logger
|
|
22
|
+
from ..util.spiketorch import fast_nanmedian, nanmean
|
|
23
|
+
from .get_templates import fit_tsvd, get_templates
|
|
24
|
+
from .superres_util import superres_sorting
|
|
25
|
+
from .template_util import weighted_average
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
logger = get_logger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class TemplateData:
|
|
33
|
+
# (n_templates, spike_length_samples, n_registered_channels or n_channels)
|
|
34
|
+
templates: np.ndarray
|
|
35
|
+
# (n_templates,) maps template index to unit index (multiple templates can share a unit index)
|
|
36
|
+
unit_ids: np.ndarray
|
|
37
|
+
# (n_templates,) spike count for each template
|
|
38
|
+
spike_counts: np.ndarray
|
|
39
|
+
# (n_templates, n_registered_channels or n_channels) spike count for each channel
|
|
40
|
+
spike_counts_by_channel: np.ndarray | None = None
|
|
41
|
+
# (n_templates, spike_length_samples, n_registered_channels or n_channels)
|
|
42
|
+
raw_std_dev: np.ndarray | None = None
|
|
43
|
+
|
|
44
|
+
registered_geom: np.ndarray | None = None
|
|
45
|
+
trough_offset_samples: int = 42
|
|
46
|
+
|
|
47
|
+
# stores (n_templates, *) arrays of template properties
|
|
48
|
+
# possibilities:
|
|
49
|
+
# - motion_estimate_bin_centers
|
|
50
|
+
# For motion_estimate binned superres sortings, these are the drift
|
|
51
|
+
# bin centers for each template.
|
|
52
|
+
properties: dict[str, np.ndarray] | None = None
|
|
53
|
+
tsvd: TruncatedSVD | PCA | None = None
|
|
54
|
+
|
|
55
|
+
def __post_init__(self):
|
|
56
|
+
assert self.trough_offset_samples < self.spike_length_samples
|
|
57
|
+
|
|
58
|
+
ntemp = len(self.templates)
|
|
59
|
+
assert self.unit_ids.shape == (ntemp,)
|
|
60
|
+
assert self.spike_counts.shape == (ntemp,)
|
|
61
|
+
if self.spike_counts_by_channel is not None:
|
|
62
|
+
assert self.spike_counts_by_channel.ndim == 2
|
|
63
|
+
assert self.spike_counts_by_channel.shape[0] == ntemp
|
|
64
|
+
if self.raw_std_dev is not None:
|
|
65
|
+
assert self.raw_std_dev.shape == self.templates.shape
|
|
66
|
+
|
|
67
|
+
nc = self.templates.shape[2]
|
|
68
|
+
if self.spike_counts_by_channel is not None:
|
|
69
|
+
assert self.spike_counts_by_channel.shape[1] == nc
|
|
70
|
+
if self.registered_geom is not None:
|
|
71
|
+
assert self.registered_geom.ndim == 2
|
|
72
|
+
assert self.registered_geom.shape[0] == nc
|
|
73
|
+
if self.properties:
|
|
74
|
+
for v in self.properties.values():
|
|
75
|
+
assert v.shape[0] == ntemp
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def spike_length_samples(self):
|
|
79
|
+
return self.templates.shape[1]
|
|
80
|
+
|
|
81
|
+
def snrs_by_channel(self):
|
|
82
|
+
amp_vecs = np.nan_to_num(np.ptp(self.templates, axis=1), nan=-np.inf)
|
|
83
|
+
if self.spike_counts_by_channel is not None:
|
|
84
|
+
amp_vecs *= np.sqrt(self.spike_counts_by_channel)
|
|
85
|
+
return amp_vecs
|
|
86
|
+
|
|
87
|
+
def main_channels(self):
|
|
88
|
+
return self.snrs_by_channel().argmax(axis=1)
|
|
89
|
+
|
|
90
|
+
def template_locations(self, mode="channel", radius=100.0):
|
|
91
|
+
assert mode in ("localization", "channel")
|
|
92
|
+
|
|
93
|
+
if mode == "channel":
|
|
94
|
+
assert self.registered_geom is not None
|
|
95
|
+
return self.registered_geom[self.main_channels()]
|
|
96
|
+
|
|
97
|
+
assert mode == "localization"
|
|
98
|
+
rdepths = localize_waveforms(
|
|
99
|
+
waveforms=self.templates,
|
|
100
|
+
geom=self.registered_geom,
|
|
101
|
+
main_channels=self.main_channels(),
|
|
102
|
+
radius=radius,
|
|
103
|
+
)
|
|
104
|
+
rdepths = np.c_[rdepths["x"], rdepths["z_abs"]]
|
|
105
|
+
return rdepths
|
|
106
|
+
|
|
107
|
+
def registered_depths_um(self, mode="channel", radius=100.0):
|
|
108
|
+
return self.template_locations(mode=mode, radius=radius)[:, 1]
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
def from_npz(cls, npz_path):
|
|
112
|
+
with np.load(npz_path, allow_pickle=True) as data:
|
|
113
|
+
data = dict(**data)
|
|
114
|
+
if "spike_length_samples" in data:
|
|
115
|
+
del data["spike_length_samples"] # todo: remove
|
|
116
|
+
if "parent_sorting_hdf5_path" in data:
|
|
117
|
+
del data["parent_sorting_hdf5_path"] # todo: remove
|
|
118
|
+
|
|
119
|
+
properties = {}
|
|
120
|
+
for k, v in data.items():
|
|
121
|
+
if k.startswith("__prop_"):
|
|
122
|
+
properties[k.removeprefix("__prop_")] = v
|
|
123
|
+
for k in properties:
|
|
124
|
+
del data[f"__prop_{k}"]
|
|
125
|
+
if not properties:
|
|
126
|
+
properties = None
|
|
127
|
+
|
|
128
|
+
return cls(**data, properties=properties)
|
|
129
|
+
|
|
130
|
+
def to_npz(self, npz_path):
|
|
131
|
+
to_save = dict(
|
|
132
|
+
templates=self.templates,
|
|
133
|
+
unit_ids=self.unit_ids,
|
|
134
|
+
spike_counts=self.spike_counts,
|
|
135
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
136
|
+
)
|
|
137
|
+
if self.registered_geom is not None:
|
|
138
|
+
to_save["registered_geom"] = self.registered_geom
|
|
139
|
+
if self.spike_counts_by_channel is not None:
|
|
140
|
+
to_save["spike_counts_by_channel"] = self.spike_counts_by_channel
|
|
141
|
+
if self.raw_std_dev is not None:
|
|
142
|
+
to_save["raw_std_dev"] = self.raw_std_dev
|
|
143
|
+
if not npz_path.parent.exists():
|
|
144
|
+
npz_path.parent.mkdir()
|
|
145
|
+
if self.properties is not None:
|
|
146
|
+
for k, p in self.properties.items():
|
|
147
|
+
to_save[f"__prop_{k}"] = p
|
|
148
|
+
np.savez(npz_path, **to_save) # type: ignore
|
|
149
|
+
|
|
150
|
+
def __getitem__(self, subset):
|
|
151
|
+
if not np.array_equal(self.unit_ids, np.arange(len(self.unit_ids))):
|
|
152
|
+
subset_ixs = np.searchsorted(self.unit_ids, subset, side="right") - 1
|
|
153
|
+
matched = self.unit_ids[subset_ixs] == subset
|
|
154
|
+
assert matched.all()
|
|
155
|
+
subset = subset_ixs
|
|
156
|
+
if self.properties is not None:
|
|
157
|
+
properties = {k: p[subset] for k, p in self.properties.items()}
|
|
158
|
+
else:
|
|
159
|
+
properties = None
|
|
160
|
+
return self.__class__(
|
|
161
|
+
templates=self.templates[subset],
|
|
162
|
+
unit_ids=self.unit_ids[subset],
|
|
163
|
+
spike_counts=self.spike_counts[subset],
|
|
164
|
+
spike_counts_by_channel=(
|
|
165
|
+
self.spike_counts_by_channel[subset]
|
|
166
|
+
if self.spike_counts_by_channel is not None
|
|
167
|
+
else None
|
|
168
|
+
),
|
|
169
|
+
raw_std_dev=(
|
|
170
|
+
self.raw_std_dev[subset] if self.raw_std_dev is not None else None
|
|
171
|
+
),
|
|
172
|
+
registered_geom=self.registered_geom,
|
|
173
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
174
|
+
properties=properties,
|
|
175
|
+
tsvd=self.tsvd,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
def coarsen(self):
|
|
179
|
+
"""Weighted average all templates that share a unit id."""
|
|
180
|
+
# update templates
|
|
181
|
+
unit_ids_unique, flat_ids = np.unique(self.unit_ids, return_inverse=True)
|
|
182
|
+
templates = weighted_average(flat_ids, self.templates, self.spike_counts)
|
|
183
|
+
|
|
184
|
+
# collect spike counts
|
|
185
|
+
spike_counts = np.zeros(len(templates))
|
|
186
|
+
np.add.at(spike_counts, flat_ids, self.spike_counts)
|
|
187
|
+
|
|
188
|
+
return replace(
|
|
189
|
+
self,
|
|
190
|
+
templates=templates,
|
|
191
|
+
unit_ids=unit_ids_unique,
|
|
192
|
+
spike_counts=spike_counts,
|
|
193
|
+
properties=None,
|
|
194
|
+
tsvd=self.tsvd,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
def unit_mask(self, unit_id):
|
|
198
|
+
return np.isin(self.unit_ids, unit_id)
|
|
199
|
+
|
|
200
|
+
def unit_templates(self, unit_id):
|
|
201
|
+
return self.templates[self.unit_mask(unit_id)]
|
|
202
|
+
|
|
203
|
+
@classmethod
|
|
204
|
+
def from_config(
|
|
205
|
+
cls,
|
|
206
|
+
recording: BaseRecording,
|
|
207
|
+
sorting: DARTsortSorting,
|
|
208
|
+
template_cfg: TemplateConfig,
|
|
209
|
+
waveform_cfg: WaveformConfig = default_waveform_cfg,
|
|
210
|
+
save_folder: Path | None = None,
|
|
211
|
+
overwrite=False,
|
|
212
|
+
motion_est=None,
|
|
213
|
+
save_npz_name: str | None = "template_data.npz",
|
|
214
|
+
units_per_job=8,
|
|
215
|
+
tsvd=None,
|
|
216
|
+
computation_cfg: ComputationConfig | None = None,
|
|
217
|
+
):
|
|
218
|
+
self = _template_data_from_config(
|
|
219
|
+
cls,
|
|
220
|
+
recording=recording,
|
|
221
|
+
sorting=sorting,
|
|
222
|
+
template_cfg=template_cfg,
|
|
223
|
+
waveform_cfg=waveform_cfg,
|
|
224
|
+
save_folder=save_folder,
|
|
225
|
+
overwrite=overwrite,
|
|
226
|
+
motion_est=motion_est,
|
|
227
|
+
save_npz_name=save_npz_name,
|
|
228
|
+
units_per_job=units_per_job,
|
|
229
|
+
tsvd=tsvd,
|
|
230
|
+
computation_cfg=computation_cfg,
|
|
231
|
+
)
|
|
232
|
+
gc.collect()
|
|
233
|
+
torch.cuda.empty_cache()
|
|
234
|
+
return self
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _template_data_from_config(
|
|
238
|
+
cls,
|
|
239
|
+
recording: BaseRecording,
|
|
240
|
+
sorting: DARTsortSorting,
|
|
241
|
+
template_cfg: TemplateConfig,
|
|
242
|
+
waveform_cfg: WaveformConfig = default_waveform_cfg,
|
|
243
|
+
save_folder: str | Path | None = None,
|
|
244
|
+
overwrite: bool = False,
|
|
245
|
+
motion_est=None,
|
|
246
|
+
save_npz_name: str | None = "template_data.npz",
|
|
247
|
+
units_per_job=8,
|
|
248
|
+
tsvd=None,
|
|
249
|
+
computation_cfg: ComputationConfig | None = None,
|
|
250
|
+
show_progress: bool = True,
|
|
251
|
+
) -> TemplateData:
|
|
252
|
+
if computation_cfg is None:
|
|
253
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
254
|
+
|
|
255
|
+
npz_path = None
|
|
256
|
+
if save_folder is not None:
|
|
257
|
+
save_folder = Path(save_folder)
|
|
258
|
+
if not save_folder.exists():
|
|
259
|
+
save_folder.mkdir()
|
|
260
|
+
assert save_npz_name is not None
|
|
261
|
+
npz_path = save_folder / save_npz_name
|
|
262
|
+
if npz_path.exists() and not overwrite:
|
|
263
|
+
return cls.from_npz(npz_path)
|
|
264
|
+
|
|
265
|
+
if template_cfg.actual_algorithm() == "by_chunk":
|
|
266
|
+
template_data = get_templates_by_chunk(
|
|
267
|
+
sorting=sorting,
|
|
268
|
+
recording=recording,
|
|
269
|
+
tsvd=tsvd,
|
|
270
|
+
motion_est=motion_est,
|
|
271
|
+
waveform_cfg=waveform_cfg,
|
|
272
|
+
computation_cfg=computation_cfg,
|
|
273
|
+
template_cfg=template_cfg,
|
|
274
|
+
show_progress=show_progress,
|
|
275
|
+
)
|
|
276
|
+
return template_data
|
|
277
|
+
|
|
278
|
+
if sorting is None:
|
|
279
|
+
raise ValueError(
|
|
280
|
+
"TemplateData.from_config needs a sorting when its .npz file "
|
|
281
|
+
"does not exist."
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
fs = recording.sampling_frequency
|
|
285
|
+
trough_offset_samples = waveform_cfg.trough_offset_samples(fs)
|
|
286
|
+
spike_length_samples = waveform_cfg.spike_length_samples(fs)
|
|
287
|
+
|
|
288
|
+
if template_cfg.denoising_method in (None, "none"):
|
|
289
|
+
low_rank_denoising = False
|
|
290
|
+
else:
|
|
291
|
+
assert template_cfg.denoising_method == "exp_weighted"
|
|
292
|
+
low_rank_denoising = True
|
|
293
|
+
|
|
294
|
+
# load motion features if necessary
|
|
295
|
+
geom = recording.get_channel_locations()
|
|
296
|
+
if template_cfg.registered_templates:
|
|
297
|
+
motion_kw = dict(
|
|
298
|
+
motion_est=motion_est,
|
|
299
|
+
geom=geom,
|
|
300
|
+
localizations_dataset_name=template_cfg.localizations_dataset_name,
|
|
301
|
+
)
|
|
302
|
+
else:
|
|
303
|
+
motion_kw = dict(geom=geom)
|
|
304
|
+
|
|
305
|
+
# handle superresolved templates
|
|
306
|
+
if template_cfg.superres_templates:
|
|
307
|
+
superres_data = superres_sorting(
|
|
308
|
+
sorting=sorting,
|
|
309
|
+
geom=geom,
|
|
310
|
+
motion_est=motion_est,
|
|
311
|
+
strategy=template_cfg.superres_strategy,
|
|
312
|
+
superres_bin_size_um=template_cfg.superres_bin_size_um,
|
|
313
|
+
min_spikes_per_bin=template_cfg.superres_bin_min_spikes,
|
|
314
|
+
)
|
|
315
|
+
group_ids = superres_data["group_ids"]
|
|
316
|
+
assert isinstance(superres_data["sorting"], DARTsortSorting)
|
|
317
|
+
sorting = superres_data["sorting"]
|
|
318
|
+
properties = superres_data["properties"]
|
|
319
|
+
else:
|
|
320
|
+
group_ids = None
|
|
321
|
+
properties = {}
|
|
322
|
+
|
|
323
|
+
# main!
|
|
324
|
+
results = get_templates(
|
|
325
|
+
recording=recording,
|
|
326
|
+
sorting=sorting,
|
|
327
|
+
trough_offset_samples=trough_offset_samples,
|
|
328
|
+
spike_length_samples=spike_length_samples,
|
|
329
|
+
spikes_per_unit=template_cfg.spikes_per_unit,
|
|
330
|
+
denoising_rank=template_cfg.denoising_rank,
|
|
331
|
+
recompute_tsvd=template_cfg.recompute_tsvd,
|
|
332
|
+
denoising_fit_radius=template_cfg.denoising_fit_radius,
|
|
333
|
+
denoising_snr_threshold=template_cfg.exp_weight_snr_threshold,
|
|
334
|
+
units_per_job=units_per_job,
|
|
335
|
+
with_raw_std_dev=template_cfg.with_raw_std_dev,
|
|
336
|
+
reducer=nanmean if template_cfg.reduction == "mean" else fast_nanmedian,
|
|
337
|
+
low_rank_denoising=low_rank_denoising,
|
|
338
|
+
denoising_tsvd=tsvd,
|
|
339
|
+
device=computation_cfg.actual_device(),
|
|
340
|
+
n_jobs=computation_cfg.actual_n_jobs(),
|
|
341
|
+
show_progress=show_progress,
|
|
342
|
+
**motion_kw, # type: ignore
|
|
343
|
+
)
|
|
344
|
+
if template_cfg.superres_templates:
|
|
345
|
+
assert group_ids is not None
|
|
346
|
+
unit_ids = group_ids[results["unit_ids"]] # type: ignore
|
|
347
|
+
else:
|
|
348
|
+
unit_ids = results["unit_ids"]
|
|
349
|
+
rgeom = results["registered_geom"]
|
|
350
|
+
if rgeom is None:
|
|
351
|
+
rgeom = geom
|
|
352
|
+
if tsvd is None:
|
|
353
|
+
tsvd = results["denoising_tsvd"]
|
|
354
|
+
obj = cls(
|
|
355
|
+
results["templates"],
|
|
356
|
+
unit_ids=unit_ids,
|
|
357
|
+
spike_counts=results["spike_counts"],
|
|
358
|
+
spike_counts_by_channel=results["spike_counts_by_channel"],
|
|
359
|
+
raw_std_dev=results["raw_std_devs"],
|
|
360
|
+
registered_geom=rgeom,
|
|
361
|
+
trough_offset_samples=trough_offset_samples,
|
|
362
|
+
properties=properties,
|
|
363
|
+
tsvd=tsvd,
|
|
364
|
+
)
|
|
365
|
+
if save_folder is not None:
|
|
366
|
+
obj.to_npz(npz_path)
|
|
367
|
+
|
|
368
|
+
return obj
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def get_templates_by_chunk(
|
|
372
|
+
*,
|
|
373
|
+
sorting: DARTsortSorting,
|
|
374
|
+
recording: BaseRecording,
|
|
375
|
+
tsvd,
|
|
376
|
+
motion_est,
|
|
377
|
+
waveform_cfg: WaveformConfig,
|
|
378
|
+
computation_cfg: ComputationConfig,
|
|
379
|
+
template_cfg: TemplateConfig,
|
|
380
|
+
show_progress: bool,
|
|
381
|
+
block_size=384,
|
|
382
|
+
hard_block_size=512,
|
|
383
|
+
random_seed=0,
|
|
384
|
+
dtype=np.float32,
|
|
385
|
+
) -> TemplateData:
|
|
386
|
+
if template_cfg.denoising_method == "none":
|
|
387
|
+
block_size = hard_block_size = template_cfg.raw_templates_at_once
|
|
388
|
+
else:
|
|
389
|
+
block_size = template_cfg.templates_at_once
|
|
390
|
+
hard_block_size = template_cfg.max_templates_at_once
|
|
391
|
+
|
|
392
|
+
assert sorting.labels is not None
|
|
393
|
+
unit_ids = np.unique(sorting.labels)
|
|
394
|
+
unit_ids = unit_ids[unit_ids >= 0]
|
|
395
|
+
n_units = unit_ids.shape[0]
|
|
396
|
+
n_blocks = max(1, n_units // block_size)
|
|
397
|
+
n_blocks += n_units / n_blocks > hard_block_size
|
|
398
|
+
assert n_units / n_blocks <= hard_block_size
|
|
399
|
+
units_per_block = int(np.ceil(n_units / n_blocks).astype(int).item())
|
|
400
|
+
|
|
401
|
+
need_tsvd = template_cfg.denoising_method != "none"
|
|
402
|
+
if need_tsvd and tsvd is None:
|
|
403
|
+
logger.dartsortdebug("Fit or load TSVD...")
|
|
404
|
+
tsvd = fit_tsvd(
|
|
405
|
+
recording,
|
|
406
|
+
sorting,
|
|
407
|
+
dtype=dtype,
|
|
408
|
+
denoising_rank=template_cfg.denoising_rank,
|
|
409
|
+
denoising_fit_radius=template_cfg.denoising_fit_radius,
|
|
410
|
+
denoising_spikes_fit=template_cfg.denoising_fit_sampling_cfg.n_waveforms_fit,
|
|
411
|
+
recompute_tsvd=template_cfg.recompute_tsvd,
|
|
412
|
+
trough_offset_samples=waveform_cfg.trough_offset_samples(
|
|
413
|
+
recording.sampling_frequency
|
|
414
|
+
),
|
|
415
|
+
spike_length_samples=waveform_cfg.spike_length_samples(
|
|
416
|
+
recording.sampling_frequency
|
|
417
|
+
),
|
|
418
|
+
random_seed=random_seed,
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
labels_tmp = np.full_like(sorting.labels, -1)
|
|
422
|
+
template_datas = []
|
|
423
|
+
if n_blocks > 1:
|
|
424
|
+
block_iter = trange(
|
|
425
|
+
0, n_units, units_per_block, desc=f"Template blocks [{n_units}:{n_blocks}]"
|
|
426
|
+
)
|
|
427
|
+
else:
|
|
428
|
+
block_iter = range(0, n_units, units_per_block)
|
|
429
|
+
for block_start in block_iter:
|
|
430
|
+
block_end = min(block_start + units_per_block, n_units)
|
|
431
|
+
if n_blocks == 1:
|
|
432
|
+
assert block_end - block_start == n_units
|
|
433
|
+
|
|
434
|
+
ids_in_block = unit_ids[block_start:block_end]
|
|
435
|
+
spikes_in_block = np.flatnonzero(np.isin(sorting.labels, ids_in_block))
|
|
436
|
+
labels_tmp[:] = -1
|
|
437
|
+
labels_tmp[spikes_in_block] = sorting.labels[spikes_in_block]
|
|
438
|
+
|
|
439
|
+
block_sorting = sorting.ephemeral_replace(labels=labels_tmp)
|
|
440
|
+
block_templates = _get_templates_by_chunk_block(
|
|
441
|
+
sorting=block_sorting,
|
|
442
|
+
recording=recording,
|
|
443
|
+
tsvd=tsvd,
|
|
444
|
+
motion_est=motion_est,
|
|
445
|
+
waveform_cfg=waveform_cfg,
|
|
446
|
+
computation_cfg=computation_cfg,
|
|
447
|
+
template_cfg=template_cfg,
|
|
448
|
+
show_progress=show_progress,
|
|
449
|
+
)
|
|
450
|
+
template_datas.append(block_templates)
|
|
451
|
+
assert np.array_equal(block_templates.unit_ids, ids_in_block)
|
|
452
|
+
|
|
453
|
+
template_data = stack_template_datas(template_datas)
|
|
454
|
+
assert np.array_equal(template_data.unit_ids, unit_ids)
|
|
455
|
+
assert template_data.templates.shape[0] == unit_ids.shape[0]
|
|
456
|
+
return template_data
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _get_templates_by_chunk_block(
|
|
460
|
+
sorting: DARTsortSorting,
|
|
461
|
+
recording,
|
|
462
|
+
tsvd,
|
|
463
|
+
motion_est,
|
|
464
|
+
waveform_cfg,
|
|
465
|
+
computation_cfg,
|
|
466
|
+
template_cfg,
|
|
467
|
+
show_progress: bool,
|
|
468
|
+
):
|
|
469
|
+
from ..peel.running_template import RunningTemplates
|
|
470
|
+
|
|
471
|
+
peeler = RunningTemplates.from_config(
|
|
472
|
+
sorting=sorting,
|
|
473
|
+
recording=recording,
|
|
474
|
+
tsvd=tsvd,
|
|
475
|
+
motion_est=motion_est,
|
|
476
|
+
waveform_cfg=waveform_cfg,
|
|
477
|
+
template_cfg=template_cfg,
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
template_data = peeler.compute_template_data(
|
|
481
|
+
show_progress=show_progress, computation_cfg=computation_cfg
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
assert getrefcount(peeler) == 2, (
|
|
485
|
+
f"Leaking the template peeler {getrefcount(peeler)=}."
|
|
486
|
+
)
|
|
487
|
+
del peeler
|
|
488
|
+
gc.collect()
|
|
489
|
+
torch.cuda.empty_cache()
|
|
490
|
+
|
|
491
|
+
return template_data
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def stack_template_datas(template_datas):
|
|
495
|
+
if len(template_datas) == 1:
|
|
496
|
+
return template_datas[0]
|
|
497
|
+
assert len(template_datas) > 0
|
|
498
|
+
|
|
499
|
+
if template_datas[0].spike_counts_by_channel is not None:
|
|
500
|
+
spike_counts_by_channel = np.concatenate(
|
|
501
|
+
[td.spike_counts_by_channel for td in template_datas]
|
|
502
|
+
)
|
|
503
|
+
else:
|
|
504
|
+
spike_counts_by_channel = None
|
|
505
|
+
|
|
506
|
+
if template_datas[0].raw_std_dev is not None:
|
|
507
|
+
raw_std_dev = np.concatenate([td.raw_std_dev for td in template_datas])
|
|
508
|
+
else:
|
|
509
|
+
raw_std_dev = None
|
|
510
|
+
|
|
511
|
+
if template_datas[0].properties is not None:
|
|
512
|
+
properties = {
|
|
513
|
+
k: np.concatenate([td.properties[k] for td in template_datas])
|
|
514
|
+
for k in template_datas[0].properties
|
|
515
|
+
}
|
|
516
|
+
else:
|
|
517
|
+
properties = None
|
|
518
|
+
|
|
519
|
+
return TemplateData(
|
|
520
|
+
templates=np.concatenate([td.templates for td in template_datas]),
|
|
521
|
+
unit_ids=np.concatenate([td.unit_ids for td in template_datas]),
|
|
522
|
+
spike_counts=np.concatenate([td.spike_counts for td in template_datas]),
|
|
523
|
+
spike_counts_by_channel=spike_counts_by_channel,
|
|
524
|
+
raw_std_dev=raw_std_dev,
|
|
525
|
+
registered_geom=template_datas[0].registered_geom,
|
|
526
|
+
trough_offset_samples=template_datas[0].trough_offset_samples,
|
|
527
|
+
properties=properties,
|
|
528
|
+
)
|