DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Any
|
|
3
|
+
import gc
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sklearn.decomposition import PCA, TruncatedSVD
|
|
7
|
+
from spikeinterface.core import BaseRecording
|
|
8
|
+
from scipy.spatial import KDTree
|
|
9
|
+
from scipy.sparse import coo_array
|
|
10
|
+
import torch
|
|
11
|
+
|
|
12
|
+
from . import TemplateData, realign
|
|
13
|
+
from ..util.data_util import DARTsortSorting
|
|
14
|
+
from ..util.internal_config import (
|
|
15
|
+
ComputationConfig,
|
|
16
|
+
TemplateConfig,
|
|
17
|
+
TemplateMergeConfig,
|
|
18
|
+
TemplateRealignmentConfig,
|
|
19
|
+
WaveformConfig,
|
|
20
|
+
SubtractionConfig,
|
|
21
|
+
default_template_cfg,
|
|
22
|
+
default_waveform_cfg,
|
|
23
|
+
)
|
|
24
|
+
from ..util.job_util import ensure_computation_config
|
|
25
|
+
from ..util.logging_util import get_logger
|
|
26
|
+
from ..util.py_util import resolve_path
|
|
27
|
+
from ..util.spiketorch import ptp
|
|
28
|
+
|
|
29
|
+
logger = get_logger(__name__)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def estimate_template_library(
|
|
33
|
+
recording: BaseRecording,
|
|
34
|
+
sorting: DARTsortSorting,
|
|
35
|
+
motion_est=None,
|
|
36
|
+
min_template_snr: float = 0.0,
|
|
37
|
+
min_template_ptp: float = 0.0,
|
|
38
|
+
min_template_count: int = 0,
|
|
39
|
+
max_cc_flag_rate: float = 1.0,
|
|
40
|
+
cc_flag_entropy_cutoff: float = 0.0,
|
|
41
|
+
waveform_cfg: WaveformConfig = default_waveform_cfg,
|
|
42
|
+
template_cfg: TemplateConfig = default_template_cfg,
|
|
43
|
+
realign_cfg: TemplateRealignmentConfig | None = None,
|
|
44
|
+
template_merge_cfg: TemplateMergeConfig | None = None,
|
|
45
|
+
tsvd: PCA | TruncatedSVD | None = None,
|
|
46
|
+
computation_cfg: ComputationConfig | None = None,
|
|
47
|
+
detection_cfg: Any | None = None,
|
|
48
|
+
depth_order: bool = False,
|
|
49
|
+
template_npz_path=None,
|
|
50
|
+
) -> tuple[DARTsortSorting, TemplateData]:
|
|
51
|
+
"""Postprocess spike train and estimate a TemplateData."""
|
|
52
|
+
if template_npz_path is not None:
|
|
53
|
+
template_npz_path = resolve_path(template_npz_path)
|
|
54
|
+
if template_npz_path.exists():
|
|
55
|
+
return sorting, TemplateData.from_npz(template_npz_path)
|
|
56
|
+
|
|
57
|
+
if (sorting.labels is None) or (sorting.labels < 0).all():
|
|
58
|
+
raise ValueError("No labels in sorting input to template postprocessing.")
|
|
59
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
60
|
+
|
|
61
|
+
# realign sorting and estimate template snr
|
|
62
|
+
sorting, templates0 = realign(
|
|
63
|
+
recording=recording,
|
|
64
|
+
sorting=sorting,
|
|
65
|
+
realign_cfg=realign_cfg,
|
|
66
|
+
waveform_cfg=waveform_cfg,
|
|
67
|
+
computation_cfg=computation_cfg,
|
|
68
|
+
motion_est=motion_est,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# filter out low-count/snr units
|
|
72
|
+
if templates0 is None and (min_template_count or min_template_snr):
|
|
73
|
+
templates0 = _quick_mean_templates(
|
|
74
|
+
recording=recording,
|
|
75
|
+
sorting=sorting,
|
|
76
|
+
waveform_cfg=waveform_cfg,
|
|
77
|
+
computation_cfg=computation_cfg,
|
|
78
|
+
motion_est=motion_est,
|
|
79
|
+
)
|
|
80
|
+
if min_template_count or min_template_snr or (max_cc_flag_rate < 1.0):
|
|
81
|
+
assert templates0 is not None
|
|
82
|
+
count_mask = templates0.spike_counts >= min_template_count
|
|
83
|
+
snr_mask = templates0.snrs_by_channel().max(1) >= min_template_snr
|
|
84
|
+
amp_mask = ptp(templates0.templates).max(1) >= min_template_ptp
|
|
85
|
+
mask = count_mask & snr_mask & amp_mask
|
|
86
|
+
flag_mask = cc_flag_criterion(
|
|
87
|
+
sorting,
|
|
88
|
+
detection_cfg,
|
|
89
|
+
max_cc_flag_rate,
|
|
90
|
+
cc_flag_entropy_cutoff,
|
|
91
|
+
amplitudes_dataset_name=template_cfg.amplitudes_dataset_name,
|
|
92
|
+
)
|
|
93
|
+
if flag_mask is not None:
|
|
94
|
+
mask = mask & flag_mask
|
|
95
|
+
sorting = filter_by_unit_mask(sorting, mask, mask_ids=templates0.unit_ids)
|
|
96
|
+
del templates0
|
|
97
|
+
|
|
98
|
+
_check_still_valid(sorting)
|
|
99
|
+
gc.collect()
|
|
100
|
+
torch.cuda.empty_cache()
|
|
101
|
+
|
|
102
|
+
# main task: get denoised templates from aligned spike train
|
|
103
|
+
templates = TemplateData.from_config(
|
|
104
|
+
recording=recording,
|
|
105
|
+
sorting=sorting,
|
|
106
|
+
motion_est=motion_est,
|
|
107
|
+
waveform_cfg=waveform_cfg,
|
|
108
|
+
template_cfg=template_cfg,
|
|
109
|
+
computation_cfg=computation_cfg,
|
|
110
|
+
tsvd=tsvd,
|
|
111
|
+
)
|
|
112
|
+
gc.collect()
|
|
113
|
+
torch.cuda.empty_cache()
|
|
114
|
+
|
|
115
|
+
# merge units by template distance
|
|
116
|
+
sorting, templates = _handle_merge(
|
|
117
|
+
recording=recording,
|
|
118
|
+
sorting=sorting,
|
|
119
|
+
template_data=templates,
|
|
120
|
+
motion_est=motion_est,
|
|
121
|
+
merge_cfg=template_merge_cfg,
|
|
122
|
+
computation_cfg=computation_cfg,
|
|
123
|
+
waveform_cfg=waveform_cfg,
|
|
124
|
+
template_cfg=template_cfg,
|
|
125
|
+
)
|
|
126
|
+
gc.collect()
|
|
127
|
+
torch.cuda.empty_cache()
|
|
128
|
+
|
|
129
|
+
# re-order along probe length
|
|
130
|
+
if depth_order:
|
|
131
|
+
sorting, templates = reorder_by_depth(sorting, templates)
|
|
132
|
+
|
|
133
|
+
return sorting, ensure_save(templates, template_npz_path)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def realign_and_chuck_noisy_template_units(
|
|
137
|
+
recording,
|
|
138
|
+
sorting,
|
|
139
|
+
template_data=None,
|
|
140
|
+
motion_est=None,
|
|
141
|
+
min_n_spikes=50,
|
|
142
|
+
min_template_snr=15.0,
|
|
143
|
+
waveform_cfg=default_waveform_cfg,
|
|
144
|
+
template_cfg=default_template_cfg,
|
|
145
|
+
tsvd=None,
|
|
146
|
+
computation_cfg=None,
|
|
147
|
+
template_save_folder=None,
|
|
148
|
+
template_npz_filename=None,
|
|
149
|
+
):
|
|
150
|
+
"""Get rid of noise units.
|
|
151
|
+
|
|
152
|
+
This will reindex the sorting and template data -- unit labels will
|
|
153
|
+
change, and the number of templates will change.
|
|
154
|
+
"""
|
|
155
|
+
if template_save_folder is not None:
|
|
156
|
+
if template_npz_filename is not None:
|
|
157
|
+
npz = Path(template_save_folder) / template_npz_filename
|
|
158
|
+
if npz.exists():
|
|
159
|
+
return sorting, TemplateData.from_npz(npz)
|
|
160
|
+
|
|
161
|
+
if template_data is None:
|
|
162
|
+
template_data = TemplateData.from_config(
|
|
163
|
+
recording,
|
|
164
|
+
sorting,
|
|
165
|
+
template_cfg=template_cfg,
|
|
166
|
+
motion_est=motion_est,
|
|
167
|
+
tsvd=tsvd,
|
|
168
|
+
waveform_cfg=waveform_cfg,
|
|
169
|
+
computation_cfg=computation_cfg,
|
|
170
|
+
save_folder=None,
|
|
171
|
+
save_npz_name=None,
|
|
172
|
+
)
|
|
173
|
+
assert sorting is not None
|
|
174
|
+
assert template_data.spike_counts_by_channel is not None
|
|
175
|
+
|
|
176
|
+
good_templates = snr_mask(
|
|
177
|
+
template_data, min_n_spikes=min_n_spikes, min_template_snr=min_template_snr
|
|
178
|
+
)
|
|
179
|
+
logger.dartsortdebug(
|
|
180
|
+
f"Discard {np.logical_not(good_templates).sum()} low-signal templates."
|
|
181
|
+
)
|
|
182
|
+
good_unit_ids = template_data.unit_ids[good_templates]
|
|
183
|
+
assert np.all(np.diff(good_unit_ids) >= 0)
|
|
184
|
+
unique_good_unit_ids, new_template_unit_ids = np.unique(
|
|
185
|
+
good_unit_ids, return_inverse=True
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
if template_data.properties:
|
|
189
|
+
properties = {k: v[good_templates] for k, v in template_data.properties.items()}
|
|
190
|
+
else:
|
|
191
|
+
properties = None
|
|
192
|
+
|
|
193
|
+
assert sorting.labels is not None
|
|
194
|
+
new_labels = sorting.labels.copy()
|
|
195
|
+
valid = np.isin(new_labels, unique_good_unit_ids)
|
|
196
|
+
new_labels[~valid] = -1
|
|
197
|
+
_, new_labels[valid] = np.unique(new_labels[valid], return_inverse=True)
|
|
198
|
+
|
|
199
|
+
new_sorting = sorting.ephemeral_replace(labels=new_labels)
|
|
200
|
+
new_template_data = TemplateData(
|
|
201
|
+
templates=template_data.templates[good_templates],
|
|
202
|
+
unit_ids=new_template_unit_ids,
|
|
203
|
+
spike_counts=template_data.spike_counts[good_templates],
|
|
204
|
+
spike_counts_by_channel=template_data.spike_counts_by_channel[good_templates],
|
|
205
|
+
registered_geom=template_data.registered_geom,
|
|
206
|
+
trough_offset_samples=template_data.trough_offset_samples,
|
|
207
|
+
properties=properties,
|
|
208
|
+
)
|
|
209
|
+
if template_save_folder is not None:
|
|
210
|
+
if template_npz_filename is not None:
|
|
211
|
+
npz = Path(template_save_folder) / template_npz_filename
|
|
212
|
+
new_template_data.to_npz(npz)
|
|
213
|
+
|
|
214
|
+
return new_sorting, new_template_data
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def snr_mask(template_data, min_n_spikes=50, min_template_snr=15.0):
|
|
218
|
+
template_ptps = np.ptp(template_data.templates, 1).max(1)
|
|
219
|
+
template_snrs = template_ptps * np.sqrt(template_data.spike_counts)
|
|
220
|
+
good_templates = np.logical_and(
|
|
221
|
+
template_data.spike_counts >= min_n_spikes,
|
|
222
|
+
template_snrs > min_template_snr,
|
|
223
|
+
)
|
|
224
|
+
return good_templates
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def reorder_by_depth(sorting, template_data):
|
|
228
|
+
assert template_data.registered_geom is not None
|
|
229
|
+
w = template_data.snrs_by_channel()
|
|
230
|
+
w /= w.sum(axis=1, keepdims=True)
|
|
231
|
+
meanz = np.sum(template_data.registered_geom[:, 1] * w, axis=1)
|
|
232
|
+
|
|
233
|
+
# new_to_old[i] = old id for new id i
|
|
234
|
+
new_to_old = np.argsort(meanz, kind="stable")
|
|
235
|
+
# old_to_new[i] = new id for old id i
|
|
236
|
+
old_to_new = np.argsort(new_to_old, kind="stable")
|
|
237
|
+
|
|
238
|
+
if template_data.properties:
|
|
239
|
+
properties = {k: v[new_to_old] for k, v in template_data.properties.items()}
|
|
240
|
+
else:
|
|
241
|
+
properties = {}
|
|
242
|
+
|
|
243
|
+
valid = np.flatnonzero(sorting.labels >= 0)
|
|
244
|
+
labels = np.full_like(sorting.labels, -1)
|
|
245
|
+
labels[valid] = old_to_new[sorting.labels[valid]]
|
|
246
|
+
sorting = sorting.ephemeral_replace(labels=labels)
|
|
247
|
+
|
|
248
|
+
uids = np.arange(len(new_to_old))
|
|
249
|
+
scbc = template_data.raw_std_dev
|
|
250
|
+
if scbc is not None:
|
|
251
|
+
scbc = scbc[new_to_old]
|
|
252
|
+
rsd = template_data.raw_std_dev
|
|
253
|
+
if rsd is not None:
|
|
254
|
+
rsd = rsd[new_to_old]
|
|
255
|
+
template_data = TemplateData(
|
|
256
|
+
templates=template_data.templates[new_to_old],
|
|
257
|
+
unit_ids=uids,
|
|
258
|
+
spike_counts=template_data.spike_counts[new_to_old],
|
|
259
|
+
spike_counts_by_channel=scbc,
|
|
260
|
+
raw_std_dev=rsd,
|
|
261
|
+
registered_geom=template_data.registered_geom,
|
|
262
|
+
trough_offset_samples=template_data.trough_offset_samples,
|
|
263
|
+
properties=properties,
|
|
264
|
+
)
|
|
265
|
+
return sorting, template_data
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def ensure_save(template_data, template_npz_path):
|
|
269
|
+
if template_npz_path is not None:
|
|
270
|
+
template_npz_path.parent.parent.mkdir(exist_ok=True)
|
|
271
|
+
template_npz_path.parent.mkdir(exist_ok=True)
|
|
272
|
+
template_data.to_npz(template_npz_path)
|
|
273
|
+
return template_data
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _check_still_valid(sorting: DARTsortSorting):
|
|
277
|
+
assert sorting.labels is not None
|
|
278
|
+
if (sorting.labels < 0).all():
|
|
279
|
+
raise ValueError("All units were thrown away during template postprocessing.")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _handle_merge(
|
|
283
|
+
*,
|
|
284
|
+
recording: BaseRecording,
|
|
285
|
+
sorting: DARTsortSorting,
|
|
286
|
+
motion_est,
|
|
287
|
+
template_data: TemplateData,
|
|
288
|
+
merge_cfg: TemplateMergeConfig | None,
|
|
289
|
+
computation_cfg: ComputationConfig,
|
|
290
|
+
waveform_cfg: WaveformConfig,
|
|
291
|
+
template_cfg: TemplateConfig,
|
|
292
|
+
) -> tuple[DARTsortSorting, TemplateData]:
|
|
293
|
+
if merge_cfg is None or not merge_cfg.merge_distance_threshold:
|
|
294
|
+
return sorting, template_data
|
|
295
|
+
|
|
296
|
+
from ..clustering.merge import merge_templates
|
|
297
|
+
|
|
298
|
+
merge_shift_samples = waveform_cfg.ms_to_samples(merge_cfg.max_shift_ms)
|
|
299
|
+
merge_res = merge_templates(
|
|
300
|
+
sorting=sorting,
|
|
301
|
+
template_data=template_data,
|
|
302
|
+
max_shift_samples=merge_shift_samples,
|
|
303
|
+
linkage=merge_cfg.linkage,
|
|
304
|
+
merge_distance_threshold=merge_cfg.merge_distance_threshold,
|
|
305
|
+
temporal_upsampling_factor=merge_cfg.temporal_upsampling_factor,
|
|
306
|
+
amplitude_scaling_variance=merge_cfg.amplitude_scaling_variance,
|
|
307
|
+
amplitude_scaling_boundary=merge_cfg.amplitude_scaling_boundary,
|
|
308
|
+
svd_compression_rank=merge_cfg.svd_compression_rank,
|
|
309
|
+
min_spatial_cosine=merge_cfg.min_spatial_cosine,
|
|
310
|
+
computation_cfg=computation_cfg,
|
|
311
|
+
show_progress=True,
|
|
312
|
+
)
|
|
313
|
+
sorting = merge_res["sorting"]
|
|
314
|
+
new_unit_ids = merge_res["new_unit_ids"]
|
|
315
|
+
del merge_res
|
|
316
|
+
assert sorting.labels is not None
|
|
317
|
+
|
|
318
|
+
# determine which units were merged and recompute only those templates
|
|
319
|
+
ul, ui, uc = np.unique(new_unit_ids, return_index=True, return_counts=True)
|
|
320
|
+
n_merged_units = ul.shape[0]
|
|
321
|
+
assert np.array_equal(ul, np.arange(len(ul)))
|
|
322
|
+
needs_recompute = ul[uc > 1]
|
|
323
|
+
if needs_recompute.size:
|
|
324
|
+
recompute_labels = np.where(
|
|
325
|
+
np.isin(sorting.labels, needs_recompute), sorting.labels, -1
|
|
326
|
+
)
|
|
327
|
+
recompute_sorting = sorting.ephemeral_replace(labels=recompute_labels)
|
|
328
|
+
# turn off merge here
|
|
329
|
+
recompute_sorting, recompute_template_data = estimate_template_library(
|
|
330
|
+
recording=recording,
|
|
331
|
+
sorting=recompute_sorting,
|
|
332
|
+
motion_est=motion_est,
|
|
333
|
+
min_template_snr=0.0,
|
|
334
|
+
min_template_count=0,
|
|
335
|
+
waveform_cfg=waveform_cfg,
|
|
336
|
+
template_cfg=template_cfg,
|
|
337
|
+
realign_cfg=None,
|
|
338
|
+
template_merge_cfg=None,
|
|
339
|
+
tsvd=template_data.tsvd,
|
|
340
|
+
computation_cfg=computation_cfg,
|
|
341
|
+
depth_order=False,
|
|
342
|
+
)
|
|
343
|
+
assert len(recompute_template_data.templates) == needs_recompute.size
|
|
344
|
+
assert (
|
|
345
|
+
recompute_template_data.trough_offset_samples
|
|
346
|
+
== template_data.trough_offset_samples
|
|
347
|
+
)
|
|
348
|
+
assert (
|
|
349
|
+
recompute_template_data.spike_length_samples
|
|
350
|
+
== template_data.spike_length_samples
|
|
351
|
+
)
|
|
352
|
+
else:
|
|
353
|
+
recompute_template_data = None
|
|
354
|
+
|
|
355
|
+
# new indices corresponding to kept units
|
|
356
|
+
new_kept_ixs = np.flatnonzero(uc <= 1)
|
|
357
|
+
# original indices corresponding to kept units
|
|
358
|
+
old_kept_ixs = ui[new_kept_ixs]
|
|
359
|
+
# new indices for recomputed units
|
|
360
|
+
new_recompute_ix = np.flatnonzero(uc > 1)
|
|
361
|
+
# original indices corresponding to recomputed units
|
|
362
|
+
old_recompute_ix = ui[new_recompute_ix]
|
|
363
|
+
|
|
364
|
+
logger.dartsortverbose(f"Merged template indices: {new_recompute_ix.tolist()}")
|
|
365
|
+
|
|
366
|
+
# pack up the the templates
|
|
367
|
+
templates = np.empty(
|
|
368
|
+
(n_merged_units, *template_data.templates.shape[1:]),
|
|
369
|
+
dtype=template_data.templates.dtype,
|
|
370
|
+
)
|
|
371
|
+
templates[new_kept_ixs] = template_data.templates[old_kept_ixs]
|
|
372
|
+
if recompute_template_data is not None:
|
|
373
|
+
templates[new_recompute_ix] = recompute_template_data.templates
|
|
374
|
+
|
|
375
|
+
spike_counts = np.empty(
|
|
376
|
+
(n_merged_units, *template_data.spike_counts.shape[1:]),
|
|
377
|
+
dtype=template_data.spike_counts.dtype,
|
|
378
|
+
)
|
|
379
|
+
spike_counts[new_kept_ixs] = template_data.spike_counts[old_kept_ixs]
|
|
380
|
+
if recompute_template_data is not None:
|
|
381
|
+
spike_counts[new_recompute_ix] = recompute_template_data.spike_counts
|
|
382
|
+
|
|
383
|
+
if template_data.spike_counts_by_channel is not None:
|
|
384
|
+
spike_counts_by_channel = np.empty(
|
|
385
|
+
(n_merged_units, *template_data.spike_counts_by_channel.shape[1:]),
|
|
386
|
+
dtype=template_data.spike_counts_by_channel.dtype,
|
|
387
|
+
)
|
|
388
|
+
spike_counts_by_channel[new_kept_ixs] = template_data.spike_counts_by_channel[
|
|
389
|
+
old_kept_ixs
|
|
390
|
+
]
|
|
391
|
+
if recompute_template_data is not None:
|
|
392
|
+
spike_counts_by_channel[new_recompute_ix] = (
|
|
393
|
+
recompute_template_data.spike_counts_by_channel
|
|
394
|
+
)
|
|
395
|
+
else:
|
|
396
|
+
spike_counts_by_channel = None
|
|
397
|
+
|
|
398
|
+
if template_data.raw_std_dev is not None:
|
|
399
|
+
raw_std_dev = np.empty(
|
|
400
|
+
(n_merged_units, *template_data.raw_std_dev.shape[1:]),
|
|
401
|
+
dtype=template_data.raw_std_dev.dtype,
|
|
402
|
+
)
|
|
403
|
+
raw_std_dev[new_kept_ixs] = template_data.raw_std_dev[old_kept_ixs]
|
|
404
|
+
if recompute_template_data is not None:
|
|
405
|
+
raw_std_dev[new_recompute_ix] = recompute_template_data.raw_std_dev
|
|
406
|
+
else:
|
|
407
|
+
raw_std_dev = None
|
|
408
|
+
|
|
409
|
+
template_data = TemplateData(
|
|
410
|
+
templates,
|
|
411
|
+
unit_ids=np.arange(len(templates)),
|
|
412
|
+
spike_counts=spike_counts,
|
|
413
|
+
spike_counts_by_channel=spike_counts_by_channel,
|
|
414
|
+
raw_std_dev=raw_std_dev,
|
|
415
|
+
registered_geom=template_data.registered_geom,
|
|
416
|
+
trough_offset_samples=template_data.trough_offset_samples,
|
|
417
|
+
)
|
|
418
|
+
return sorting, template_data
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _quick_mean_templates(
|
|
422
|
+
recording, sorting, waveform_cfg, computation_cfg, motion_est
|
|
423
|
+
):
|
|
424
|
+
return TemplateData.from_config(
|
|
425
|
+
recording=recording,
|
|
426
|
+
sorting=sorting,
|
|
427
|
+
motion_est=motion_est,
|
|
428
|
+
waveform_cfg=waveform_cfg,
|
|
429
|
+
template_cfg=TemplateConfig(denoising_method="none"),
|
|
430
|
+
computation_cfg=computation_cfg,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def filter_by_unit_mask(
|
|
435
|
+
sorting: DARTsortSorting, keep_mask: np.ndarray, mask_ids: np.ndarray | None = None
|
|
436
|
+
) -> DARTsortSorting:
|
|
437
|
+
assert sorting.labels is not None
|
|
438
|
+
|
|
439
|
+
if mask_ids is not None:
|
|
440
|
+
assert mask_ids.shape == keep_mask.shape
|
|
441
|
+
k_full = mask_ids.max() + 1
|
|
442
|
+
assert k_full >= keep_mask.shape[0]
|
|
443
|
+
mask = np.zeros(k_full, dtype=bool)
|
|
444
|
+
mask[mask_ids[keep_mask]] = True
|
|
445
|
+
keep_mask = mask
|
|
446
|
+
|
|
447
|
+
discard_mask = np.logical_not(keep_mask)
|
|
448
|
+
if not discard_mask.any():
|
|
449
|
+
return sorting
|
|
450
|
+
|
|
451
|
+
valid = np.flatnonzero(sorting.labels >= 0)
|
|
452
|
+
chuck = valid[discard_mask[sorting.labels[valid]]]
|
|
453
|
+
sorting.labels[chuck] = -1
|
|
454
|
+
|
|
455
|
+
return sorting.flatten()
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def flag_possible_cc_error_spikes(
|
|
459
|
+
sorting: DARTsortSorting,
|
|
460
|
+
subtraction_cfg,
|
|
461
|
+
amplitudes_dataset_name="denoised_ptp_amplitudes",
|
|
462
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
463
|
+
from ..util.py_util import timer
|
|
464
|
+
|
|
465
|
+
times = sorting.times_samples
|
|
466
|
+
channels = sorting.channels
|
|
467
|
+
amps = getattr(sorting, amplitudes_dataset_name)
|
|
468
|
+
xy = sorting.geom[channels] # type: ignore
|
|
469
|
+
n = len(times)
|
|
470
|
+
|
|
471
|
+
# rescale units so that the max allowed temporal and spatial dists are 10
|
|
472
|
+
times = times / subtraction_cfg.temporal_dedup_radius_samples
|
|
473
|
+
xy_subtract = xy / subtraction_cfg.subtract_radius
|
|
474
|
+
kdt_subtract = KDTree(np.c_[times, xy_subtract])
|
|
475
|
+
|
|
476
|
+
# get all possible neighbors
|
|
477
|
+
with timer("a"):
|
|
478
|
+
sdm = kdt_subtract.sparse_distance_matrix(
|
|
479
|
+
kdt_subtract, max_distance=1.0, p=np.inf, output_type="ndarray"
|
|
480
|
+
)
|
|
481
|
+
# this is more an adjacency matrix. 0s will be discarded later.
|
|
482
|
+
v = np.ones(sdm["i"].shape, dtype=np.float32)
|
|
483
|
+
coo = coo_array((v, (sdm["i"], sdm["j"])), shape=(n, n), dtype=v.dtype)
|
|
484
|
+
print(f"{coo=}")
|
|
485
|
+
|
|
486
|
+
# remove the exclusions by marking their distance as large
|
|
487
|
+
if subtraction_cfg.spatial_dedup_radius:
|
|
488
|
+
xy_dedup = xy / subtraction_cfg.spatial_dedup_radius
|
|
489
|
+
kdt_dedup = KDTree(np.c_[times, xy_dedup])
|
|
490
|
+
with timer("b"):
|
|
491
|
+
sdm_dedup = kdt_dedup.sparse_distance_matrix(
|
|
492
|
+
kdt_dedup, max_distance=1.0, p=np.inf, output_type="ndarray"
|
|
493
|
+
)
|
|
494
|
+
del kdt_dedup, xy_dedup
|
|
495
|
+
v = np.ones(sdm_dedup["i"].shape, dtype=np.float32)
|
|
496
|
+
coo_dedup = coo_array(
|
|
497
|
+
(v, (sdm_dedup["i"], sdm_dedup["j"])), shape=(n, n), dtype=v.dtype
|
|
498
|
+
)
|
|
499
|
+
print(f"{coo_dedup=}")
|
|
500
|
+
coo = coo - coo_dedup
|
|
501
|
+
coo = coo.tocoo()
|
|
502
|
+
print(f"{coo=}")
|
|
503
|
+
|
|
504
|
+
# dedup neighbors are removed
|
|
505
|
+
coo.sum_duplicates()
|
|
506
|
+
coo.eliminate_zeros()
|
|
507
|
+
|
|
508
|
+
# apply higher amplitude criterion
|
|
509
|
+
bigger = np.flatnonzero(amps[coo.coords[0]] > amps[coo.coords[1]])
|
|
510
|
+
coo.data[bigger] = 0.0
|
|
511
|
+
coo.eliminate_zeros()
|
|
512
|
+
|
|
513
|
+
# i am suspicious if i have any neighbors. obviously many spikes do.
|
|
514
|
+
# it's just that a unit consisting entirely of such spikes is suspicious.
|
|
515
|
+
flagged = np.zeros(len(times), dtype=bool)
|
|
516
|
+
ii, jj = coo.coords
|
|
517
|
+
flagged[ii] = True
|
|
518
|
+
|
|
519
|
+
return flagged, ii, jj
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def cc_flag_criterion(
|
|
523
|
+
sorting: DARTsortSorting,
|
|
524
|
+
detection_cfg: Any | None = None,
|
|
525
|
+
max_cc_flag_rate=1.0,
|
|
526
|
+
cc_flag_entropy_cutoff=0.0,
|
|
527
|
+
amplitudes_dataset_name="denoised_ptp_amplitudes",
|
|
528
|
+
) -> np.ndarray | None:
|
|
529
|
+
if max_cc_flag_rate == 1.0:
|
|
530
|
+
return None
|
|
531
|
+
if not isinstance(detection_cfg, SubtractionConfig):
|
|
532
|
+
# this step applied only at subtraction to clean up nn error units
|
|
533
|
+
return None
|
|
534
|
+
|
|
535
|
+
flagged, ii, jj = flag_possible_cc_error_spikes(
|
|
536
|
+
sorting=sorting,
|
|
537
|
+
subtraction_cfg=detection_cfg,
|
|
538
|
+
amplitudes_dataset_name=amplitudes_dataset_name,
|
|
539
|
+
)
|
|
540
|
+
assert sorting.labels is not None
|
|
541
|
+
li = sorting.labels[ii]
|
|
542
|
+
lj = sorting.labels[jj]
|
|
543
|
+
|
|
544
|
+
rate = np.zeros(sorting.unit_ids.shape)
|
|
545
|
+
entropy = np.zeros_like(rate)
|
|
546
|
+
for j, u in enumerate(sorting.unit_ids):
|
|
547
|
+
inu = np.flatnonzero(sorting.labels == u)
|
|
548
|
+
rate[j] = flagged[inu].mean()
|
|
549
|
+
_, friend_count = np.unique(lj[li == u], return_counts=True)
|
|
550
|
+
friend_p = friend_count / friend_count.sum()
|
|
551
|
+
entropy[j] = -(np.log(friend_p) * friend_p).sum()
|
|
552
|
+
|
|
553
|
+
bad = (rate > max_cc_flag_rate) & (entropy < cc_flag_entropy_cutoff)
|
|
554
|
+
logger.dartsortdebug(f"CCG peak criterion flagged {bad.sum().item()} units.")
|
|
555
|
+
return np.logical_not(bad)
|