DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
import time
|
|
3
|
+
import warnings
|
|
4
|
+
from logging import getLogger
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Generator, cast
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import torch
|
|
10
|
+
from probeinterface import Probe
|
|
11
|
+
from scipy.spatial import KDTree
|
|
12
|
+
from scipy.spatial.distance import pdist
|
|
13
|
+
from spikeinterface.generation.drift_tools import (
|
|
14
|
+
DriftingTemplates,
|
|
15
|
+
InjectDriftingTemplatesRecording,
|
|
16
|
+
move_dense_templates,
|
|
17
|
+
)
|
|
18
|
+
from tqdm.auto import tqdm
|
|
19
|
+
|
|
20
|
+
from ..config import DeveloperConfig
|
|
21
|
+
from ..templates import TemplateData
|
|
22
|
+
from ..util.data_util import DARTsortSorting
|
|
23
|
+
from ..util.internal_config import ComputationConfig, unshifted_raw_template_cfg
|
|
24
|
+
from ..util.py_util import resolve_path
|
|
25
|
+
from . import analysis, comparison, simkit
|
|
26
|
+
|
|
27
|
+
logger = getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_drifty_hybrid_recording(
|
|
31
|
+
recording,
|
|
32
|
+
templates,
|
|
33
|
+
motion_estimate,
|
|
34
|
+
sorting=None,
|
|
35
|
+
displacement_sampling_frequency=5.0,
|
|
36
|
+
seed=0,
|
|
37
|
+
firing_rates=None,
|
|
38
|
+
peak_channels=None,
|
|
39
|
+
amplitude_scale_std=0.1,
|
|
40
|
+
amplitude_factor=None,
|
|
41
|
+
):
|
|
42
|
+
"""
|
|
43
|
+
:param: recording
|
|
44
|
+
:param: templates object
|
|
45
|
+
:param: motion estimate object
|
|
46
|
+
:param: firing_rates
|
|
47
|
+
:param: peak_channels
|
|
48
|
+
:param: amplitude_scale_std -- std of gamma distributed amplitude variation if
|
|
49
|
+
amplitude_factor is None
|
|
50
|
+
:param: amplitude_factor array of length n_spikes with amplitude factors
|
|
51
|
+
"""
|
|
52
|
+
num_units = templates.num_units
|
|
53
|
+
rg = np.random.default_rng(seed=seed)
|
|
54
|
+
|
|
55
|
+
if peak_channels is None:
|
|
56
|
+
(central_disp_index,) = np.flatnonzero(np.all(templates.displacements == 0, 1))
|
|
57
|
+
central_templates = templates.templates_array_moved[central_disp_index]
|
|
58
|
+
peak_channels = np.ptp(central_templates, 1).argmax(1)
|
|
59
|
+
|
|
60
|
+
if sorting is None:
|
|
61
|
+
sorting = simkit.simulate_sorting(
|
|
62
|
+
num_units,
|
|
63
|
+
recording.get_num_samples(),
|
|
64
|
+
firing_rates=firing_rates,
|
|
65
|
+
rg=rg,
|
|
66
|
+
spike_length_samples=templates.num_samples,
|
|
67
|
+
sampling_frequency=recording.sampling_frequency,
|
|
68
|
+
)
|
|
69
|
+
n_spikes = sorting.count_total_num_spikes()
|
|
70
|
+
|
|
71
|
+
# Default amplitude scalings for spikes drawn from gamma
|
|
72
|
+
if amplitude_factor is None:
|
|
73
|
+
if amplitude_scale_std:
|
|
74
|
+
shape = 1.0 / (amplitude_scale_std**1.5)
|
|
75
|
+
amplitude_factor = rg.gamma(shape, scale=1.0 / (shape - 1), size=n_spikes)
|
|
76
|
+
else:
|
|
77
|
+
amplitude_factor = np.ones(n_spikes)
|
|
78
|
+
|
|
79
|
+
depths = recording.get_probe().contact_positions[:, 1][peak_channels]
|
|
80
|
+
t_start = recording.sample_index_to_time(0)
|
|
81
|
+
t_end = recording.sample_index_to_time(recording.get_num_samples() - 1)
|
|
82
|
+
motion_times_s = np.arange(
|
|
83
|
+
t_start, t_end, step=1.0 / displacement_sampling_frequency
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
disp_y = motion_estimate.disp_at_s(motion_times_s, depths, grid=True)
|
|
87
|
+
|
|
88
|
+
disp = np.zeros((motion_times_s.shape[0], 2, num_units))
|
|
89
|
+
disp[:, 1, :] = disp_y.T
|
|
90
|
+
# this tricks SI into using one displacement per unit
|
|
91
|
+
displacement_unit_factor = np.eye(num_units)
|
|
92
|
+
|
|
93
|
+
if not sorting.check_serializability(type="json"):
|
|
94
|
+
warnings.warn(
|
|
95
|
+
"Your sorting is not serializable, which could lead to problems later."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
rec = InjectDriftingTemplatesRecording(
|
|
99
|
+
sorting=sorting,
|
|
100
|
+
drifting_templates=templates,
|
|
101
|
+
parent_recording=recording,
|
|
102
|
+
displacement_vectors=[disp],
|
|
103
|
+
displacement_sampling_frequency=displacement_sampling_frequency,
|
|
104
|
+
displacement_unit_factor=displacement_unit_factor,
|
|
105
|
+
amplitude_factor=amplitude_factor,
|
|
106
|
+
)
|
|
107
|
+
rec.annotate(peak_channel=peak_channels.tolist())
|
|
108
|
+
return rec
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def precompute_displaced_registered_templates(
|
|
112
|
+
template_data: TemplateData,
|
|
113
|
+
geometry: np.ndarray,
|
|
114
|
+
displacements: np.ndarray,
|
|
115
|
+
sampling_frequency: float = 30000,
|
|
116
|
+
template_subset=slice(None),
|
|
117
|
+
) -> DriftingTemplates:
|
|
118
|
+
"""Use spikeinterface tools to turn templates on registered geom into
|
|
119
|
+
precomputed drifting templates on the regular geom.
|
|
120
|
+
"""
|
|
121
|
+
assert template_data.registered_geom is not None
|
|
122
|
+
source_probe = Probe(ndim=template_data.registered_geom.ndim)
|
|
123
|
+
source_probe.set_contacts(positions=template_data.registered_geom)
|
|
124
|
+
target_probe = Probe(ndim=geometry.ndim)
|
|
125
|
+
target_probe.set_contacts(positions=geometry)
|
|
126
|
+
|
|
127
|
+
shifted_templates = move_dense_templates(
|
|
128
|
+
templates_array=template_data.templates[template_subset],
|
|
129
|
+
displacements=displacements,
|
|
130
|
+
source_probe=source_probe,
|
|
131
|
+
dest_probe=target_probe,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
ret = DriftingTemplates.from_precomputed_templates(
|
|
135
|
+
templates_array_moved=shifted_templates,
|
|
136
|
+
displacements=displacements,
|
|
137
|
+
sampling_frequency=sampling_frequency,
|
|
138
|
+
nbefore=template_data.trough_offset_samples,
|
|
139
|
+
probe=target_probe,
|
|
140
|
+
)
|
|
141
|
+
return ret
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def closest_clustering(
|
|
145
|
+
gt_st, peel_st, geom=None, match_dt_ms=0.1, match_radius_um=0.0, p=2.0
|
|
146
|
+
):
|
|
147
|
+
frames_per_ms = gt_st.sampling_frequency / 1000
|
|
148
|
+
delta_frames = match_dt_ms * frames_per_ms
|
|
149
|
+
rescale = [delta_frames]
|
|
150
|
+
gt_pos = gt_st.times_samples[:, None]
|
|
151
|
+
peel_pos = peel_st.times_samples[:, None]
|
|
152
|
+
if match_radius_um:
|
|
153
|
+
assert isinstance(geom, np.ndarray)
|
|
154
|
+
rescale = rescale + (geom.shape[1] * [match_radius_um])
|
|
155
|
+
gt_pos = np.c_[gt_pos, geom[gt_st.channels]]
|
|
156
|
+
peel_pos = np.c_[peel_pos, geom[peel_st.channels]]
|
|
157
|
+
else:
|
|
158
|
+
gt_pos = gt_pos.astype(float)
|
|
159
|
+
peel_pos = peel_pos.astype(float)
|
|
160
|
+
gt_pos /= rescale
|
|
161
|
+
peel_pos /= rescale
|
|
162
|
+
peelix2gtix = greedy_match(gt_pos, peel_pos, dx=1.0 / frames_per_ms)
|
|
163
|
+
labels = peelix2gtix.copy() # type: ignore
|
|
164
|
+
labels[labels >= 0] = gt_st.labels[labels[labels >= 0]]
|
|
165
|
+
|
|
166
|
+
return peel_st.ephemeral_replace(labels=labels, gt_match_ix=peelix2gtix)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def greedy_match(
|
|
170
|
+
gt_coords,
|
|
171
|
+
test_coords,
|
|
172
|
+
max_val=1.0,
|
|
173
|
+
dx=1.0 / 30,
|
|
174
|
+
workers=-1,
|
|
175
|
+
p=2,
|
|
176
|
+
show_progress=True,
|
|
177
|
+
):
|
|
178
|
+
"""Greedily match spikes in a test sorting to those in a GT sorting
|
|
179
|
+
|
|
180
|
+
Iteratively expands a spatiotemporal neighborhood around each test spike
|
|
181
|
+
until a GT spike lands in the neighborhood; then arbitrarily pick from
|
|
182
|
+
the GT spikes in the neighborhood as a match.
|
|
183
|
+
|
|
184
|
+
Each spike in the GT and test spike trains will be matched to only one
|
|
185
|
+
partner in the other sorting.
|
|
186
|
+
|
|
187
|
+
Returns gt unit labels for each tested spike and an array describing
|
|
188
|
+
whether each gt spike was matched.
|
|
189
|
+
|
|
190
|
+
TODO: Be a little smarter. If there are ties, pick "best friend" unit.
|
|
191
|
+
Could be done with two iterations.
|
|
192
|
+
|
|
193
|
+
TODO: Return the gt->test assignments, which are easily computable here,
|
|
194
|
+
if needed.
|
|
195
|
+
|
|
196
|
+
Returns
|
|
197
|
+
-------
|
|
198
|
+
assignments: np.ndarray
|
|
199
|
+
GT spike ind for each tested spike
|
|
200
|
+
gt_unmatched: np.ndarray
|
|
201
|
+
Whether a GT spike had no tested match
|
|
202
|
+
"""
|
|
203
|
+
assignments = np.full(len(test_coords), -1, dtype=np.intp)
|
|
204
|
+
gt_unmatched = np.ones(len(gt_coords), dtype=bool)
|
|
205
|
+
|
|
206
|
+
thresholds = np.arange(0.0 + 1e-5, max_val + dx + 2e-5, dx)
|
|
207
|
+
if show_progress:
|
|
208
|
+
thresholds = tqdm(thresholds, desc="Greedy match")
|
|
209
|
+
|
|
210
|
+
for j, thresh in enumerate(thresholds):
|
|
211
|
+
test_unmatched = np.flatnonzero(assignments < 0)
|
|
212
|
+
if not test_unmatched.size:
|
|
213
|
+
break
|
|
214
|
+
test_kdtree = KDTree(test_coords[test_unmatched])
|
|
215
|
+
gt_ix = np.flatnonzero(gt_unmatched)
|
|
216
|
+
d, i = test_kdtree.query(
|
|
217
|
+
gt_coords[gt_ix],
|
|
218
|
+
k=1,
|
|
219
|
+
distance_upper_bound=min(thresh, max_val), # type: ignore
|
|
220
|
+
workers=workers,
|
|
221
|
+
p=p,
|
|
222
|
+
)
|
|
223
|
+
i = np.atleast_1d(i)
|
|
224
|
+
# handle multiple gt spikes getting matched to the same peel ix
|
|
225
|
+
thresh_matched = i < test_kdtree.n
|
|
226
|
+
_, ii = np.unique(i, return_index=True)
|
|
227
|
+
i = i[ii]
|
|
228
|
+
thresh_matched = thresh_matched[ii]
|
|
229
|
+
|
|
230
|
+
gt_ix = gt_ix[ii]
|
|
231
|
+
gt_ix = gt_ix[thresh_matched]
|
|
232
|
+
i = i[thresh_matched]
|
|
233
|
+
assignments[test_unmatched[i]] = gt_ix
|
|
234
|
+
gt_unmatched[gt_ix] = False
|
|
235
|
+
|
|
236
|
+
if not gt_unmatched.any():
|
|
237
|
+
break
|
|
238
|
+
if thresh > max_val:
|
|
239
|
+
break
|
|
240
|
+
|
|
241
|
+
return assignments, gt_unmatched
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def greedy_match_counts(
|
|
245
|
+
gt_sorting,
|
|
246
|
+
tested_sorting,
|
|
247
|
+
radius_um=35.0,
|
|
248
|
+
radius_frames=12,
|
|
249
|
+
show_progress=True,
|
|
250
|
+
):
|
|
251
|
+
"""A greedy confusion matrix computed using greedy_match()."""
|
|
252
|
+
from scipy.optimize import linear_sum_assignment
|
|
253
|
+
|
|
254
|
+
gt_t = gt_sorting.times_samples / radius_frames
|
|
255
|
+
tested_t = tested_sorting.times_samples / radius_frames
|
|
256
|
+
|
|
257
|
+
geom = getattr(gt_sorting, "geom", getattr(tested_sorting, "geom", None))
|
|
258
|
+
assert geom is not None
|
|
259
|
+
gt_x = geom[gt_sorting.channels] / radius_um
|
|
260
|
+
tested_x = geom[tested_sorting.channels] / radius_um
|
|
261
|
+
|
|
262
|
+
step = min(1.0 / radius_frames, pdist(geom).min() / radius_um) / 2
|
|
263
|
+
|
|
264
|
+
test2gt_spike, gt_unmatched = greedy_match(
|
|
265
|
+
np.c_[gt_t, gt_x], np.c_[tested_t, tested_x], dx=step
|
|
266
|
+
)
|
|
267
|
+
counts = np.zeros(
|
|
268
|
+
(gt_sorting.unit_ids.max() + 1, tested_sorting.unit_ids.max() + 1),
|
|
269
|
+
dtype=np.int32,
|
|
270
|
+
)
|
|
271
|
+
test_matched_spike = np.flatnonzero(
|
|
272
|
+
np.logical_and(test2gt_spike >= 0, tested_sorting.labels >= 0)
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
matched_gt_labels = gt_sorting.labels[test2gt_spike[test_matched_spike]]
|
|
276
|
+
matched_test_labels = tested_sorting.labels[test_matched_spike]
|
|
277
|
+
|
|
278
|
+
np.add.at(counts, (matched_gt_labels, matched_test_labels), 1)
|
|
279
|
+
|
|
280
|
+
row_ind, col_ind = linear_sum_assignment(counts, maximize=True)
|
|
281
|
+
|
|
282
|
+
return dict(
|
|
283
|
+
counts=counts,
|
|
284
|
+
test2gt_spike=test2gt_spike,
|
|
285
|
+
gt_unmatched=gt_unmatched,
|
|
286
|
+
test_matched_spike=test_matched_spike,
|
|
287
|
+
matched_gt_labels=matched_gt_labels,
|
|
288
|
+
matched_test_labels=matched_test_labels,
|
|
289
|
+
counts_ord=counts[row_ind][:, col_ind],
|
|
290
|
+
gt_unit_order=row_ind,
|
|
291
|
+
test_unit_order=col_ind,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def sorting_from_times_labels(
|
|
296
|
+
times_samples,
|
|
297
|
+
labels,
|
|
298
|
+
recording=None,
|
|
299
|
+
motion_est=None,
|
|
300
|
+
sampling_frequency=30000.0,
|
|
301
|
+
determine_channels=True,
|
|
302
|
+
template_cfg=unshifted_raw_template_cfg,
|
|
303
|
+
n_jobs=0,
|
|
304
|
+
spikes_per_unit=50,
|
|
305
|
+
):
|
|
306
|
+
channels = np.zeros_like(labels)
|
|
307
|
+
if sampling_frequency is None:
|
|
308
|
+
if recording is not None:
|
|
309
|
+
sampling_frequency = recording.sampling_frequency
|
|
310
|
+
|
|
311
|
+
efeat = {}
|
|
312
|
+
if recording is not None:
|
|
313
|
+
efeat["times_seconds"] = recording.sample_index_to_time(times_samples)
|
|
314
|
+
|
|
315
|
+
sorting = DARTsortSorting(
|
|
316
|
+
times_samples=times_samples,
|
|
317
|
+
channels=channels,
|
|
318
|
+
labels=labels,
|
|
319
|
+
sampling_frequency=sampling_frequency,
|
|
320
|
+
parent_h5_path=None,
|
|
321
|
+
ephemeral_features=efeat,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
if not determine_channels:
|
|
325
|
+
return sorting
|
|
326
|
+
|
|
327
|
+
assert recording is not None
|
|
328
|
+
|
|
329
|
+
_, labels_flat = np.unique(labels, return_inverse=True)
|
|
330
|
+
sorting = sorting.ephemeral_replace(labels=labels_flat)
|
|
331
|
+
template_cfg = dataclasses.replace(template_cfg, spikes_per_unit=spikes_per_unit)
|
|
332
|
+
comp_cfg = ComputationConfig(n_jobs_cpu=n_jobs, n_jobs_gpu=n_jobs)
|
|
333
|
+
td = TemplateData.from_config(
|
|
334
|
+
recording, sorting, template_cfg, computation_cfg=comp_cfg
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
channels = np.nan_to_num(np.ptp(td.coarsen().templates, 1)).argmax(1)[labels_flat]
|
|
338
|
+
if motion_est is not None:
|
|
339
|
+
from scipy.spatial import KDTree
|
|
340
|
+
|
|
341
|
+
rgeom = td.registered_geom
|
|
342
|
+
assert rgeom is not None
|
|
343
|
+
guess_pos = rgeom[channels]
|
|
344
|
+
times_seconds = recording.sample_index_to_time(times_samples)
|
|
345
|
+
# anti-correct these already stable positions so that they start movin
|
|
346
|
+
guess_pos[:, 1] += motion_est.disp_at_s(times_seconds, depth_um=guess_pos[:, 1])
|
|
347
|
+
|
|
348
|
+
gkdt = KDTree(recording.get_channel_locations())
|
|
349
|
+
# closest original channels to shifted positions
|
|
350
|
+
# these positions can drift off the probe if the main channel does! so in that case
|
|
351
|
+
# we can't really upper bound the distance query. i guess it would be, like, the
|
|
352
|
+
# largest distance that a unit would ever extend, or something, but let's not worry.
|
|
353
|
+
d, channels = gkdt.query(guess_pos, workers=n_jobs)
|
|
354
|
+
|
|
355
|
+
assert isinstance(channels, np.ndarray)
|
|
356
|
+
sorting = sorting.ephemeral_replace(channels=channels)
|
|
357
|
+
|
|
358
|
+
return sorting, td
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def sorting_from_spikeinterface(
|
|
362
|
+
sorting,
|
|
363
|
+
recording=None,
|
|
364
|
+
determine_channels=True,
|
|
365
|
+
template_cfg=unshifted_raw_template_cfg,
|
|
366
|
+
n_jobs=0,
|
|
367
|
+
):
|
|
368
|
+
sv = sorting.to_spike_vector()
|
|
369
|
+
return sorting_from_times_labels(
|
|
370
|
+
sv["sample_index"],
|
|
371
|
+
sv["unit_index"],
|
|
372
|
+
sampling_frequency=sorting.sampling_frequency,
|
|
373
|
+
recording=recording,
|
|
374
|
+
determine_channels=determine_channels,
|
|
375
|
+
template_cfg=template_cfg,
|
|
376
|
+
n_jobs=n_jobs,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _same(x):
|
|
381
|
+
return x
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def load_dartsort_step_sortings(
|
|
385
|
+
sorting_dir,
|
|
386
|
+
load_simple_features=False,
|
|
387
|
+
load_feature_names=("times_seconds", "geom", "channel_index"),
|
|
388
|
+
detection_h5_names=(
|
|
389
|
+
"subtraction.h5",
|
|
390
|
+
"threshold.h5",
|
|
391
|
+
"universal.h5",
|
|
392
|
+
"matching0.h5",
|
|
393
|
+
),
|
|
394
|
+
detection_h5_path: Path | str | None = None,
|
|
395
|
+
step_format="refined{step}",
|
|
396
|
+
recluster_format="recluster{step}",
|
|
397
|
+
mtime_gap_minutes=0,
|
|
398
|
+
name_formatter=None,
|
|
399
|
+
) -> Generator[tuple[str, DARTsortSorting] | tuple[None, None], None, None]:
|
|
400
|
+
"""Returns list of step names and sortings, ordered.
|
|
401
|
+
|
|
402
|
+
The mtime thing is trying to prevent reading hdf5 files which are in active
|
|
403
|
+
use, although its not a guarantee... h5 locking... need to figure it out.
|
|
404
|
+
"""
|
|
405
|
+
mtime_dt = mtime_gap_minutes * 60 if mtime_gap_minutes else 0
|
|
406
|
+
if detection_h5_path is None:
|
|
407
|
+
for dh5n in detection_h5_names:
|
|
408
|
+
detection_h5_path = cast(Path, sorting_dir / dh5n)
|
|
409
|
+
if not detection_h5_path.exists():
|
|
410
|
+
continue
|
|
411
|
+
if mtime_dt:
|
|
412
|
+
age = time.time() - detection_h5_path.stat().st_mtime
|
|
413
|
+
if age < mtime_dt:
|
|
414
|
+
continue
|
|
415
|
+
h5s = [resolve_path(detection_h5_path)]
|
|
416
|
+
break
|
|
417
|
+
else:
|
|
418
|
+
h5s = []
|
|
419
|
+
else:
|
|
420
|
+
h5s = [resolve_path(detection_h5_path)]
|
|
421
|
+
|
|
422
|
+
for j in range(1, 100):
|
|
423
|
+
mh5 = sorting_dir / f"matching{j}.h5"
|
|
424
|
+
if not mh5.exists():
|
|
425
|
+
break
|
|
426
|
+
if mtime_dt:
|
|
427
|
+
if time.time() - mh5.stat().st_mtime < mtime_dt:
|
|
428
|
+
break
|
|
429
|
+
h5s.append(resolve_path(mh5))
|
|
430
|
+
|
|
431
|
+
# let's check that there is at least something to do...
|
|
432
|
+
labels_npys = sorting_dir.glob("*_labels.npy")
|
|
433
|
+
relevant_files = [sorting_dir / "dartsort_sorting.npz", *labels_npys, *h5s[1:]]
|
|
434
|
+
if not any(f.exists() for f in relevant_files):
|
|
435
|
+
h5s = []
|
|
436
|
+
|
|
437
|
+
if name_formatter is None:
|
|
438
|
+
name_formatter = _same
|
|
439
|
+
|
|
440
|
+
for step, h5 in enumerate(h5s):
|
|
441
|
+
if not h5.exists():
|
|
442
|
+
continue
|
|
443
|
+
st0 = DARTsortSorting.from_peeling_hdf5(
|
|
444
|
+
h5,
|
|
445
|
+
load_simple_features=load_simple_features,
|
|
446
|
+
load_feature_names=load_feature_names,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
# initial clust or later step res?
|
|
450
|
+
if h5 == h5s[0]:
|
|
451
|
+
npy = sorting_dir / "initial_labels.npy"
|
|
452
|
+
if npy.exists():
|
|
453
|
+
yield (
|
|
454
|
+
name_formatter("initial"),
|
|
455
|
+
st0.ephemeral_replace(labels=np.load(npy)),
|
|
456
|
+
)
|
|
457
|
+
else:
|
|
458
|
+
warnings.warn(f"Initial {npy} does not exist.")
|
|
459
|
+
yield None, None
|
|
460
|
+
|
|
461
|
+
other_initial_npys = sorted(sorting_dir.glob("initial_*_labels.npy"))
|
|
462
|
+
for npy in other_initial_npys:
|
|
463
|
+
npy = sorting_dir / npy
|
|
464
|
+
stem = npy.stem.removesuffix("_labels")
|
|
465
|
+
yield (
|
|
466
|
+
name_formatter(stem),
|
|
467
|
+
st0.ephemeral_replace(labels=np.load(npy)),
|
|
468
|
+
)
|
|
469
|
+
else:
|
|
470
|
+
yield (name_formatter(h5.stem), st0)
|
|
471
|
+
|
|
472
|
+
# reclustering, if applicable
|
|
473
|
+
reclustr = recluster_format.format(step=step)
|
|
474
|
+
recluster_npy = sorting_dir / f"{reclustr}_labels.npy"
|
|
475
|
+
if recluster_npy.exists():
|
|
476
|
+
yield (
|
|
477
|
+
name_formatter(reclustr),
|
|
478
|
+
st0.ephemeral_replace(labels=np.load(recluster_npy)),
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
# refinement steps
|
|
482
|
+
stepstr = step_format.format(step=step)
|
|
483
|
+
for npy in sorted(sorting_dir.glob(f"{stepstr}*.npy")):
|
|
484
|
+
stem = npy.stem.removesuffix("_labels")
|
|
485
|
+
if stem == stepstr:
|
|
486
|
+
continue
|
|
487
|
+
yield (
|
|
488
|
+
name_formatter(stem),
|
|
489
|
+
st0.ephemeral_replace(labels=np.load(npy)),
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def load_dartsort_step_unit_info_dataframes(
|
|
494
|
+
sorting_dir,
|
|
495
|
+
gt_analysis,
|
|
496
|
+
recording,
|
|
497
|
+
sorting_name=None,
|
|
498
|
+
detection_h5_names=(
|
|
499
|
+
"subtraction.h5",
|
|
500
|
+
"threshold.h5",
|
|
501
|
+
"universal.h5",
|
|
502
|
+
"matching0.h5",
|
|
503
|
+
),
|
|
504
|
+
detection_h5_path: Path | str | None = None,
|
|
505
|
+
step_format="refined{step}",
|
|
506
|
+
):
|
|
507
|
+
step_sortings = load_dartsort_step_sortings(
|
|
508
|
+
sorting_dir,
|
|
509
|
+
detection_h5_names=detection_h5_names,
|
|
510
|
+
detection_h5_path=detection_h5_path,
|
|
511
|
+
step_format=step_format,
|
|
512
|
+
)
|
|
513
|
+
for step_ix, (step_name, step_sorting) in enumerate(step_sortings):
|
|
514
|
+
if step_name is None:
|
|
515
|
+
continue
|
|
516
|
+
assert step_sorting is not None
|
|
517
|
+
name = f"{sorting_name}: {step_name}" if sorting_name else step_name
|
|
518
|
+
step_analysis = analysis.DARTsortAnalysis.from_sorting(
|
|
519
|
+
recording=recording,
|
|
520
|
+
sorting=step_sorting,
|
|
521
|
+
motion_est=None,
|
|
522
|
+
name=name,
|
|
523
|
+
template_cfg=None,
|
|
524
|
+
vis_radius=0,
|
|
525
|
+
)
|
|
526
|
+
step_comparison = comparison.DARTsortGroundTruthComparison(
|
|
527
|
+
gt_analysis, step_analysis
|
|
528
|
+
)
|
|
529
|
+
df = step_comparison.unit_info_dataframe()
|
|
530
|
+
df["stepix"] = step_ix
|
|
531
|
+
df["stepname"] = step_name
|
|
532
|
+
yield df
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def config_grid(
|
|
536
|
+
common_params: dict | None = None,
|
|
537
|
+
name_prefix="",
|
|
538
|
+
config_cls: Any = DeveloperConfig,
|
|
539
|
+
**grid_params,
|
|
540
|
+
):
|
|
541
|
+
"""Configuration grid search helper
|
|
542
|
+
|
|
543
|
+
Generate a set product of aptly named configuration dictionaries or objects.
|
|
544
|
+
|
|
545
|
+
Example usage:
|
|
546
|
+
|
|
547
|
+
```python
|
|
548
|
+
config_grid(
|
|
549
|
+
{'save_intermediate_labels': True, 'n_jobs_cpu': 2},
|
|
550
|
+
badness={".1":dict(denoiser_badness_factor=0.1),".25":dict(denoiser_badness_factor=0.25)},
|
|
551
|
+
crit={"hell":dict(merge_criterion="heldout_loglik"),"helb":dict(merge_criterion="heldout_elbo")},
|
|
552
|
+
)
|
|
553
|
+
=> nice nested grid with names
|
|
554
|
+
```
|
|
555
|
+
|
|
556
|
+
Arguments
|
|
557
|
+
---------
|
|
558
|
+
common_params: dict
|
|
559
|
+
Arguments shared across all settings.
|
|
560
|
+
name_prefix: str
|
|
561
|
+
Will appear, with _, at the start of all keys in the returned dict.
|
|
562
|
+
config_cls: class or None
|
|
563
|
+
**grid_params
|
|
564
|
+
Each key-value pair here should be as follows. The key is like the
|
|
565
|
+
"tag" for this setting. The value is a dict whose keys are setting
|
|
566
|
+
nicknames and values are themselves dicts containing the actual
|
|
567
|
+
configurations. See the example above.
|
|
568
|
+
|
|
569
|
+
Returns
|
|
570
|
+
-------
|
|
571
|
+
grid: dict[str, config_cls]
|
|
572
|
+
My length is product(v for k, v in grid_params.items()).
|
|
573
|
+
"""
|
|
574
|
+
if common_params is None:
|
|
575
|
+
common_params = {}
|
|
576
|
+
|
|
577
|
+
# base case (won't actually end up in the final grid)
|
|
578
|
+
grid = [(name_prefix, common_params)]
|
|
579
|
+
|
|
580
|
+
for param_nickname, param_settings in grid_params.items():
|
|
581
|
+
new_grid = []
|
|
582
|
+
|
|
583
|
+
for setting_nickname, kw in param_settings.items():
|
|
584
|
+
# induction
|
|
585
|
+
for cur_name, cur_params in grid:
|
|
586
|
+
prefix = f"{cur_name}_" if cur_name else ""
|
|
587
|
+
new_name = f"{prefix}{param_nickname}{setting_nickname}"
|
|
588
|
+
# new overrides old if overlapping
|
|
589
|
+
new_grid.append((new_name, cur_params | kw))
|
|
590
|
+
|
|
591
|
+
grid = new_grid
|
|
592
|
+
|
|
593
|
+
# convert to dict of configs
|
|
594
|
+
if config_cls is not None:
|
|
595
|
+
grid = {name: config_cls(**kw) for name, kw in grid}
|
|
596
|
+
else:
|
|
597
|
+
grid = dict(grid)
|
|
598
|
+
|
|
599
|
+
return grid
|