DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
import string
|
|
3
|
+
from typing import Literal
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from scipy.spatial import KDTree
|
|
9
|
+
from spikeinterface.comparison import GroundTruthComparison
|
|
10
|
+
from tqdm.auto import tqdm
|
|
11
|
+
|
|
12
|
+
from ..clustering import merge
|
|
13
|
+
from .analysis import DARTsortAnalysis
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class DARTsortGroundTruthComparison:
|
|
18
|
+
gt_analysis: DARTsortAnalysis
|
|
19
|
+
tested_analysis: DARTsortAnalysis
|
|
20
|
+
|
|
21
|
+
delta_time: float = 0.8
|
|
22
|
+
match_score: float = 0.0
|
|
23
|
+
chance_score: float = 0.0
|
|
24
|
+
well_detected_score: float = 0.8
|
|
25
|
+
exhaustive_gt: bool = False
|
|
26
|
+
n_jobs: int = -1
|
|
27
|
+
match_mode: str = "hungarian"
|
|
28
|
+
compute_labels: bool = True
|
|
29
|
+
verbose: bool = False
|
|
30
|
+
device: str | None = None
|
|
31
|
+
|
|
32
|
+
compute_distances: bool = True
|
|
33
|
+
compute_unsorted_recall: bool = True
|
|
34
|
+
unsorted_match_radius: float = 50.0
|
|
35
|
+
distance_kind: Literal["rms", "max", "deconv"] = "deconv"
|
|
36
|
+
|
|
37
|
+
def __post_init__(self):
|
|
38
|
+
self._check()
|
|
39
|
+
self.comparison = GroundTruthComparison(
|
|
40
|
+
gt_sorting=self.gt_analysis.sorting.to_numpy_sorting(),
|
|
41
|
+
tested_sorting=self.tested_analysis.sorting.to_numpy_sorting(),
|
|
42
|
+
gt_name=self.gt_analysis.name,
|
|
43
|
+
tested_name=self.tested_analysis.name,
|
|
44
|
+
delta_time=self.delta_time,
|
|
45
|
+
match_score=self.match_score,
|
|
46
|
+
chance_score=self.chance_score,
|
|
47
|
+
well_detected_score=self.well_detected_score,
|
|
48
|
+
exhaustive_gt=self.exhaustive_gt,
|
|
49
|
+
match_mode=self.match_mode,
|
|
50
|
+
verbose=self.verbose,
|
|
51
|
+
compute_labels=self.compute_labels,
|
|
52
|
+
)
|
|
53
|
+
self.delta_frames = self.comparison.delta_frames
|
|
54
|
+
self.has_templates = self.tested_analysis.template_data is not None
|
|
55
|
+
self._agreement_scores = None
|
|
56
|
+
|
|
57
|
+
if self.compute_distances and self.has_templates:
|
|
58
|
+
if self.verbose:
|
|
59
|
+
print("Calculating GT/tested template distances...")
|
|
60
|
+
self._calculate_template_distances()
|
|
61
|
+
|
|
62
|
+
self._unsorted_detection = self._greedy_confusion = None
|
|
63
|
+
if self.compute_unsorted_recall:
|
|
64
|
+
self._calculate_greedy_confusion_and_detection()
|
|
65
|
+
|
|
66
|
+
def _check(self):
|
|
67
|
+
gt_td = self.gt_analysis.template_data
|
|
68
|
+
tested_td = self.tested_analysis.template_data
|
|
69
|
+
if gt_td is not None and tested_td is not None:
|
|
70
|
+
gt_rg = gt_td.registered_geom
|
|
71
|
+
tested_rg = tested_td.registered_geom
|
|
72
|
+
if gt_rg is None:
|
|
73
|
+
assert tested_rg is None
|
|
74
|
+
assert gt_rg is not None
|
|
75
|
+
assert tested_rg is not None
|
|
76
|
+
if not np.array_equal(gt_rg, tested_rg):
|
|
77
|
+
raise ValueError(
|
|
78
|
+
f"Template data had different registered geoms: "
|
|
79
|
+
f"{gt_rg.shape=} {tested_rg.shape=}"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def gt_name(self):
|
|
84
|
+
return self.gt_analysis.name or "GT"
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def tested_name(self):
|
|
88
|
+
return self.tested_analysis.name or "Tested"
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def unit_ids(self):
|
|
92
|
+
return self.gt_analysis.sorting.unit_ids
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def n_gt_units(self):
|
|
96
|
+
return len(self.gt_analysis.sorting.unit_ids)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def agreement_scores(self):
|
|
100
|
+
"""Make sure that all GT and tested units are present."""
|
|
101
|
+
if self._agreement_scores is not None:
|
|
102
|
+
return self._agreement_scores
|
|
103
|
+
a = self.comparison.agreement_scores.copy()
|
|
104
|
+
gtids = np.arange(self.gt_analysis.sorting.unit_ids.max() + 1)
|
|
105
|
+
tids = np.arange(self.tested_analysis.sorting.unit_ids.max() + 1)
|
|
106
|
+
a = a.reindex(index=gtids, columns=tids, fill_value=0.0)
|
|
107
|
+
self._agreement_scores = a
|
|
108
|
+
return a
|
|
109
|
+
|
|
110
|
+
def unit_amplitudes(self, unit_id):
|
|
111
|
+
return self.gt_analysis.unit_amplitudes(unit_id)
|
|
112
|
+
|
|
113
|
+
def get_match(self, gt_unit):
|
|
114
|
+
assert self.comparison.hungarian_match_12 is not None
|
|
115
|
+
return int(self.comparison.hungarian_match_12[gt_unit])
|
|
116
|
+
|
|
117
|
+
def get_best_match(self, gt_unit):
|
|
118
|
+
assert self.comparison.best_match_12 is not None
|
|
119
|
+
return int(self.comparison.best_match_12[gt_unit])
|
|
120
|
+
|
|
121
|
+
def unit_info_dataframe(self, force_distances=False, perf_only=False):
|
|
122
|
+
amplitudes = self.gt_analysis.unit_amplitudes()
|
|
123
|
+
firing_rates = self.gt_analysis.firing_rates()
|
|
124
|
+
df = self.comparison.get_performance()
|
|
125
|
+
assert isinstance(df, pd.DataFrame)
|
|
126
|
+
df = df.astype(float) # not sure what the problem was...
|
|
127
|
+
df["gt_ptp_amplitude"] = amplitudes
|
|
128
|
+
df["gt_firing_rate"] = firing_rates
|
|
129
|
+
if perf_only:
|
|
130
|
+
return df
|
|
131
|
+
|
|
132
|
+
# coll, matched_coll, missed_coll = self.unit_collidedness()
|
|
133
|
+
# assert coll.shape == matched_coll.shape == missed_coll.shape == df.index.shape
|
|
134
|
+
# df["gt_collidedness"] = coll
|
|
135
|
+
# df["gt_matched_collidedness"] = matched_coll
|
|
136
|
+
# df["gt_missed_collidedness"] = missed_coll
|
|
137
|
+
try:
|
|
138
|
+
df["gt_dt_rms"] = self.unit_matched_misalignment_rms()
|
|
139
|
+
except ValueError:
|
|
140
|
+
pass
|
|
141
|
+
if self.has_templates and (force_distances or self.compute_distances):
|
|
142
|
+
dist = np.nan_to_num(self.template_distances, nan=np.inf).min(axis=1)
|
|
143
|
+
df["min_temp_dist"] = dist
|
|
144
|
+
rec = []
|
|
145
|
+
for uid in df.index:
|
|
146
|
+
assert self.unsorted_detection is not None
|
|
147
|
+
rec.append(self.unsorted_detection[self.gt_analysis.in_unit(uid)].mean())
|
|
148
|
+
df["unsorted_recall"] = rec
|
|
149
|
+
return df
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def template_distances(self):
|
|
153
|
+
self._calculate_template_distances()
|
|
154
|
+
return self._template_distances
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def unsorted_detection(self):
|
|
158
|
+
self._calculate_greedy_confusion_and_detection()
|
|
159
|
+
return self._unsorted_detection
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def greedy_confusion(self):
|
|
163
|
+
self._calculate_greedy_confusion_and_detection()
|
|
164
|
+
return self._greedy_confusion
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def greedy_iou(self):
|
|
168
|
+
self._calculate_greedy_confusion_and_detection()
|
|
169
|
+
return self._greedy_iou
|
|
170
|
+
|
|
171
|
+
def unit_collidedness(self):
|
|
172
|
+
uids = self.gt_analysis.sorting.unit_ids
|
|
173
|
+
c = np.full(len(uids), np.nan)
|
|
174
|
+
matched_c = c.copy()
|
|
175
|
+
missed_c = c.copy()
|
|
176
|
+
|
|
177
|
+
collidedness = getattr(self.gt_analysis.sorting, "collidedness", None)
|
|
178
|
+
if collidedness is None:
|
|
179
|
+
return c, matched_c, missed_c
|
|
180
|
+
|
|
181
|
+
for j, uid in enumerate(uids):
|
|
182
|
+
inu, matchu, missu = self.matched_and_missed(uid)
|
|
183
|
+
if inu.size:
|
|
184
|
+
c[j] = collidedness[inu].mean()
|
|
185
|
+
if matchu.size:
|
|
186
|
+
matched_c[j] = collidedness[matchu].mean()
|
|
187
|
+
if missu.size:
|
|
188
|
+
missed_c[j] = collidedness[missu].mean()
|
|
189
|
+
|
|
190
|
+
return c, matched_c, missed_c
|
|
191
|
+
|
|
192
|
+
def unit_matched_misalignment_rms(self):
|
|
193
|
+
uids = self.gt_analysis.sorting.unit_ids
|
|
194
|
+
match_dt_rms = np.full(len(uids), np.nan)
|
|
195
|
+
for j, uid in enumerate(uids):
|
|
196
|
+
try:
|
|
197
|
+
udt = self.matched_misalignment(uid)
|
|
198
|
+
if udt is None:
|
|
199
|
+
continue
|
|
200
|
+
if udt.size:
|
|
201
|
+
match_dt_rms[j] = np.sqrt(np.square(udt).mean())
|
|
202
|
+
except ValueError as e:
|
|
203
|
+
warnings.warn(
|
|
204
|
+
f"ValueError in misalignment. SI matching bug. {e=}" # type: ignore
|
|
205
|
+
)
|
|
206
|
+
return match_dt_rms
|
|
207
|
+
|
|
208
|
+
def matched_misalignment(self, gt_unit_id):
|
|
209
|
+
spikes = self.get_spikes_by_category(gt_unit_id)
|
|
210
|
+
gt_matched_t = self.gt_analysis.sorting.times_samples[
|
|
211
|
+
spikes["matched_gt_indices"]
|
|
212
|
+
]
|
|
213
|
+
test_matched_t = self.tested_analysis.sorting.times_samples[
|
|
214
|
+
spikes["matched_tested_indices"]
|
|
215
|
+
]
|
|
216
|
+
if test_matched_t.shape != gt_matched_t.shape:
|
|
217
|
+
return None
|
|
218
|
+
match_dt = test_matched_t - gt_matched_t
|
|
219
|
+
return match_dt
|
|
220
|
+
|
|
221
|
+
def nearby_gt_templates(self, gt_unit_id, n_neighbors=5):
|
|
222
|
+
return self.gt_analysis.nearby_coarse_templates(
|
|
223
|
+
gt_unit_id, n_neighbors=n_neighbors
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
def nearby_tested_templates(self, gt_unit_id, n_neighbors=5):
|
|
227
|
+
gt_td = self.gt_analysis.coarse_template_data
|
|
228
|
+
tested_td = self.tested_analysis.coarse_template_data
|
|
229
|
+
assert gt_td is not None
|
|
230
|
+
assert tested_td is not None
|
|
231
|
+
|
|
232
|
+
gt_unit_ix = np.searchsorted(gt_td.unit_ids, gt_unit_id)
|
|
233
|
+
assert gt_td.unit_ids[gt_unit_ix] == gt_unit_id
|
|
234
|
+
|
|
235
|
+
unit_dists = self.template_distances[gt_unit_ix]
|
|
236
|
+
distance_order = np.argsort(unit_dists)
|
|
237
|
+
|
|
238
|
+
# assert distance_order[0] == unit_ix
|
|
239
|
+
neighb_ixs = distance_order[:n_neighbors]
|
|
240
|
+
neighb_ids = tested_td.unit_ids[neighb_ixs]
|
|
241
|
+
neighb_dists = self.template_distances[gt_unit_ix, neighb_ixs[None, :]]
|
|
242
|
+
neighb_coarse_templates = tested_td.templates[neighb_ixs]
|
|
243
|
+
|
|
244
|
+
return neighb_ixs, neighb_ids, neighb_dists, neighb_coarse_templates
|
|
245
|
+
|
|
246
|
+
def _calculate_template_distances(self):
|
|
247
|
+
"""Compute the merge distance matrix"""
|
|
248
|
+
if hasattr(self, "_template_distances"):
|
|
249
|
+
return
|
|
250
|
+
|
|
251
|
+
(
|
|
252
|
+
dists,
|
|
253
|
+
shifts,
|
|
254
|
+
snrs_a,
|
|
255
|
+
snrs_b,
|
|
256
|
+
a_mask,
|
|
257
|
+
b_mask,
|
|
258
|
+
) = merge.cross_match_distance_matrix(
|
|
259
|
+
self.gt_analysis.coarse_template_data,
|
|
260
|
+
self.tested_analysis.coarse_template_data,
|
|
261
|
+
sym_function=np.minimum,
|
|
262
|
+
n_jobs=1,
|
|
263
|
+
svd_compression_rank=10,
|
|
264
|
+
device="cpu",
|
|
265
|
+
min_spatial_cosine=0.7,
|
|
266
|
+
distance_kind=self.distance_kind,
|
|
267
|
+
)
|
|
268
|
+
self._template_distances = dists
|
|
269
|
+
|
|
270
|
+
def _calculate_greedy_confusion_and_detection(self):
|
|
271
|
+
from ..evaluate.hybrid_util import greedy_match, greedy_match_counts
|
|
272
|
+
|
|
273
|
+
if self._unsorted_detection is not None:
|
|
274
|
+
return
|
|
275
|
+
if self.verbose:
|
|
276
|
+
print("Calculate unsorted detection...")
|
|
277
|
+
frames_per_ms = self.gt_analysis.recording.sampling_frequency / 1000
|
|
278
|
+
delta_frames = self.delta_time * frames_per_ms
|
|
279
|
+
greedy_res = greedy_match_counts(
|
|
280
|
+
self.gt_analysis.sorting,
|
|
281
|
+
self.tested_analysis.sorting,
|
|
282
|
+
radius_frames=int(delta_frames),
|
|
283
|
+
show_progress=self.verbose,
|
|
284
|
+
)
|
|
285
|
+
c = greedy_res["counts"]
|
|
286
|
+
self._greedy_confusion = c
|
|
287
|
+
u = (c.sum(0, keepdims=True) + c.sum(1, keepdims=True)) - c
|
|
288
|
+
self._greedy_iou = c / np.maximum(u, 1)
|
|
289
|
+
|
|
290
|
+
self._tested_to_gt = greedy_res["test2gt_spike"]
|
|
291
|
+
self._unsorted_detection = np.logical_not(greedy_res["gt_unmatched"])
|
|
292
|
+
gtns = self.gt_analysis.sorting.n_spikes
|
|
293
|
+
assert self._unsorted_detection.shape == (gtns,)
|
|
294
|
+
|
|
295
|
+
def matched_and_missed(self, gt_unit):
|
|
296
|
+
(gt_spike_labels,) = self.comparison.get_labels1(gt_unit)
|
|
297
|
+
in_gt_unit = self.gt_analysis.in_unit(gt_unit)
|
|
298
|
+
matched_gt_mask = gt_spike_labels == "TP"
|
|
299
|
+
matched_gt_indices = in_gt_unit[matched_gt_mask]
|
|
300
|
+
only_gt_indices = in_gt_unit[np.logical_not(matched_gt_mask)]
|
|
301
|
+
return in_gt_unit, matched_gt_indices, only_gt_indices
|
|
302
|
+
|
|
303
|
+
def get_spikes_by_category(self, gt_unit, tested_unit=None):
|
|
304
|
+
if tested_unit is None:
|
|
305
|
+
tested_unit = self.get_match(gt_unit)
|
|
306
|
+
|
|
307
|
+
# convert to global index space
|
|
308
|
+
in_gt_unit, matched_gt_indices, only_gt_indices = self.matched_and_missed(
|
|
309
|
+
gt_unit
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
if tested_unit >= 0:
|
|
313
|
+
(tested_spike_labels,) = self.comparison.get_labels2(tested_unit)
|
|
314
|
+
in_tested_unit = self.tested_analysis.in_unit(tested_unit)
|
|
315
|
+
matched_tested_mask = tested_spike_labels == "TP"
|
|
316
|
+
matched_tested_indices = in_tested_unit[matched_tested_mask]
|
|
317
|
+
only_tested_indices = in_tested_unit[np.logical_not(matched_tested_mask)]
|
|
318
|
+
if matched_gt_indices.size != matched_tested_indices.size:
|
|
319
|
+
# raise ValueError(
|
|
320
|
+
warnings.warn(
|
|
321
|
+
f"Strange match sizes for {gt_unit=} {tested_unit=}: "
|
|
322
|
+
f"{matched_gt_indices.shape=} {matched_tested_indices.shape=} "
|
|
323
|
+
f"{matched_tested_mask.sum()=}"
|
|
324
|
+
)
|
|
325
|
+
else:
|
|
326
|
+
matched_tested_indices = np.zeros(shape=(0,), dtype=np.int64)
|
|
327
|
+
only_tested_indices = np.zeros(shape=(0,), dtype=np.int64)
|
|
328
|
+
|
|
329
|
+
# add unsorted matches and misses
|
|
330
|
+
if self.unsorted_detection is None:
|
|
331
|
+
unsorted_tp_indices = unsorted_fn_indices = None
|
|
332
|
+
else:
|
|
333
|
+
unsorted_match_mask = self.unsorted_detection[in_gt_unit]
|
|
334
|
+
unsorted_tp_indices = in_gt_unit[unsorted_match_mask]
|
|
335
|
+
unsorted_fn_indices = in_gt_unit[np.logical_not(unsorted_match_mask)]
|
|
336
|
+
|
|
337
|
+
return dict(
|
|
338
|
+
tested_unit=tested_unit,
|
|
339
|
+
matched_tested_indices=matched_tested_indices,
|
|
340
|
+
matched_gt_indices=matched_gt_indices,
|
|
341
|
+
only_gt_indices=only_gt_indices,
|
|
342
|
+
only_tested_indices=only_tested_indices,
|
|
343
|
+
unsorted_tp_indices=unsorted_tp_indices,
|
|
344
|
+
unsorted_fn_indices=unsorted_fn_indices,
|
|
345
|
+
fn_times_samples=self.gt_analysis.sorting.times_samples[only_gt_indices],
|
|
346
|
+
fp_times_samples=self.tested_analysis.sorting.times_samples[
|
|
347
|
+
only_tested_indices
|
|
348
|
+
],
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
def get_greedy_correspondence_in_si_match(self):
|
|
352
|
+
tlabels = self.tested_analysis.sorting.labels
|
|
353
|
+
gt_labels_for_tested = np.full_like(tlabels, -1)
|
|
354
|
+
to_ms = 1000.0 / self.gt_analysis.recording.sampling_frequency
|
|
355
|
+
for gtu in tqdm(self.gt_analysis.unit_ids, desc="Spike match"):
|
|
356
|
+
tu = self.get_match(gtu)
|
|
357
|
+
if tu < 0:
|
|
358
|
+
continue
|
|
359
|
+
in_gt = self.gt_analysis.in_unit(gtu)
|
|
360
|
+
in_tu = self.tested_analysis.in_unit(tu)
|
|
361
|
+
gt_times_ms = self.gt_analysis.sorting.times_samples[in_gt] * to_ms
|
|
362
|
+
tested_times_ms = self.tested_analysis.sorting.times_samples[in_tu] * to_ms
|
|
363
|
+
gt_kdt = KDTree(gt_times_ms[:, None])
|
|
364
|
+
dd, ii = gt_kdt.query(
|
|
365
|
+
tested_times_ms[:, None], distance_upper_bound=self.delta_time
|
|
366
|
+
)
|
|
367
|
+
ii = np.atleast_1d(ii)
|
|
368
|
+
gt_labels_for_tested[in_tu[ii < gt_kdt.n]] = in_gt[ii[ii < gt_kdt.n]]
|
|
369
|
+
|
|
370
|
+
return dict(
|
|
371
|
+
gt_labels_for_tested=gt_labels_for_tested,
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
def get_raw_waveforms_by_category(
|
|
375
|
+
self,
|
|
376
|
+
gt_unit,
|
|
377
|
+
tested_unit=None,
|
|
378
|
+
max_samples_per_category=100,
|
|
379
|
+
random_seed=0,
|
|
380
|
+
):
|
|
381
|
+
rg = np.random.default_rng(random_seed)
|
|
382
|
+
ind_groups = self.get_spikes_by_category(gt_unit, tested_unit=tested_unit)
|
|
383
|
+
tested_unit = ind_groups["tested_unit"]
|
|
384
|
+
|
|
385
|
+
# waveforms are read at GT unit max channel
|
|
386
|
+
gt_max_chan = self.gt_analysis.unit_max_channel(gt_unit)
|
|
387
|
+
waveform_kw = dict(
|
|
388
|
+
max_count=max_samples_per_category, random_seed=rg, main_channel=gt_max_chan
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# return vars dict. lots of stuff going in here.
|
|
392
|
+
w: dict[str, int | np.ndarray | None] = dict(
|
|
393
|
+
gt_unit=gt_unit, tested_unit=tested_unit, max_chan=gt_max_chan
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
# load TP waveforms
|
|
397
|
+
# which, waveforms, max_chan, show_geom, show_channel_index
|
|
398
|
+
tp_waves = self.gt_analysis.unit_raw_waveforms(
|
|
399
|
+
which=ind_groups["matched_gt_indices"], # type: ignore
|
|
400
|
+
**waveform_kw, # type: ignore
|
|
401
|
+
)
|
|
402
|
+
if tp_waves is None:
|
|
403
|
+
w["which_tp"] = None
|
|
404
|
+
w["tp"] = None
|
|
405
|
+
w["geom"] = self.gt_analysis.registered_geom
|
|
406
|
+
w["channel_index"] = self.gt_analysis.vis_channel_index
|
|
407
|
+
else:
|
|
408
|
+
w["which_tp"] = tp_waves.which
|
|
409
|
+
w["tp"] = tp_waves.waveforms
|
|
410
|
+
w["geom"] = tp_waves.geom
|
|
411
|
+
w["channel_index"] = tp_waves.channel_index
|
|
412
|
+
|
|
413
|
+
# load FN waveforms
|
|
414
|
+
# which, waveforms, max_chan, show_geom, show_channel_index
|
|
415
|
+
fn_waves = self.gt_analysis.unit_raw_waveforms(
|
|
416
|
+
which=ind_groups["only_gt_indices"], # type: ignore
|
|
417
|
+
**waveform_kw, # type: ignore
|
|
418
|
+
)
|
|
419
|
+
if fn_waves is None:
|
|
420
|
+
w["which_fn"] = None
|
|
421
|
+
w["fn"] = None
|
|
422
|
+
else:
|
|
423
|
+
w["which_fn"] = fn_waves.which
|
|
424
|
+
w["fn"] = fn_waves.waveforms
|
|
425
|
+
|
|
426
|
+
# load FP waveforms
|
|
427
|
+
# which, waveforms, max_chan, show_geom, show_channel_index
|
|
428
|
+
fp_waves = self.tested_analysis.unit_raw_waveforms(
|
|
429
|
+
which=ind_groups["only_tested_indices"], # type: ignore
|
|
430
|
+
**waveform_kw, # type: ignore
|
|
431
|
+
)
|
|
432
|
+
if fp_waves is None:
|
|
433
|
+
w["which_fp"] = None
|
|
434
|
+
w["fp"] = None
|
|
435
|
+
else:
|
|
436
|
+
w["which_fp"] = fp_waves.which
|
|
437
|
+
w["fp"] = fp_waves.waveforms
|
|
438
|
+
|
|
439
|
+
if self.unsorted_detection is None:
|
|
440
|
+
w["unsorted_tp"] = w["unsorted_fn"] = None
|
|
441
|
+
else:
|
|
442
|
+
utp_waves = self.gt_analysis.unit_raw_waveforms(
|
|
443
|
+
which=ind_groups["unsorted_tp_indices"], # type: ignore
|
|
444
|
+
**waveform_kw, # type: ignore
|
|
445
|
+
)
|
|
446
|
+
if utp_waves is None:
|
|
447
|
+
w["which_unsorted_tp"] = None
|
|
448
|
+
w["unsorted_tp"] = None
|
|
449
|
+
else:
|
|
450
|
+
w["which_unsorted_tp"] = utp_waves.which
|
|
451
|
+
w["unsorted_tp"] = utp_waves.waveforms
|
|
452
|
+
ufn_waves = self.gt_analysis.unit_raw_waveforms(
|
|
453
|
+
which=ind_groups["unsorted_fn_indices"], # type: ignore
|
|
454
|
+
**waveform_kw, # type: ignore
|
|
455
|
+
)
|
|
456
|
+
if ufn_waves is None:
|
|
457
|
+
w["which_unsorted_fn"] = None
|
|
458
|
+
w["unsorted_fn"] = None
|
|
459
|
+
else:
|
|
460
|
+
w["which_unsorted_fn"] = ufn_waves.which
|
|
461
|
+
w["unsorted_fn"] = ufn_waves.waveforms
|
|
462
|
+
|
|
463
|
+
return w
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
class DARTsortGTVersus:
|
|
467
|
+
default_ids = string.ascii_uppercase
|
|
468
|
+
|
|
469
|
+
def __init__(
|
|
470
|
+
self,
|
|
471
|
+
gt_analysis: DARTsortAnalysis,
|
|
472
|
+
*other_analyses: DARTsortAnalysis,
|
|
473
|
+
sorter_var="sorter",
|
|
474
|
+
comparison_kw=None,
|
|
475
|
+
comparisons=None,
|
|
476
|
+
):
|
|
477
|
+
comparison_kw = comparison_kw or {}
|
|
478
|
+
self.sorter_var = sorter_var
|
|
479
|
+
|
|
480
|
+
# some things we can only do for head to head comparisons
|
|
481
|
+
# but sometimes it's a battle royale :P
|
|
482
|
+
self.is_two = len(other_analyses) == 2
|
|
483
|
+
self.n_vs = len(other_analyses)
|
|
484
|
+
|
|
485
|
+
self.gt_name = gt_analysis.name or "GT"
|
|
486
|
+
self.gt_templates = gt_analysis.template_data
|
|
487
|
+
self.gt_sorting = gt_analysis.sorting
|
|
488
|
+
self.gt_analysis = gt_analysis
|
|
489
|
+
|
|
490
|
+
self.other_analyses = other_analyses
|
|
491
|
+
self.other_names = [oa.name for oa in other_analyses]
|
|
492
|
+
self.other_templates = [
|
|
493
|
+
(oa.name or f"Test{c}") for oa, c in zip(other_analyses, self.default_ids)
|
|
494
|
+
]
|
|
495
|
+
self.other_sortings = [oa.sorting for oa in other_analyses]
|
|
496
|
+
|
|
497
|
+
if comparisons is None:
|
|
498
|
+
comparisons = [None] * self.n_vs
|
|
499
|
+
else:
|
|
500
|
+
assert len(comparisons) == self.n_vs
|
|
501
|
+
|
|
502
|
+
self.cmps = []
|
|
503
|
+
for cmp, oa in zip(comparisons, other_analyses):
|
|
504
|
+
if cmp is not None:
|
|
505
|
+
assert cmp.tested_analysis.name == oa.name
|
|
506
|
+
else:
|
|
507
|
+
cmp = DARTsortGroundTruthComparison(
|
|
508
|
+
gt_analysis=gt_analysis, tested_analysis=oa, **comparison_kw
|
|
509
|
+
)
|
|
510
|
+
self.cmps.append(cmp)
|
|
511
|
+
|
|
512
|
+
if self.is_two:
|
|
513
|
+
self.a_name, self.b_name = self.other_names
|
|
514
|
+
self.a_templates, self.b_templates = self.other_templates
|
|
515
|
+
self.a_sorting, self.b_sorting = self.other_sortings
|
|
516
|
+
self.a_cmp, self.b_cmp = self.cmps
|
|
517
|
+
self._unit_vs_df = None
|
|
518
|
+
self.n_gt_units = self.cmps[0].n_gt_units
|
|
519
|
+
|
|
520
|
+
def unit_versus_dataframe(self) -> pd.DataFrame:
|
|
521
|
+
"""Combine the performance into one dataframe."""
|
|
522
|
+
if self._unit_vs_df is not None:
|
|
523
|
+
return self._unit_vs_df.copy(deep=True)
|
|
524
|
+
dfs = [ocmp.unit_info_dataframe().reset_index() for ocmp in self.cmps]
|
|
525
|
+
for df, sorter in zip(dfs, self.other_names):
|
|
526
|
+
df[self.sorter_var] = sorter
|
|
527
|
+
self._unit_vs_df = pd.concat(dfs, axis=0)
|
|
528
|
+
return self._unit_vs_df.copy(deep=True)
|
|
529
|
+
|
|
530
|
+
def tagname(self):
|
|
531
|
+
vsstr = "vs_" + ",".join(
|
|
532
|
+
oa.name or str(j) for j, oa in enumerate(self.other_analyses)
|
|
533
|
+
)
|
|
534
|
+
return f"{self.gt_name}_{vsstr}"
|