DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/vis/mixture.py
ADDED
|
@@ -0,0 +1,1489 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Iterable, cast
|
|
4
|
+
|
|
5
|
+
import matplotlib.pyplot as plt
|
|
6
|
+
import numpy as np
|
|
7
|
+
import torch
|
|
8
|
+
from dredge.motion_util import MotionEstimate
|
|
9
|
+
from matplotlib.lines import Line2D
|
|
10
|
+
from scipy.sparse.csgraph import connected_components
|
|
11
|
+
from tqdm.auto import tqdm
|
|
12
|
+
|
|
13
|
+
from ..clustering.gmm.mixture import (
|
|
14
|
+
Scores,
|
|
15
|
+
StreamingSpikeData,
|
|
16
|
+
TruncatedMixtureModel,
|
|
17
|
+
TruncatedSpikeData,
|
|
18
|
+
NeighborhoodLUT,
|
|
19
|
+
instantiate_and_bootstrap_tmm,
|
|
20
|
+
labels_from_scores,
|
|
21
|
+
run_split,
|
|
22
|
+
try_kmeans,
|
|
23
|
+
)
|
|
24
|
+
from ..transform import TemporalPCA
|
|
25
|
+
from ..util import spiketorch
|
|
26
|
+
from ..util.data_util import DARTsortSorting, get_tpca, resolve_path
|
|
27
|
+
from ..util.internal_config import (
|
|
28
|
+
ComputationConfig,
|
|
29
|
+
InterpolationParams,
|
|
30
|
+
RefinementConfig,
|
|
31
|
+
default_refinement_cfg,
|
|
32
|
+
)
|
|
33
|
+
from ..util.interpolation_util import (
|
|
34
|
+
NeighborhoodFiller,
|
|
35
|
+
StableFeaturesInterpolator,
|
|
36
|
+
pad_geom,
|
|
37
|
+
)
|
|
38
|
+
from ..util.job_util import ensure_computation_config
|
|
39
|
+
from ..util.multiprocessing_util import CloudpicklePoolExecutor, get_pool
|
|
40
|
+
from ..util.py_util import databag
|
|
41
|
+
from .analysis_plots import distance_matrix_dendro
|
|
42
|
+
from .colors import glasbey1024
|
|
43
|
+
from .layout import BasePlot, flow_layout
|
|
44
|
+
from .waveforms import geomplot
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@databag
|
|
48
|
+
class MixtureVisData:
|
|
49
|
+
"""Collection passed around to the plots in this file, with helper methods for vis."""
|
|
50
|
+
|
|
51
|
+
tmm: TruncatedMixtureModel
|
|
52
|
+
train_data: TruncatedSpikeData
|
|
53
|
+
val_data: TruncatedSpikeData | None
|
|
54
|
+
full_data: StreamingSpikeData
|
|
55
|
+
sorting: DARTsortSorting
|
|
56
|
+
motion_est: MotionEstimate | None
|
|
57
|
+
train_scores: Scores
|
|
58
|
+
full_scores: Scores
|
|
59
|
+
eval_scores: Scores
|
|
60
|
+
train_times: np.ndarray
|
|
61
|
+
train_labels: np.ndarray
|
|
62
|
+
train_ixs: np.ndarray
|
|
63
|
+
full_labels: np.ndarray
|
|
64
|
+
tpca: TemporalPCA
|
|
65
|
+
inf_diag_unit_distance_matrix: torch.Tensor
|
|
66
|
+
prgeom: np.ndarray
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def times_seconds(self):
|
|
70
|
+
return self.sorting.times_seconds # type: ignore
|
|
71
|
+
|
|
72
|
+
def to_sorting(self) -> DARTsortSorting:
|
|
73
|
+
return self.sorting.ephemeral_replace(labels=self.full_labels)
|
|
74
|
+
|
|
75
|
+
def inu_and_times_full(self, unit_id: int):
|
|
76
|
+
inu_full = np.flatnonzero(self.full_labels == unit_id)
|
|
77
|
+
times = self.times_seconds[inu_full]
|
|
78
|
+
return inu_full, times
|
|
79
|
+
|
|
80
|
+
def train_inds_and_chans(
|
|
81
|
+
self, unit_id: int, count=None
|
|
82
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
83
|
+
inu_train = np.flatnonzero(self.train_labels == unit_id)
|
|
84
|
+
if count and inu_train.size > count:
|
|
85
|
+
rg = np.random.default_rng(0)
|
|
86
|
+
inu_train = rg.choice(inu_train, size=count, replace=False)
|
|
87
|
+
inu_train.sort()
|
|
88
|
+
|
|
89
|
+
neighb_ids = self.train_data.neighborhood_ids[inu_train].cpu()
|
|
90
|
+
chans = self.tmm.neighb_cov.obs_ix.cpu()[neighb_ids]
|
|
91
|
+
return inu_train, chans.numpy(force=True)
|
|
92
|
+
|
|
93
|
+
def chans_in_radius(self, unit_id: int, radius: float):
|
|
94
|
+
mean = self.tmm.b.means[unit_id].view(self.tmm.neighb_cov.feat_rank, -1)
|
|
95
|
+
my_chan = mean.square().sum(0).argmax()
|
|
96
|
+
my_xy = self.tmm.neighb_cov.prgeom[my_chan]
|
|
97
|
+
dxy = self.tmm.neighb_cov.prgeom - my_xy
|
|
98
|
+
inf_dist = dxy.abs_().amax(dim=1)
|
|
99
|
+
(close,) = (inf_dist < radius).cpu().nonzero(as_tuple=True)
|
|
100
|
+
return close
|
|
101
|
+
|
|
102
|
+
def friends(self, unit_id: int, count: int, me_last=True):
|
|
103
|
+
dists, neighbors = self.inf_diag_unit_distance_matrix[unit_id].sort()
|
|
104
|
+
count = min(count, dists.numel() - 1)
|
|
105
|
+
assert count >= 0
|
|
106
|
+
if count == 0:
|
|
107
|
+
return np.zeros(1), np.array([unit_id])
|
|
108
|
+
dists = dists[:count]
|
|
109
|
+
neighbors = neighbors[:count]
|
|
110
|
+
assert not (neighbors == unit_id).any()
|
|
111
|
+
assert dists.isfinite().all()
|
|
112
|
+
neighbors = np.concatenate([[unit_id], neighbors.numpy(force=True)])
|
|
113
|
+
dists = np.concatenate([[0.0], dists.numpy(force=True)])
|
|
114
|
+
if me_last:
|
|
115
|
+
dists = np.ascontiguousarray(dists[::-1])
|
|
116
|
+
neighbors = np.ascontiguousarray(neighbors[::-1])
|
|
117
|
+
return dists, neighbors
|
|
118
|
+
|
|
119
|
+
def reconstruct_flat(self, features: torch.Tensor) -> np.ndarray:
|
|
120
|
+
frank = self.tmm.neighb_cov.feat_rank
|
|
121
|
+
single = features.ndim == 1
|
|
122
|
+
if single:
|
|
123
|
+
features = features[None]
|
|
124
|
+
assert features.ndim == 2
|
|
125
|
+
n = features.shape[0]
|
|
126
|
+
features = features.view(n, frank, -1)
|
|
127
|
+
features = features.to(self.tpca.b.components.device)
|
|
128
|
+
recon = self.tpca.force_reconstruct(features)
|
|
129
|
+
if single:
|
|
130
|
+
recon = recon[0]
|
|
131
|
+
return recon.numpy(force=True)
|
|
132
|
+
|
|
133
|
+
def random_train_waveforms(
|
|
134
|
+
self, unit_id: int, count=128
|
|
135
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
|
136
|
+
inu_train, chans = self.train_inds_and_chans(unit_id, count=count)
|
|
137
|
+
features = self.train_data.x[inu_train]
|
|
138
|
+
waveforms = self.reconstruct_flat(features)
|
|
139
|
+
features = features.numpy(force=True)
|
|
140
|
+
return inu_train, chans, features, waveforms
|
|
141
|
+
|
|
142
|
+
def full_lut_scores(self, show_progress=2, n_candidates=None):
|
|
143
|
+
lut0 = self.tmm.lut
|
|
144
|
+
lut1 = torch.ones_like(lut0.lut)
|
|
145
|
+
unit_ids, neighb_ids = lut1.nonzero(as_tuple=True)
|
|
146
|
+
lut1[unit_ids, neighb_ids] = torch.arange(unit_ids.shape[0]).to(lut1)
|
|
147
|
+
new_lut = NeighborhoodLUT(unit_ids=unit_ids, neighb_ids=neighb_ids, lut=lut1)
|
|
148
|
+
self.tmm.update_lut(new_lut, no_parameter_changes=True)
|
|
149
|
+
# old_nc = self.full_data.n_candidates
|
|
150
|
+
sc = self.tmm.soft_assign(
|
|
151
|
+
data=self.full_data,
|
|
152
|
+
needs_bootstrap=False,
|
|
153
|
+
full_proposal_view=True,
|
|
154
|
+
show_progress=show_progress,
|
|
155
|
+
)
|
|
156
|
+
return sc
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class MixtureComponentPlot(BasePlot):
|
|
160
|
+
width = 1
|
|
161
|
+
height = 1
|
|
162
|
+
kind = "mixture"
|
|
163
|
+
|
|
164
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
165
|
+
raise NotImplementedError
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class TextInfo(MixtureComponentPlot):
|
|
169
|
+
kind = "small"
|
|
170
|
+
height = 0.5
|
|
171
|
+
|
|
172
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
173
|
+
axis = panel.subplots()
|
|
174
|
+
axis.axis("off")
|
|
175
|
+
|
|
176
|
+
ntot = (mix_data.full_labels == unit_id).sum()
|
|
177
|
+
ntrain = (mix_data.train_labels == unit_id).sum()
|
|
178
|
+
|
|
179
|
+
msg = f"$\\mathbf{{unit\\ {unit_id}}}$\n"
|
|
180
|
+
msg += f"n total: {ntot}\n"
|
|
181
|
+
msg += f"n train: {ntrain}\n"
|
|
182
|
+
|
|
183
|
+
axis.text(
|
|
184
|
+
0.025,
|
|
185
|
+
0.5,
|
|
186
|
+
msg,
|
|
187
|
+
fontsize="small",
|
|
188
|
+
ha="left",
|
|
189
|
+
va="center",
|
|
190
|
+
transform=axis.transAxes,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class ISIHistogram(MixtureComponentPlot):
|
|
195
|
+
kind = "small"
|
|
196
|
+
height = 1.0
|
|
197
|
+
|
|
198
|
+
def __init__(self, bin_ms=0.1, max_ms=5):
|
|
199
|
+
self.bin_ms = bin_ms
|
|
200
|
+
self.max_ms = max_ms
|
|
201
|
+
|
|
202
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
203
|
+
axis = panel.subplots()
|
|
204
|
+
inu_full, times_s = mix_data.inu_and_times_full(unit_id)
|
|
205
|
+
dt_ms = np.diff(times_s) * 1000
|
|
206
|
+
bin_edges = np.arange(
|
|
207
|
+
-0.5 * self.bin_ms, self.max_ms + self.bin_ms + 1e-5, self.bin_ms
|
|
208
|
+
)
|
|
209
|
+
counts, _ = np.histogram(dt_ms, bin_edges)
|
|
210
|
+
axis.stairs(counts, bin_edges, color=glasbey1024[unit_id], fill=True)
|
|
211
|
+
axis.set_xlabel(f"isi (ms, {dt_ms.size + 1} tot. sp.)")
|
|
212
|
+
axis.set_ylabel("count")
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class ChansHeatmap(MixtureComponentPlot):
|
|
216
|
+
kind = "small"
|
|
217
|
+
height = 1.5
|
|
218
|
+
|
|
219
|
+
def __init__(self, cmap="magma", snr_cmap="viridis"):
|
|
220
|
+
self.cmap = plt.get_cmap(cmap)
|
|
221
|
+
self.snr_cmap = plt.get_cmap(snr_cmap)
|
|
222
|
+
|
|
223
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
224
|
+
inu_train, chans = mix_data.train_inds_and_chans(unit_id)
|
|
225
|
+
chans = chans[chans < mix_data.tmm.neighb_cov.n_channels]
|
|
226
|
+
uchans, counts = np.unique(chans, return_counts=True)
|
|
227
|
+
|
|
228
|
+
mean = mix_data.tmm.b.means[unit_id].view(
|
|
229
|
+
-1, mix_data.tmm.neighb_cov.n_channels
|
|
230
|
+
)
|
|
231
|
+
ptp = spiketorch.ptp(mean, dim=0).cpu()
|
|
232
|
+
(support,) = ptp[uchans].nonzero(as_tuple=True)
|
|
233
|
+
support = uchans[support]
|
|
234
|
+
ptp = ptp[support]
|
|
235
|
+
|
|
236
|
+
ax = panel.subplots()
|
|
237
|
+
xy = mix_data.tmm.neighb_cov.prgeom.numpy(force=True)
|
|
238
|
+
ax.scatter(*xy[support].T, c=ptp, lw=2, s=5)
|
|
239
|
+
s = ax.scatter(*xy[uchans].T, c=counts, lw=0, cmap=self.cmap, s=5)
|
|
240
|
+
plt.colorbar(s, ax=ax, shrink=0.3, label="chan count", pad=0.01)
|
|
241
|
+
ax.scatter(*xy[support[ptp.argmax()]].T, color="gold", lw=1, fc="none", s=5)
|
|
242
|
+
ax.set_title(
|
|
243
|
+
f"chans:{chans.min().item()}-{chans.max().item()}", fontsize="small"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class LikelihoodsView(MixtureComponentPlot):
|
|
248
|
+
def __init__(self, viol_ms=1.0, layout="vert"):
|
|
249
|
+
self.viol_ms = viol_ms
|
|
250
|
+
self.layout = layout
|
|
251
|
+
if layout == "vert":
|
|
252
|
+
self.kind = "small"
|
|
253
|
+
self.width = 1
|
|
254
|
+
self.height = 4
|
|
255
|
+
self.ncols = 1
|
|
256
|
+
self.nrows = 3
|
|
257
|
+
self.width_ratios = [1]
|
|
258
|
+
elif layout == "horz":
|
|
259
|
+
self.kind = "block"
|
|
260
|
+
self.width = 2
|
|
261
|
+
self.height = 1
|
|
262
|
+
self.ncols = 3
|
|
263
|
+
self.nrows = 1
|
|
264
|
+
self.width_ratios = [3, 2, 1]
|
|
265
|
+
else:
|
|
266
|
+
assert False
|
|
267
|
+
|
|
268
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
269
|
+
inu, t = mix_data.inu_and_times_full(unit_id)
|
|
270
|
+
my_ll = mix_data.full_scores.log_liks[:, 0][inu]
|
|
271
|
+
noise_ll = mix_data.full_scores.log_liks[:, -1][inu]
|
|
272
|
+
dt_ms = np.diff(t) * 1000
|
|
273
|
+
viol = dt_ms <= self.viol_ms
|
|
274
|
+
viol = np.logical_or(
|
|
275
|
+
np.pad(viol, (1, 0), constant_values=False),
|
|
276
|
+
np.pad(viol, (0, 1), constant_values=False),
|
|
277
|
+
)
|
|
278
|
+
viol = np.flatnonzero(viol)
|
|
279
|
+
my_min = my_ll.min()
|
|
280
|
+
my_max = my_ll.max()
|
|
281
|
+
noise_min = noise_ll.min()
|
|
282
|
+
noise_max = noise_ll.max()
|
|
283
|
+
mn = min(my_min, noise_min)
|
|
284
|
+
mx = max(my_max, noise_max)
|
|
285
|
+
assert math.isfinite(mn)
|
|
286
|
+
return my_ll, noise_ll, t, viol, mn, mx
|
|
287
|
+
|
|
288
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
289
|
+
my_ll, noise_ll, t, viol, mn, mx = self.compute(mix_data, unit_id)
|
|
290
|
+
|
|
291
|
+
ax_time, ax_noise, ax_hist = panel.subplots(
|
|
292
|
+
ncols=self.ncols,
|
|
293
|
+
nrows=self.nrows,
|
|
294
|
+
width_ratios=self.width_ratios,
|
|
295
|
+
sharey=True,
|
|
296
|
+
)
|
|
297
|
+
ax_time.set_ylabel("log likelihood")
|
|
298
|
+
ax_time.set_xlabel("time (s)")
|
|
299
|
+
ax_noise.set_xlabel("noise loglik")
|
|
300
|
+
ax_hist.set_xlabel("count")
|
|
301
|
+
c = glasbey1024[unit_id]
|
|
302
|
+
|
|
303
|
+
ax_time.scatter(t, noise_ll, s=3, lw=0, color="darkgray")
|
|
304
|
+
ax_time.scatter(t, my_ll, s=3, lw=0, color=c)
|
|
305
|
+
ax_time.scatter(t[viol], my_ll[viol], s=3, lw=0, color="k")
|
|
306
|
+
|
|
307
|
+
ax_noise.scatter(noise_ll, my_ll, s=3, lw=0, color=c, zorder=1)
|
|
308
|
+
ax_noise.plot([mn, mx], [mn, mx], color="k", lw=0.8, zorder=11)
|
|
309
|
+
|
|
310
|
+
histk = dict(histtype="step", orientation="horizontal")
|
|
311
|
+
_, bins, _ = ax_hist.hist(my_ll, color=c, label="unit", bins=64, **histk)
|
|
312
|
+
ax_hist.hist(noise_ll, color="darkgray", label="noise", bins=bins, **histk)
|
|
313
|
+
ax_hist.legend(
|
|
314
|
+
loc="lower right",
|
|
315
|
+
frameon=False,
|
|
316
|
+
borderpad=0.1,
|
|
317
|
+
borderaxespad=0.1,
|
|
318
|
+
handletextpad=0.3,
|
|
319
|
+
handlelength=1.0,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class NeighborMeans(MixtureComponentPlot):
|
|
324
|
+
kind = "block"
|
|
325
|
+
width = 2
|
|
326
|
+
height = 2
|
|
327
|
+
|
|
328
|
+
def __init__(self, count=5, vis_radius=50.0):
|
|
329
|
+
self.count = count
|
|
330
|
+
self.vis_radius = vis_radius
|
|
331
|
+
|
|
332
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
333
|
+
dists, neighbors = mix_data.friends(unit_id, count=self.count)
|
|
334
|
+
k = len(neighbors)
|
|
335
|
+
chans = mix_data.chans_in_radius(unit_id, self.vis_radius)
|
|
336
|
+
means = mix_data.tmm.b.means[neighbors].view(
|
|
337
|
+
k, -1, mix_data.tmm.neighb_cov.n_channels
|
|
338
|
+
)
|
|
339
|
+
means = means[:, :, chans].view(k, -1)
|
|
340
|
+
means = mix_data.reconstruct_flat(means)
|
|
341
|
+
return chans, neighbors, means
|
|
342
|
+
|
|
343
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
344
|
+
chans, neighbors, means = self.compute(mix_data, unit_id)
|
|
345
|
+
colors = [glasbey1024[j] for j in neighbors]
|
|
346
|
+
ax = panel.subplots()
|
|
347
|
+
chans = chans[None].broadcast_to(len(means), *chans.shape)
|
|
348
|
+
geomplot(
|
|
349
|
+
waveforms=means,
|
|
350
|
+
channels=chans.numpy(force=True),
|
|
351
|
+
geom=mix_data.prgeom,
|
|
352
|
+
show_zero=False,
|
|
353
|
+
colors=colors,
|
|
354
|
+
zlim=None,
|
|
355
|
+
ax=ax,
|
|
356
|
+
subar=True,
|
|
357
|
+
annotate_z=True,
|
|
358
|
+
)
|
|
359
|
+
panel.legend(
|
|
360
|
+
handles=[Line2D([0, 1], [0, 0], color=c) for c in colors],
|
|
361
|
+
labels=list(neighbors),
|
|
362
|
+
loc="outside upper center",
|
|
363
|
+
frameon=False,
|
|
364
|
+
ncols=min(5, len(neighbors)),
|
|
365
|
+
fontsize="small",
|
|
366
|
+
borderpad=0,
|
|
367
|
+
labelspacing=0.25,
|
|
368
|
+
handlelength=1.0,
|
|
369
|
+
handletextpad=0.4,
|
|
370
|
+
borderaxespad=0.0,
|
|
371
|
+
columnspacing=1.0,
|
|
372
|
+
)
|
|
373
|
+
ax.axis("off")
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
class NeighborDistances(MixtureComponentPlot):
|
|
377
|
+
kind = "block"
|
|
378
|
+
width = 2
|
|
379
|
+
height = 2
|
|
380
|
+
|
|
381
|
+
def __init__(self, count=5, cmap="plasma"):
|
|
382
|
+
self.count = count
|
|
383
|
+
self.cmap = plt.get_cmap(cmap)
|
|
384
|
+
|
|
385
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
386
|
+
dists, neighbors = mix_data.friends(unit_id, count=self.count)
|
|
387
|
+
d = mix_data.inf_diag_unit_distance_matrix[neighbors][:, neighbors]
|
|
388
|
+
d = d.numpy(force=True)
|
|
389
|
+
np.fill_diagonal(d, 0.0)
|
|
390
|
+
return d, neighbors
|
|
391
|
+
|
|
392
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
393
|
+
d, neighbors = self.compute(mix_data, unit_id)
|
|
394
|
+
distance_matrix_dendro(
|
|
395
|
+
panel,
|
|
396
|
+
d,
|
|
397
|
+
unit_ids=neighbors,
|
|
398
|
+
dendrogram_linkage=None,
|
|
399
|
+
show_unit_labels=True,
|
|
400
|
+
vmax=mix_data.tmm.max_distance,
|
|
401
|
+
image_cmap=self.cmap,
|
|
402
|
+
show_values=True,
|
|
403
|
+
)
|
|
404
|
+
panel.suptitle(f"{mix_data.tmm.distance_kind} dists", fontsize="small")
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class MergeView(MixtureComponentPlot):
|
|
408
|
+
kind = "block"
|
|
409
|
+
width = 2
|
|
410
|
+
height = 2.5
|
|
411
|
+
|
|
412
|
+
def __init__(self):
|
|
413
|
+
pass
|
|
414
|
+
|
|
415
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
416
|
+
# get pair mask
|
|
417
|
+
gsize = mix_data.tmm.max_group_size - 1
|
|
418
|
+
dists, neighbors = mix_data.friends(unit_id, count=gsize)
|
|
419
|
+
d = mix_data.inf_diag_unit_distance_matrix[neighbors][:, neighbors]
|
|
420
|
+
d.diagonal().fill_(0.0)
|
|
421
|
+
pair_mask = d < mix_data.tmm.merge_max_distance
|
|
422
|
+
|
|
423
|
+
# determine connected components and re-order (stably)
|
|
424
|
+
n_comps, labels = connected_components(pair_mask.numpy(force=True))
|
|
425
|
+
if n_comps > 1:
|
|
426
|
+
reorder = [np.flatnonzero(labels == l) for l in range(n_comps)]
|
|
427
|
+
reorder = np.concatenate(reorder)
|
|
428
|
+
d = d[reorder][:, reorder]
|
|
429
|
+
pair_mask = pair_mask[reorder][:, reorder]
|
|
430
|
+
neighbors = neighbors[reorder]
|
|
431
|
+
|
|
432
|
+
# run it
|
|
433
|
+
group_res = mix_data.tmm.try_merge_group(
|
|
434
|
+
group=torch.asarray(neighbors),
|
|
435
|
+
train_data=mix_data.train_data,
|
|
436
|
+
eval_data=mix_data.val_data,
|
|
437
|
+
scores=mix_data.eval_scores,
|
|
438
|
+
pair_mask=pair_mask,
|
|
439
|
+
apply_adj_mask=True,
|
|
440
|
+
debug=True,
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
return neighbors, d, pair_mask, group_res
|
|
444
|
+
|
|
445
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
446
|
+
neighbors, dist, pair_mask, group_res = self.compute(mix_data, unit_id)
|
|
447
|
+
panel_mask, panel_info = panel.subfigures(nrows=2, height_ratios=[2, 0.5])
|
|
448
|
+
distance_matrix_dendro(
|
|
449
|
+
panel_mask,
|
|
450
|
+
pair_mask,
|
|
451
|
+
show_values_from=dist,
|
|
452
|
+
unit_ids=neighbors,
|
|
453
|
+
dendrogram_linkage=None,
|
|
454
|
+
show_unit_labels=True,
|
|
455
|
+
image_cmap="RdGy_r",
|
|
456
|
+
show_values=True,
|
|
457
|
+
with_colorbar=False,
|
|
458
|
+
value_color="w",
|
|
459
|
+
)
|
|
460
|
+
ax_info = panel_info.subplots()
|
|
461
|
+
ax_info.axis("off")
|
|
462
|
+
if group_res is None:
|
|
463
|
+
msg = "bail"
|
|
464
|
+
else:
|
|
465
|
+
propstr = ",".join(f"{p:0.2f}" for p in group_res.sub_proportions.cpu())
|
|
466
|
+
msg = (
|
|
467
|
+
f"part: {group_res.grouping.group_ids.tolist()}\n"
|
|
468
|
+
f"imp: {group_res.improvement:.3f}\n"
|
|
469
|
+
f"props: {propstr}"
|
|
470
|
+
)
|
|
471
|
+
ax_info.text(
|
|
472
|
+
0.2, 0.5, msg, fontsize="small", va="center", transform=ax_info.transAxes
|
|
473
|
+
)
|
|
474
|
+
panel.suptitle("merge pair mask", fontsize="small")
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
class MeanView(MixtureComponentPlot):
|
|
478
|
+
def __init__(
|
|
479
|
+
self,
|
|
480
|
+
n_waveforms_show=128,
|
|
481
|
+
alpha=0.25,
|
|
482
|
+
show_zero=True,
|
|
483
|
+
mini=False,
|
|
484
|
+
mini_rad=41.0,
|
|
485
|
+
):
|
|
486
|
+
self.n_waveforms_show = n_waveforms_show
|
|
487
|
+
self.alpha = alpha
|
|
488
|
+
self.show_zero = show_zero
|
|
489
|
+
self.mini = mini
|
|
490
|
+
self.mini_rad = mini_rad
|
|
491
|
+
if mini:
|
|
492
|
+
self.kind = "tall"
|
|
493
|
+
self.width = 2.5
|
|
494
|
+
self.height = 2
|
|
495
|
+
else:
|
|
496
|
+
self.kind = "bigblock"
|
|
497
|
+
self.width = 3
|
|
498
|
+
self.height = 4
|
|
499
|
+
|
|
500
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
501
|
+
mean = mix_data.tmm.b.means[unit_id]
|
|
502
|
+
mean_recon = mix_data.reconstruct_flat(mean)
|
|
503
|
+
mean_recon = mean_recon.reshape(1, -1, mix_data.tmm.neighb_cov.n_channels)
|
|
504
|
+
inu_train, wchans, features, waveforms = mix_data.random_train_waveforms(
|
|
505
|
+
unit_id=unit_id, count=self.n_waveforms_show
|
|
506
|
+
)
|
|
507
|
+
if self.mini:
|
|
508
|
+
targchans = mix_data.chans_in_radius(unit_id, radius=self.mini_rad)
|
|
509
|
+
wchans = torch.asarray(wchans).to(targchans)
|
|
510
|
+
vchans = torch.isin(wchans, targchans)
|
|
511
|
+
vii, vcc = vchans.cpu().nonzero(as_tuple=True)
|
|
512
|
+
wchans = wchans[vii, vcc].cpu()
|
|
513
|
+
waveforms = waveforms[vii, :, vcc]
|
|
514
|
+
wchans = wchans[:, None]
|
|
515
|
+
assert wchans.ndim == 2
|
|
516
|
+
waveforms = waveforms[:, :, None]
|
|
517
|
+
assert waveforms.ndim == 3
|
|
518
|
+
maa = np.nanmax(np.abs(waveforms)).item()
|
|
519
|
+
return waveforms, wchans, mean_recon, maa
|
|
520
|
+
|
|
521
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
522
|
+
waveforms, wchans, mean_recon, maa = self.compute(mix_data, unit_id)
|
|
523
|
+
|
|
524
|
+
ax = panel.subplots()
|
|
525
|
+
ax.axis("off")
|
|
526
|
+
lines, pchans = geomplot( # type: ignore
|
|
527
|
+
waveforms=waveforms,
|
|
528
|
+
channels=wchans,
|
|
529
|
+
color="k",
|
|
530
|
+
alpha=self.alpha,
|
|
531
|
+
ax=ax,
|
|
532
|
+
max_abs_amp=maa,
|
|
533
|
+
geom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
|
|
534
|
+
show_zero=self.show_zero,
|
|
535
|
+
linewidth=0.5,
|
|
536
|
+
return_chans=True,
|
|
537
|
+
)
|
|
538
|
+
pchans = np.array(list(pchans))
|
|
539
|
+
geomplot(
|
|
540
|
+
waveforms=mean_recon[:, :, pchans],
|
|
541
|
+
channels=pchans[None],
|
|
542
|
+
max_abs_amp=maa,
|
|
543
|
+
geom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
|
|
544
|
+
color=glasbey1024[unit_id],
|
|
545
|
+
ax=ax,
|
|
546
|
+
show_zero=False,
|
|
547
|
+
annotate_z=True,
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
class CovarianceView(MixtureComponentPlot):
|
|
552
|
+
kind = "bigblock"
|
|
553
|
+
width = 3
|
|
554
|
+
height = 3
|
|
555
|
+
|
|
556
|
+
def __init__(
|
|
557
|
+
self, n_waveforms_show=128, neigs=16, cov_vert=False, reference_cov="interp"
|
|
558
|
+
):
|
|
559
|
+
self.n_waveforms_show = n_waveforms_show
|
|
560
|
+
self.colors = dict(emp="k", interp="gray", noise="r", signal="g", model="b")
|
|
561
|
+
self.neigs = neigs
|
|
562
|
+
self.cov_vert = cov_vert
|
|
563
|
+
self.reference_cov = reference_cov
|
|
564
|
+
|
|
565
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
566
|
+
# load data
|
|
567
|
+
inu_train, wchans, features, waveforms = mix_data.random_train_waveforms(
|
|
568
|
+
unit_id=unit_id, count=self.n_waveforms_show
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
# pick channels and put data there
|
|
572
|
+
chan_set = torch.asarray(wchans).unique().to(device=mix_data.tmm.b.means.device)
|
|
573
|
+
chan_set = chan_set[chan_set < mix_data.tmm.neighb_cov.n_channels]
|
|
574
|
+
features = features.reshape(
|
|
575
|
+
features.shape[0], -1, mix_data.tmm.neighb_cov.max_nc_obs
|
|
576
|
+
)
|
|
577
|
+
features = torch.asarray(features)
|
|
578
|
+
# -- with nans
|
|
579
|
+
feat_nan = features.new_full(
|
|
580
|
+
(*features.shape[:2], mix_data.tmm.neighb_cov.n_channels + 1), torch.nan
|
|
581
|
+
)
|
|
582
|
+
ix = torch.asarray(wchans)
|
|
583
|
+
ix = ix[:, None, :].broadcast_to(features.shape)
|
|
584
|
+
feat_nan.scatter_(
|
|
585
|
+
dim=2,
|
|
586
|
+
index=ix,
|
|
587
|
+
src=features,
|
|
588
|
+
)
|
|
589
|
+
feat_nan = feat_nan[:, :, chan_set.cpu()]
|
|
590
|
+
feat_nan = feat_nan.view(feat_nan.shape[0], -1)
|
|
591
|
+
# -- with interp
|
|
592
|
+
feat_interp = mix_data.tmm.erp.interp_to_chans(
|
|
593
|
+
waveforms=features.to(mix_data.tmm.b.means),
|
|
594
|
+
neighborhood_ids=mix_data.train_data.neighborhood_ids[inu_train],
|
|
595
|
+
target_channels=chan_set,
|
|
596
|
+
)
|
|
597
|
+
feat_interp = feat_interp.view(feat_interp.shape[0], -1)
|
|
598
|
+
|
|
599
|
+
# empirical covariance
|
|
600
|
+
mean_nan = torch.nanmean(feat_nan, dim=0)
|
|
601
|
+
feat_nan -= mean_nan
|
|
602
|
+
cov_nan = cast(torch.Tensor, spiketorch.nancov(feat_nan, correction=0))
|
|
603
|
+
|
|
604
|
+
# interp covariance
|
|
605
|
+
cov_interp = torch.cov(feat_interp.T)
|
|
606
|
+
|
|
607
|
+
# noise covariance
|
|
608
|
+
cov_noise = mix_data.tmm.noise.marginal_covariance(channels=chan_set).to_dense()
|
|
609
|
+
assert cov_noise.shape == cov_interp.shape
|
|
610
|
+
|
|
611
|
+
# signal covariance
|
|
612
|
+
if mix_data.tmm.signal_rank:
|
|
613
|
+
basis = mix_data.tmm.b.bases[unit_id]
|
|
614
|
+
basis = basis.view(
|
|
615
|
+
mix_data.tmm.signal_rank, -1, mix_data.tmm.neighb_cov.n_channels
|
|
616
|
+
)
|
|
617
|
+
basis = basis[:, :, chan_set].view(mix_data.tmm.signal_rank, -1)
|
|
618
|
+
basis = basis * mix_data.tmm.latent_prior_std
|
|
619
|
+
cov_signal = basis.T @ basis
|
|
620
|
+
assert cov_signal.shape == cov_noise.shape
|
|
621
|
+
else:
|
|
622
|
+
cov_signal = torch.zeros_like(cov_noise)
|
|
623
|
+
|
|
624
|
+
# model covariance
|
|
625
|
+
cov_model = cov_noise + cov_signal
|
|
626
|
+
|
|
627
|
+
# np-ify
|
|
628
|
+
cov_nan = cov_nan.numpy(force=True)
|
|
629
|
+
cov_interp = cov_interp.numpy(force=True)
|
|
630
|
+
cov_noise = cov_noise.numpy(force=True)
|
|
631
|
+
cov_signal = cov_signal.numpy(force=True)
|
|
632
|
+
cov_model = cov_model.numpy(force=True)
|
|
633
|
+
|
|
634
|
+
# spectra: empirical, noise, sginal, model
|
|
635
|
+
ev_nan = np.linalg.eigvalsh(cov_nan)[::-1]
|
|
636
|
+
ev_interp = np.linalg.eigvalsh(cov_interp)[::-1]
|
|
637
|
+
ev_noise = np.linalg.eigvalsh(cov_noise)[::-1]
|
|
638
|
+
ev_signal = np.linalg.eigvalsh(cov_signal)[::-1]
|
|
639
|
+
ev_model = np.linalg.eigvalsh(cov_model)[::-1]
|
|
640
|
+
|
|
641
|
+
return {
|
|
642
|
+
"emp": (cov_nan, ev_nan),
|
|
643
|
+
"interp": (cov_interp, ev_interp),
|
|
644
|
+
"noise": (cov_noise, ev_noise),
|
|
645
|
+
"signal": (cov_signal, ev_signal),
|
|
646
|
+
"model": (cov_model, ev_model),
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
650
|
+
stats = self.compute(mix_data, unit_id)
|
|
651
|
+
cov_emp = stats[self.reference_cov][0]
|
|
652
|
+
vm = np.percentile(np.abs(cov_emp), 98)
|
|
653
|
+
|
|
654
|
+
toph = 2 + int(self.cov_vert)
|
|
655
|
+
top, bottom = panel.subfigures(nrows=2, height_ratios=[toph, 1])
|
|
656
|
+
if self.cov_vert:
|
|
657
|
+
axes_top = top.subplots(ncols=2, nrows=len(stats))
|
|
658
|
+
else:
|
|
659
|
+
axes_top = top.subplots(ncols=len(stats), nrows=2)
|
|
660
|
+
axes_top = axes_top.T
|
|
661
|
+
ax_bottom = bottom.subplots()
|
|
662
|
+
|
|
663
|
+
cov_kw = dict(
|
|
664
|
+
aspect=1.0, interpolation="none", vmin=-vm, vmax=vm, cmap="seismic"
|
|
665
|
+
)
|
|
666
|
+
res_kw = cov_kw | dict(vmin=-vm, vmax=vm)
|
|
667
|
+
|
|
668
|
+
for row_top, (name, (cov, ev)) in zip(axes_top, stats.items()):
|
|
669
|
+
c = self.colors[name]
|
|
670
|
+
if self.cov_vert:
|
|
671
|
+
row_top[0].set_ylabel(name, color=c)
|
|
672
|
+
else:
|
|
673
|
+
row_top[0].set_title(name, color=c, fontsize=6)
|
|
674
|
+
ima = row_top[0].imshow(cov, **cov_kw)
|
|
675
|
+
imb = row_top[1].imshow(cov_emp - cov, **res_kw)
|
|
676
|
+
ax_bottom.plot(ev[: self.neigs], color=c, label=name)
|
|
677
|
+
if not self.cov_vert:
|
|
678
|
+
plt.colorbar(ima, ax=row_top[0], shrink=0.3) # type: ignore
|
|
679
|
+
plt.colorbar(imb, ax=row_top[1], shrink=0.3) # type: ignore
|
|
680
|
+
|
|
681
|
+
if self.cov_vert:
|
|
682
|
+
axes_top[0, 0].set_title("cov", fontsize="small")
|
|
683
|
+
axes_top[0, 1].set_title(
|
|
684
|
+
f"{self.reference_cov} - cov",
|
|
685
|
+
fontsize="small",
|
|
686
|
+
color=self.colors[self.reference_cov],
|
|
687
|
+
)
|
|
688
|
+
else:
|
|
689
|
+
axes_top[0, 0].set_ylabel("cov", fontsize="small")
|
|
690
|
+
axes_top[0, 1].set_ylabel(
|
|
691
|
+
f"{self.reference_cov} - cov",
|
|
692
|
+
fontsize="small",
|
|
693
|
+
color=self.colors[self.reference_cov],
|
|
694
|
+
)
|
|
695
|
+
for ax in axes_top.flat:
|
|
696
|
+
ax.set_xticks([])
|
|
697
|
+
ax.set_yticks([])
|
|
698
|
+
ax_bottom.grid(which="both")
|
|
699
|
+
ax_bottom.legend(
|
|
700
|
+
borderpad=0.1,
|
|
701
|
+
borderaxespad=0.1,
|
|
702
|
+
handletextpad=0.3,
|
|
703
|
+
handlelength=1.0,
|
|
704
|
+
loc="upper right",
|
|
705
|
+
ncols=2,
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
class SplitView(MixtureComponentPlot):
|
|
710
|
+
kind = "tall"
|
|
711
|
+
width = 2.5
|
|
712
|
+
height = 5
|
|
713
|
+
|
|
714
|
+
def __init__(
|
|
715
|
+
self,
|
|
716
|
+
colors=["r", "g", "b", "darkorange", "darkviolet", "mediumturquoise"],
|
|
717
|
+
bail_color="k",
|
|
718
|
+
vis_radius=50.0,
|
|
719
|
+
dist_cmap="plasma",
|
|
720
|
+
):
|
|
721
|
+
self.colors = np.array(colors)
|
|
722
|
+
self.bail_color = bail_color
|
|
723
|
+
self.vis_radius = vis_radius
|
|
724
|
+
self.dist_cmap = plt.get_cmap(dist_cmap)
|
|
725
|
+
|
|
726
|
+
def compute(self, mix_data: MixtureVisData, unit_id: int):
|
|
727
|
+
split_res, debug_info = mix_data.tmm.split_unit(
|
|
728
|
+
unit_id=unit_id,
|
|
729
|
+
train_data=mix_data.train_data,
|
|
730
|
+
eval_data=mix_data.val_data,
|
|
731
|
+
scores=mix_data.eval_scores,
|
|
732
|
+
debug=True,
|
|
733
|
+
)
|
|
734
|
+
assert debug_info is not None
|
|
735
|
+
|
|
736
|
+
if debug_info.split_data is not None:
|
|
737
|
+
n_spikes = debug_info.split_data.x.shape[0]
|
|
738
|
+
else:
|
|
739
|
+
n_spikes = 0
|
|
740
|
+
|
|
741
|
+
# compute colors by kmeans label
|
|
742
|
+
if debug_info.kmeans_responsibilities is not None:
|
|
743
|
+
assert debug_info.kmeans_responsibilities.shape[0] == n_spikes
|
|
744
|
+
kmeans_labels = debug_info.kmeans_responsibilities.argmax(dim=1)
|
|
745
|
+
kmeans_labels = kmeans_labels.cpu()
|
|
746
|
+
colors = self.colors[kmeans_labels]
|
|
747
|
+
else:
|
|
748
|
+
kmeans_labels = torch.zeros(n_spikes, dtype=torch.long)
|
|
749
|
+
colors = np.broadcast_to(np.array([self.bail_color]), (n_spikes,))
|
|
750
|
+
|
|
751
|
+
# compute channels by kmeans label
|
|
752
|
+
if debug_info.kmeans_responsibilities is not None:
|
|
753
|
+
assert debug_info.split_data is not None
|
|
754
|
+
chans_by_km = {}
|
|
755
|
+
for l in kmeans_labels.unique():
|
|
756
|
+
neighbs_l = debug_info.split_data.neighborhood_ids[kmeans_labels == l]
|
|
757
|
+
chans_l = mix_data.tmm.neighb_cov.obs_ix[neighbs_l]
|
|
758
|
+
chans_l = chans_l[chans_l < mix_data.tmm.neighb_cov.n_channels]
|
|
759
|
+
chans_by_km[l.item()] = chans_l.numpy(force=True)
|
|
760
|
+
else:
|
|
761
|
+
chans_by_km = None
|
|
762
|
+
|
|
763
|
+
# compute amplitudes
|
|
764
|
+
if debug_info.split_data is not None:
|
|
765
|
+
x = debug_info.split_data.x.view(
|
|
766
|
+
n_spikes, -1, mix_data.tmm.neighb_cov.max_nc_obs
|
|
767
|
+
)
|
|
768
|
+
t = mix_data.train_times[debug_info.split_data.indices.cpu()]
|
|
769
|
+
amps = x.square().sum(dim=1).amax(dim=1).sqrt_().numpy(force=True)
|
|
770
|
+
else:
|
|
771
|
+
t = amps = None
|
|
772
|
+
|
|
773
|
+
# compute pca embedding of kmeans_x
|
|
774
|
+
# TODO: also embed split_model means and maybe final model?
|
|
775
|
+
if debug_info.kmeans_x is not None:
|
|
776
|
+
assert debug_info.kmeans_x.shape[0] == n_spikes
|
|
777
|
+
assert debug_info.kmeans_chans is not None
|
|
778
|
+
kmean = debug_info.kmeans_x.mean(0)
|
|
779
|
+
loadings, components, *_ = spiketorch.svd_lowrank_helper(
|
|
780
|
+
x=debug_info.kmeans_x,
|
|
781
|
+
rank=2,
|
|
782
|
+
with_loadings=True,
|
|
783
|
+
M=kmean[None].broadcast_to(debug_info.kmeans_x.shape),
|
|
784
|
+
)
|
|
785
|
+
assert loadings is not None
|
|
786
|
+
loadings = loadings.numpy(force=True)
|
|
787
|
+
else:
|
|
788
|
+
loadings = None
|
|
789
|
+
|
|
790
|
+
# distances
|
|
791
|
+
if debug_info.split_model is not None:
|
|
792
|
+
dists = debug_info.split_model.unit_distance_matrix()
|
|
793
|
+
dists = dists.numpy(force=True)
|
|
794
|
+
else:
|
|
795
|
+
dists = None
|
|
796
|
+
|
|
797
|
+
# mean reconstructions near my main channel
|
|
798
|
+
mean_chans = mix_data.chans_in_radius(unit_id, radius=self.vis_radius)
|
|
799
|
+
if debug_info.split_model is not None:
|
|
800
|
+
means = debug_info.split_model.b.means
|
|
801
|
+
means = means.view(means.shape[0], -1, mix_data.tmm.neighb_cov.n_channels)
|
|
802
|
+
means = means[:, :, mean_chans]
|
|
803
|
+
means = means.view(means.shape[0], -1)
|
|
804
|
+
means = mix_data.reconstruct_flat(means)
|
|
805
|
+
else:
|
|
806
|
+
means = None
|
|
807
|
+
|
|
808
|
+
# compute text info
|
|
809
|
+
txt = ""
|
|
810
|
+
if debug_info.bailed:
|
|
811
|
+
txt += f"bail: {debug_info.bail_reason}\n"
|
|
812
|
+
if debug_info.merge_res is not None:
|
|
813
|
+
txt += (
|
|
814
|
+
f"part: {debug_info.merge_res.grouping.group_ids.tolist()}\n"
|
|
815
|
+
f"imp: {debug_info.merge_res.improvement:.3f}\n"
|
|
816
|
+
)
|
|
817
|
+
else:
|
|
818
|
+
txt += f"no allowed partitions\n"
|
|
819
|
+
if debug_info.split_model is not None:
|
|
820
|
+
sprops = debug_info.split_model.b.log_proportions.cpu()
|
|
821
|
+
sprops = sprops - mix_data.tmm.b.log_proportions[unit_id].cpu()
|
|
822
|
+
propstr = "+".join(f"{p:0.3f}" for p in sprops.exp())
|
|
823
|
+
txt += f"norm kmeans model props:\n {propstr}={sprops.exp().sum():.2f}\n"
|
|
824
|
+
if debug_info.merge_res is not None:
|
|
825
|
+
mresprops = debug_info.merge_res.sub_proportions
|
|
826
|
+
propstr = "+".join(f"{p:0.3f}" for p in mresprops)
|
|
827
|
+
txt += f"mres props: {propstr}={mresprops.sum():.2f}\n"
|
|
828
|
+
if debug_info.kmeans_responsibilities is not None:
|
|
829
|
+
kcountstr = ",".join(
|
|
830
|
+
str(p.item()) for p in kmeans_labels.unique(return_counts=True)[1]
|
|
831
|
+
)
|
|
832
|
+
txt += f"kmeans counts: {kcountstr}\n"
|
|
833
|
+
if split_res is not None:
|
|
834
|
+
propstr = "+".join(f"{p:0.3f}" for p in split_res.sub_proportions.cpu())
|
|
835
|
+
countstr = ",".join(
|
|
836
|
+
str(p.item())
|
|
837
|
+
for p in split_res.train_assignments.unique(return_counts=True)[1]
|
|
838
|
+
)
|
|
839
|
+
txt += f"props: {propstr}={split_res.sub_proportions.cpu().sum():.2f}\n"
|
|
840
|
+
txt += f"counts: {countstr}\n"
|
|
841
|
+
txt = txt.rstrip()
|
|
842
|
+
|
|
843
|
+
return (
|
|
844
|
+
txt,
|
|
845
|
+
colors,
|
|
846
|
+
t,
|
|
847
|
+
amps,
|
|
848
|
+
loadings,
|
|
849
|
+
dists,
|
|
850
|
+
means,
|
|
851
|
+
mean_chans,
|
|
852
|
+
chans_by_km,
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
|
|
856
|
+
c = self.compute(mix_data, unit_id)
|
|
857
|
+
txt, colors, t, amps, loadings, dists, means, mean_chans, chans_by_km = c
|
|
858
|
+
|
|
859
|
+
# layout
|
|
860
|
+
amp_info_row, mean_row, dist_pc_row = panel.subfigures(
|
|
861
|
+
nrows=3, height_ratios=[1, 1, 1.5]
|
|
862
|
+
)
|
|
863
|
+
ax_amp, ax_info = amp_info_row.subplots(ncols=2)
|
|
864
|
+
ax_mean = mean_row.subplots()
|
|
865
|
+
fig_dist_chans, fig_pc = dist_pc_row.subfigures(ncols=2)
|
|
866
|
+
ax_pc = fig_pc.subplots()
|
|
867
|
+
fig_dist, fig_chans = fig_dist_chans.subfigures(nrows=2, height_ratios=[3, 2])
|
|
868
|
+
ax_chans = fig_chans.subplots()
|
|
869
|
+
|
|
870
|
+
# unit dist matrix / pair mask
|
|
871
|
+
if dists is not None:
|
|
872
|
+
dists_inf = dists.copy()
|
|
873
|
+
dists_inf[dists > mix_data.tmm.max_distance] = np.inf
|
|
874
|
+
distance_matrix_dendro(
|
|
875
|
+
panel=fig_dist,
|
|
876
|
+
distances=dists_inf,
|
|
877
|
+
show_values_from=dists,
|
|
878
|
+
show_unit_labels=True,
|
|
879
|
+
vmax=mix_data.tmm.max_distance,
|
|
880
|
+
image_cmap=self.dist_cmap,
|
|
881
|
+
show_values=True,
|
|
882
|
+
label_colors=self.colors,
|
|
883
|
+
with_colorbar=False,
|
|
884
|
+
)
|
|
885
|
+
else:
|
|
886
|
+
ax_dist = fig_dist.subplots()
|
|
887
|
+
ax_dist.axis("off")
|
|
888
|
+
ax_dist.text(0, 0, "no dists", fontsize=5)
|
|
889
|
+
|
|
890
|
+
if chans_by_km is not None:
|
|
891
|
+
low = min(c.min().item() for c in chans_by_km.values())
|
|
892
|
+
hi = max(c.max().item() for c in chans_by_km.values())
|
|
893
|
+
bins = np.arange(low - 0.5, hi + 0.6, step=1.0)
|
|
894
|
+
ax_chans.hist(
|
|
895
|
+
list(chans_by_km.values()),
|
|
896
|
+
stacked=True,
|
|
897
|
+
color=self.colors[list(chans_by_km.keys())],
|
|
898
|
+
bins=bins,
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
# merge res text info
|
|
902
|
+
ax_info.axis("off")
|
|
903
|
+
ax_info.text(
|
|
904
|
+
0.2,
|
|
905
|
+
0.5,
|
|
906
|
+
txt,
|
|
907
|
+
fontsize="small",
|
|
908
|
+
va="center",
|
|
909
|
+
transform=ax_info.transAxes,
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
# pc scatter
|
|
913
|
+
if loadings is not None:
|
|
914
|
+
ax_pc.scatter(*loadings.T, c=colors, s=5, lw=0, alpha=0.5)
|
|
915
|
+
ax_pc.grid()
|
|
916
|
+
else:
|
|
917
|
+
ax_pc.axis("off")
|
|
918
|
+
ax_pc.text(0, 0, "no kmeans pca", fontsize=5)
|
|
919
|
+
|
|
920
|
+
# recon mean view
|
|
921
|
+
ax_mean.axis("off")
|
|
922
|
+
if means is not None:
|
|
923
|
+
chans = mean_chans[None].broadcast_to(len(means), *mean_chans.shape)
|
|
924
|
+
geomplot(
|
|
925
|
+
waveforms=means,
|
|
926
|
+
channels=chans.numpy(force=True),
|
|
927
|
+
geom=mix_data.prgeom,
|
|
928
|
+
show_zero=False,
|
|
929
|
+
colors=self.colors[: len(means)],
|
|
930
|
+
zlim=None,
|
|
931
|
+
ax=ax_mean,
|
|
932
|
+
subar=True,
|
|
933
|
+
annotate_z=True,
|
|
934
|
+
)
|
|
935
|
+
else:
|
|
936
|
+
ax_mean.text(0, 0, "no subunit means", fontsize=5)
|
|
937
|
+
|
|
938
|
+
# amp view
|
|
939
|
+
if amps is not None:
|
|
940
|
+
ax_amp.scatter(t, amps, c=colors, s=5, lw=0, alpha=0.5)
|
|
941
|
+
ax_amp.grid()
|
|
942
|
+
ax_amp.set_xlabel("t (s)")
|
|
943
|
+
ax_amp.set_ylabel("feat maxchan norm")
|
|
944
|
+
else:
|
|
945
|
+
ax_amp.axis("off")
|
|
946
|
+
ax_amp.text(0, 0, "no amps", fontsize=5)
|
|
947
|
+
panel.suptitle("split result vis")
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def default_mixture_plots():
|
|
951
|
+
return (
|
|
952
|
+
TextInfo(),
|
|
953
|
+
ISIHistogram(),
|
|
954
|
+
ChansHeatmap(),
|
|
955
|
+
LikelihoodsView(),
|
|
956
|
+
NeighborMeans(),
|
|
957
|
+
NeighborDistances(),
|
|
958
|
+
MergeView(),
|
|
959
|
+
MeanView(),
|
|
960
|
+
CovarianceView(),
|
|
961
|
+
MeanView(mini=True),
|
|
962
|
+
SplitView(),
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def fit_mixture_and_visualize_all_components(
|
|
967
|
+
*,
|
|
968
|
+
sorting: DARTsortSorting,
|
|
969
|
+
motion_est,
|
|
970
|
+
refinement_cfg: RefinementConfig = default_refinement_cfg,
|
|
971
|
+
computation_cfg: ComputationConfig | None = None,
|
|
972
|
+
save_folder: str | Path,
|
|
973
|
+
plots=default_mixture_plots,
|
|
974
|
+
max_height=9,
|
|
975
|
+
figsize=(17, 10),
|
|
976
|
+
hspace=0.01,
|
|
977
|
+
dpi=200,
|
|
978
|
+
image_ext="png",
|
|
979
|
+
n_jobs=0,
|
|
980
|
+
show_progress=True,
|
|
981
|
+
overwrite=False,
|
|
982
|
+
unit_ids=None,
|
|
983
|
+
n_units=None,
|
|
984
|
+
seed=0,
|
|
985
|
+
**other_global_params,
|
|
986
|
+
):
|
|
987
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
988
|
+
save_folder = resolve_path(save_folder)
|
|
989
|
+
if unit_ids is None:
|
|
990
|
+
unit_ids = sorting.unit_ids
|
|
991
|
+
if n_units is not None and n_units < len(unit_ids):
|
|
992
|
+
rg = np.random.default_rng(seed)
|
|
993
|
+
unit_ids = rg.choice(unit_ids, size=n_units, replace=False)
|
|
994
|
+
if not overwrite and all_summaries_done(unit_ids, save_folder, ext=image_ext):
|
|
995
|
+
return
|
|
996
|
+
mix_data = fit_mixture_for_vis(
|
|
997
|
+
sorting=sorting,
|
|
998
|
+
motion_est=motion_est,
|
|
999
|
+
refinement_cfg=refinement_cfg,
|
|
1000
|
+
computation_cfg=computation_cfg,
|
|
1001
|
+
)
|
|
1002
|
+
return make_mixture_summaries(
|
|
1003
|
+
mix_data=mix_data,
|
|
1004
|
+
save_folder=save_folder,
|
|
1005
|
+
plots=plots,
|
|
1006
|
+
max_height=max_height,
|
|
1007
|
+
figsize=figsize,
|
|
1008
|
+
hspace=hspace,
|
|
1009
|
+
dpi=dpi,
|
|
1010
|
+
image_ext=image_ext,
|
|
1011
|
+
n_jobs=n_jobs,
|
|
1012
|
+
show_progress=show_progress,
|
|
1013
|
+
overwrite=overwrite,
|
|
1014
|
+
unit_ids=unit_ids,
|
|
1015
|
+
n_units=n_units,
|
|
1016
|
+
seed=seed,
|
|
1017
|
+
**other_global_params,
|
|
1018
|
+
)
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def fit_mixture_for_vis(
|
|
1022
|
+
*,
|
|
1023
|
+
sorting: DARTsortSorting,
|
|
1024
|
+
motion_est,
|
|
1025
|
+
refinement_cfg: RefinementConfig = default_refinement_cfg,
|
|
1026
|
+
computation_cfg: ComputationConfig | None = None,
|
|
1027
|
+
em: bool = True,
|
|
1028
|
+
split: bool = False,
|
|
1029
|
+
) -> MixtureVisData:
|
|
1030
|
+
# run model to convergence and soft assign train/full sets
|
|
1031
|
+
mix_data = instantiate_and_bootstrap_tmm(
|
|
1032
|
+
sorting=sorting,
|
|
1033
|
+
motion_est=motion_est,
|
|
1034
|
+
refinement_cfg=refinement_cfg,
|
|
1035
|
+
computation_cfg=computation_cfg,
|
|
1036
|
+
)
|
|
1037
|
+
if em:
|
|
1038
|
+
mix_data.tmm.em(mix_data.train_data)
|
|
1039
|
+
if split:
|
|
1040
|
+
run_split(mix_data.tmm, mix_data.train_data, mix_data.val_data, prog_level=1)
|
|
1041
|
+
mix_data.tmm.em(mix_data.train_data)
|
|
1042
|
+
|
|
1043
|
+
train_scores = mix_data.tmm.soft_assign(
|
|
1044
|
+
data=mix_data.train_data,
|
|
1045
|
+
needs_bootstrap=False,
|
|
1046
|
+
full_proposal_view=True,
|
|
1047
|
+
)
|
|
1048
|
+
train_labels = labels_from_scores(train_scores)
|
|
1049
|
+
full_scores = mix_data.tmm.soft_assign(
|
|
1050
|
+
data=mix_data.full_data,
|
|
1051
|
+
needs_bootstrap=False,
|
|
1052
|
+
full_proposal_view=True,
|
|
1053
|
+
)
|
|
1054
|
+
full_labels = labels_from_scores(full_scores)
|
|
1055
|
+
if mix_data.val_data is None:
|
|
1056
|
+
eval_scores = train_scores
|
|
1057
|
+
else:
|
|
1058
|
+
eval_scores = mix_data.tmm.soft_assign(
|
|
1059
|
+
data=mix_data.val_data,
|
|
1060
|
+
needs_bootstrap=False,
|
|
1061
|
+
full_proposal_view=True,
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
dists = mix_data.tmm.unit_distance_matrix().cpu().clone()
|
|
1065
|
+
dists.diagonal().fill_(torch.inf)
|
|
1066
|
+
|
|
1067
|
+
tpca = get_tpca(sorting)
|
|
1068
|
+
assert tpca is not None
|
|
1069
|
+
assert isinstance(tpca, TemporalPCA)
|
|
1070
|
+
if isinstance(mix_data.train_ixs, slice):
|
|
1071
|
+
assert mix_data.train_ixs == slice(None)
|
|
1072
|
+
assert train_labels.shape == full_labels.shape
|
|
1073
|
+
train_ixs = np.arange(train_labels.shape[0])
|
|
1074
|
+
else:
|
|
1075
|
+
train_ixs = mix_data.train_ixs.numpy(force=True)
|
|
1076
|
+
|
|
1077
|
+
return MixtureVisData(
|
|
1078
|
+
tmm=mix_data.tmm,
|
|
1079
|
+
train_data=mix_data.train_data,
|
|
1080
|
+
val_data=mix_data.val_data,
|
|
1081
|
+
full_data=mix_data.full_data,
|
|
1082
|
+
sorting=sorting,
|
|
1083
|
+
motion_est=motion_est,
|
|
1084
|
+
train_scores=train_scores,
|
|
1085
|
+
full_scores=full_scores,
|
|
1086
|
+
eval_scores=eval_scores,
|
|
1087
|
+
train_times=sorting.times_seconds[mix_data.train_ixs], # type: ignore
|
|
1088
|
+
train_ixs=train_ixs,
|
|
1089
|
+
train_labels=train_labels,
|
|
1090
|
+
full_labels=full_labels,
|
|
1091
|
+
tpca=tpca,
|
|
1092
|
+
inf_diag_unit_distance_matrix=dists,
|
|
1093
|
+
prgeom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
|
|
1094
|
+
)
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def make_mixture_component_summary(
|
|
1098
|
+
mix_data: MixtureVisData,
|
|
1099
|
+
unit_id: int,
|
|
1100
|
+
plots=default_mixture_plots,
|
|
1101
|
+
max_height=9,
|
|
1102
|
+
figsize=(17, 10),
|
|
1103
|
+
card_hspace=0.01,
|
|
1104
|
+
figure=None,
|
|
1105
|
+
**other_global_params,
|
|
1106
|
+
):
|
|
1107
|
+
if callable(plots):
|
|
1108
|
+
plots = plots()
|
|
1109
|
+
plots = cast(Iterable[MixtureComponentPlot], plots)
|
|
1110
|
+
for p in plots:
|
|
1111
|
+
p.notify_global_params(**other_global_params)
|
|
1112
|
+
figure = flow_layout(
|
|
1113
|
+
plots,
|
|
1114
|
+
max_height=max_height,
|
|
1115
|
+
figsize=figsize,
|
|
1116
|
+
card_hspace=card_hspace,
|
|
1117
|
+
figure=figure,
|
|
1118
|
+
mix_data=mix_data,
|
|
1119
|
+
unit_id=unit_id,
|
|
1120
|
+
)
|
|
1121
|
+
return figure
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def make_mixture_summaries(
|
|
1125
|
+
mix_data: MixtureVisData,
|
|
1126
|
+
save_folder: str | Path,
|
|
1127
|
+
plots=default_mixture_plots,
|
|
1128
|
+
max_height=9,
|
|
1129
|
+
figsize=(17, 10),
|
|
1130
|
+
card_hspace=0.01,
|
|
1131
|
+
dpi=200,
|
|
1132
|
+
image_ext="png",
|
|
1133
|
+
n_jobs=0,
|
|
1134
|
+
show_progress=True,
|
|
1135
|
+
overwrite=False,
|
|
1136
|
+
unit_ids=None,
|
|
1137
|
+
n_units=None,
|
|
1138
|
+
seed=0,
|
|
1139
|
+
**other_global_params,
|
|
1140
|
+
):
|
|
1141
|
+
save_folder = resolve_path(save_folder)
|
|
1142
|
+
if unit_ids is None:
|
|
1143
|
+
unit_ids = mix_data.tmm.unit_ids.numpy(force=True).tolist()
|
|
1144
|
+
if n_units is not None and n_units < len(unit_ids):
|
|
1145
|
+
rg = np.random.default_rng(seed)
|
|
1146
|
+
unit_ids = rg.choice(unit_ids, size=n_units, replace=False)
|
|
1147
|
+
if not overwrite and all_summaries_done(unit_ids, save_folder, ext=image_ext):
|
|
1148
|
+
return
|
|
1149
|
+
|
|
1150
|
+
save_folder.mkdir(exist_ok=True, parents=True)
|
|
1151
|
+
global_params = dict(**other_global_params)
|
|
1152
|
+
n_jobs, Executor, context = get_pool(n_jobs, cls=CloudpicklePoolExecutor) # type: ignore
|
|
1153
|
+
|
|
1154
|
+
initargs = (
|
|
1155
|
+
mix_data,
|
|
1156
|
+
plots,
|
|
1157
|
+
max_height,
|
|
1158
|
+
figsize,
|
|
1159
|
+
card_hspace,
|
|
1160
|
+
dpi,
|
|
1161
|
+
save_folder,
|
|
1162
|
+
image_ext,
|
|
1163
|
+
overwrite,
|
|
1164
|
+
global_params,
|
|
1165
|
+
plt.rcParams,
|
|
1166
|
+
)
|
|
1167
|
+
# if ispar and not use_threads:
|
|
1168
|
+
# initargs = (cloudpickle.dumps(initargs),)
|
|
1169
|
+
with Executor(
|
|
1170
|
+
max_workers=n_jobs,
|
|
1171
|
+
mp_context=context,
|
|
1172
|
+
initializer=_summary_init,
|
|
1173
|
+
initargs=initargs,
|
|
1174
|
+
) as pool:
|
|
1175
|
+
results = pool.map(_summary_job, unit_ids)
|
|
1176
|
+
if show_progress:
|
|
1177
|
+
results = tqdm(
|
|
1178
|
+
results,
|
|
1179
|
+
desc="GMM summaries",
|
|
1180
|
+
smoothing=0,
|
|
1181
|
+
total=len(unit_ids),
|
|
1182
|
+
)
|
|
1183
|
+
for res in results:
|
|
1184
|
+
pass
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def all_summaries_done(unit_ids, save_folder, ext="png"):
|
|
1188
|
+
return save_folder.exists() and all(
|
|
1189
|
+
(save_folder / f"unit{unit_id:04d}.{ext}").exists() for unit_id in unit_ids
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
class SummaryJobContext:
|
|
1194
|
+
def __init__(
|
|
1195
|
+
self,
|
|
1196
|
+
mix_data,
|
|
1197
|
+
plots,
|
|
1198
|
+
max_height,
|
|
1199
|
+
figsize,
|
|
1200
|
+
card_hspace,
|
|
1201
|
+
dpi,
|
|
1202
|
+
save_folder,
|
|
1203
|
+
image_ext,
|
|
1204
|
+
overwrite,
|
|
1205
|
+
global_params,
|
|
1206
|
+
rc,
|
|
1207
|
+
):
|
|
1208
|
+
self.mix_data = mix_data
|
|
1209
|
+
self.plots = plots
|
|
1210
|
+
self.max_height = max_height
|
|
1211
|
+
self.figsize = figsize
|
|
1212
|
+
self.card_hspace = card_hspace
|
|
1213
|
+
self.dpi = dpi
|
|
1214
|
+
self.save_folder = save_folder
|
|
1215
|
+
self.image_ext = image_ext
|
|
1216
|
+
self.overwrite = overwrite
|
|
1217
|
+
self.global_params = global_params
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
_summary_job_context = None
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _summary_init(*args):
|
|
1224
|
+
global _summary_job_context
|
|
1225
|
+
_summary_job_context = SummaryJobContext(*args)
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _summary_job(unit_id):
|
|
1229
|
+
global _summary_job_context
|
|
1230
|
+
assert _summary_job_context is not None
|
|
1231
|
+
tmp_out = None
|
|
1232
|
+
try:
|
|
1233
|
+
ext = _summary_job_context.image_ext
|
|
1234
|
+
tmp_out = _summary_job_context.save_folder / f"tmp_unit{unit_id:04d}.{ext}"
|
|
1235
|
+
final_out = _summary_job_context.save_folder / f"unit{unit_id:04d}.{ext}"
|
|
1236
|
+
if tmp_out.exists():
|
|
1237
|
+
tmp_out.unlink()
|
|
1238
|
+
if not _summary_job_context.overwrite and final_out.exists():
|
|
1239
|
+
return
|
|
1240
|
+
if _summary_job_context.overwrite and final_out.exists():
|
|
1241
|
+
final_out.unlink()
|
|
1242
|
+
|
|
1243
|
+
fig = plt.figure(
|
|
1244
|
+
figsize=_summary_job_context.figsize,
|
|
1245
|
+
layout="constrained",
|
|
1246
|
+
dpi=_summary_job_context.dpi,
|
|
1247
|
+
)
|
|
1248
|
+
make_mixture_component_summary(
|
|
1249
|
+
_summary_job_context.mix_data,
|
|
1250
|
+
unit_id,
|
|
1251
|
+
card_hspace=_summary_job_context.card_hspace,
|
|
1252
|
+
plots=_summary_job_context.plots,
|
|
1253
|
+
max_height=_summary_job_context.max_height,
|
|
1254
|
+
figsize=_summary_job_context.figsize,
|
|
1255
|
+
figure=fig,
|
|
1256
|
+
**_summary_job_context.global_params,
|
|
1257
|
+
)
|
|
1258
|
+
|
|
1259
|
+
# the save is done sort of atomically to help with the resuming and avoid
|
|
1260
|
+
# half-baked image files
|
|
1261
|
+
fig.savefig(tmp_out, dpi=_summary_job_context.dpi)
|
|
1262
|
+
tmp_out.rename(final_out)
|
|
1263
|
+
plt.close(fig)
|
|
1264
|
+
except Exception:
|
|
1265
|
+
import traceback
|
|
1266
|
+
|
|
1267
|
+
print("// error in unit", unit_id)
|
|
1268
|
+
print(traceback.format_exc())
|
|
1269
|
+
raise
|
|
1270
|
+
finally:
|
|
1271
|
+
if tmp_out is not None and tmp_out.exists():
|
|
1272
|
+
tmp_out.unlink()
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
# -- one-offs
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def vis_split_interpolation(
|
|
1279
|
+
mix_data: MixtureVisData,
|
|
1280
|
+
unit_id: int,
|
|
1281
|
+
erp: NeighborhoodFiller | None = None,
|
|
1282
|
+
whiten: bool = True,
|
|
1283
|
+
figscale=5,
|
|
1284
|
+
n_per_group=8,
|
|
1285
|
+
seed=0,
|
|
1286
|
+
layout="horz",
|
|
1287
|
+
overlay=True,
|
|
1288
|
+
):
|
|
1289
|
+
if erp is None:
|
|
1290
|
+
erp = mix_data.tmm.erp
|
|
1291
|
+
|
|
1292
|
+
split_data = mix_data.train_data.dense_slice_by_unit(
|
|
1293
|
+
unit_id, gen=mix_data.tmm.rg, min_count=2 * mix_data.tmm.min_count
|
|
1294
|
+
)
|
|
1295
|
+
assert split_data is not None
|
|
1296
|
+
|
|
1297
|
+
kmeans_responsibilities, kmeans_x, kmeans_chans = try_kmeans(
|
|
1298
|
+
data=split_data,
|
|
1299
|
+
k=mix_data.tmm.split_k,
|
|
1300
|
+
erp=erp,
|
|
1301
|
+
gen=mix_data.tmm.rg,
|
|
1302
|
+
feature_rank=mix_data.tmm.noise.rank,
|
|
1303
|
+
min_count=mix_data.tmm.min_count,
|
|
1304
|
+
min_channel_count=mix_data.tmm.min_channel_count,
|
|
1305
|
+
debug=True,
|
|
1306
|
+
whiten=whiten,
|
|
1307
|
+
)
|
|
1308
|
+
|
|
1309
|
+
n_spikes = split_data.x.shape[0]
|
|
1310
|
+
assert kmeans_x is not None
|
|
1311
|
+
assert kmeans_x.shape[0] == n_spikes
|
|
1312
|
+
assert kmeans_chans is not None
|
|
1313
|
+
if kmeans_responsibilities is not None:
|
|
1314
|
+
assert kmeans_responsibilities.shape[0] == n_spikes
|
|
1315
|
+
kmeans_labels = kmeans_responsibilities.argmax(dim=1)
|
|
1316
|
+
kmeans_labels = kmeans_labels.cpu()
|
|
1317
|
+
else:
|
|
1318
|
+
kmeans_labels = torch.zeros(n_spikes, dtype=torch.long)
|
|
1319
|
+
|
|
1320
|
+
vis_ix = []
|
|
1321
|
+
rg = np.random.default_rng(seed)
|
|
1322
|
+
ulabels = kmeans_labels.unique()
|
|
1323
|
+
for l in ulabels:
|
|
1324
|
+
(in_l,) = (kmeans_labels == l).nonzero(as_tuple=True)
|
|
1325
|
+
in_l = in_l.numpy(force=True)
|
|
1326
|
+
if in_l.size > n_per_group:
|
|
1327
|
+
in_l = rg.choice(in_l, size=n_per_group, replace=False)
|
|
1328
|
+
vis_ix.append(in_l)
|
|
1329
|
+
|
|
1330
|
+
# interpolated waveforms
|
|
1331
|
+
interp_wfs = []
|
|
1332
|
+
for vix in vis_ix:
|
|
1333
|
+
f = kmeans_x[vix].reshape(len(vix), -1)
|
|
1334
|
+
f = mix_data.reconstruct_flat(f)
|
|
1335
|
+
interp_wfs.append(f)
|
|
1336
|
+
|
|
1337
|
+
# original observed waveforms
|
|
1338
|
+
orig_wfs = []
|
|
1339
|
+
orig_chans = []
|
|
1340
|
+
for vix in vis_ix:
|
|
1341
|
+
f = (split_data.whitenedx if whiten else split_data.x)[vix]
|
|
1342
|
+
f = mix_data.reconstruct_flat(f)
|
|
1343
|
+
orig_wfs.append(f)
|
|
1344
|
+
orig_chans.append(
|
|
1345
|
+
split_data.neighborhoods.b.neighborhoods[split_data.neighborhood_ids[vix]]
|
|
1346
|
+
)
|
|
1347
|
+
|
|
1348
|
+
# -- draw
|
|
1349
|
+
|
|
1350
|
+
if layout == "horz":
|
|
1351
|
+
fig, axes = plt.subplots(
|
|
1352
|
+
nrows=2,
|
|
1353
|
+
ncols=ulabels.numel(),
|
|
1354
|
+
figsize=(figscale * ulabels.numel(), figscale * 2),
|
|
1355
|
+
layout="constrained",
|
|
1356
|
+
)
|
|
1357
|
+
axes = axes.T
|
|
1358
|
+
else:
|
|
1359
|
+
fig, axes = plt.subplots(
|
|
1360
|
+
ncols=2,
|
|
1361
|
+
nrows=ulabels.numel(),
|
|
1362
|
+
figsize=(figscale * 2, figscale * ulabels.numel()),
|
|
1363
|
+
layout="constrained",
|
|
1364
|
+
)
|
|
1365
|
+
maa = max([np.abs(w).max() for w in orig_wfs])
|
|
1366
|
+
|
|
1367
|
+
for col, iwf, owf, ochans, c in zip(axes, interp_wfs, orig_wfs, orig_chans, "rgb"):
|
|
1368
|
+
print(f"{np.abs(iwf).max()=} {np.abs(owf).max()=}")
|
|
1369
|
+
geomplot(
|
|
1370
|
+
waveforms=iwf,
|
|
1371
|
+
channels=kmeans_chans[None].broadcast_to(iwf.shape[0], *kmeans_chans.shape),
|
|
1372
|
+
geom=mix_data.prgeom,
|
|
1373
|
+
show_zero=False,
|
|
1374
|
+
subar=False,
|
|
1375
|
+
ax=col[1],
|
|
1376
|
+
max_abs_amp=maa,
|
|
1377
|
+
linewidth=1,
|
|
1378
|
+
zlim=None,
|
|
1379
|
+
)
|
|
1380
|
+
for ax in col[: 1 + overlay]:
|
|
1381
|
+
geomplot(
|
|
1382
|
+
waveforms=owf,
|
|
1383
|
+
channels=ochans,
|
|
1384
|
+
geom=mix_data.prgeom,
|
|
1385
|
+
show_zero=True,
|
|
1386
|
+
subar=True,
|
|
1387
|
+
ax=ax,
|
|
1388
|
+
max_abs_amp=maa,
|
|
1389
|
+
color="k",
|
|
1390
|
+
linewidth=1,
|
|
1391
|
+
zlim=None,
|
|
1392
|
+
)
|
|
1393
|
+
for ax in axes.flat:
|
|
1394
|
+
ax.axis("off")
|
|
1395
|
+
|
|
1396
|
+
return fig
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def vis_obs_interpolation(
|
|
1400
|
+
mix_data: MixtureVisData,
|
|
1401
|
+
unit_id: int,
|
|
1402
|
+
count: int = 256,
|
|
1403
|
+
erp_params: dict[str, InterpolationParams] | None = None,
|
|
1404
|
+
figscale=5.0,
|
|
1405
|
+
disp_colormap="viridis",
|
|
1406
|
+
time_colormap="plasma",
|
|
1407
|
+
):
|
|
1408
|
+
(trix,) = (mix_data.train_labels == unit_id).nonzero()
|
|
1409
|
+
if trix.size > count:
|
|
1410
|
+
trix = np.random.default_rng(0).choice(trix, size=count, replace=False)
|
|
1411
|
+
trix.sort()
|
|
1412
|
+
fix = mix_data.train_ixs[trix]
|
|
1413
|
+
n = len(trix)
|
|
1414
|
+
|
|
1415
|
+
t_s = mix_data.train_times[trix]
|
|
1416
|
+
z = mix_data.sorting.slice_feature_by_name("point_source_localizations", fix)[:, 2]
|
|
1417
|
+
if mix_data.motion_est is None:
|
|
1418
|
+
disp = np.zeros_like(z)
|
|
1419
|
+
else:
|
|
1420
|
+
disp = mix_data.motion_est.disp_at_s(t_s, z)
|
|
1421
|
+
|
|
1422
|
+
dev = mix_data.tmm.b.means.device
|
|
1423
|
+
sgeom = pad_geom(mix_data.sorting.geom, device=dev) # type: ignore
|
|
1424
|
+
tgeom = torch.asarray(mix_data.prgeom).to(sgeom)
|
|
1425
|
+
channel_index = torch.asarray(mix_data.sorting.channel_index, device=dev) # type: ignore
|
|
1426
|
+
erps = {}
|
|
1427
|
+
for k, ip in (erp_params or {}).items():
|
|
1428
|
+
erps[k] = StableFeaturesInterpolator(
|
|
1429
|
+
source_geom=sgeom,
|
|
1430
|
+
target_geom=tgeom,
|
|
1431
|
+
channel_index=channel_index,
|
|
1432
|
+
params=ip,
|
|
1433
|
+
)
|
|
1434
|
+
|
|
1435
|
+
features = {"actual": mix_data.train_data.x[trix]}
|
|
1436
|
+
origf = mix_data.sorting.slice_feature_by_name(
|
|
1437
|
+
"collisioncleaned_tpca_features", fix
|
|
1438
|
+
)
|
|
1439
|
+
origf = torch.asarray(origf).to(sgeom)
|
|
1440
|
+
chans = torch.asarray(mix_data.sorting.channels[fix]).to(channel_index)
|
|
1441
|
+
shifts = torch.asarray(-disp).to(sgeom)
|
|
1442
|
+
tchans = mix_data.tmm.neighb_cov.obs_ix[mix_data.train_data.neighborhood_ids[trix]]
|
|
1443
|
+
for name, erp in (erps or {}).items():
|
|
1444
|
+
features[name] = erp.interp(
|
|
1445
|
+
features=origf,
|
|
1446
|
+
source_main_channels=chans,
|
|
1447
|
+
target_channels=tchans,
|
|
1448
|
+
source_shifts=shifts,
|
|
1449
|
+
)
|
|
1450
|
+
|
|
1451
|
+
waveforms = {
|
|
1452
|
+
k: mix_data.reconstruct_flat(f.view(n, -1)) for k, f in features.items()
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
maa = origf.abs().nan_to_num_().max().cpu().item()
|
|
1456
|
+
dcolors = plt.get_cmap(disp_colormap)(spiketorch.minmax(disp))
|
|
1457
|
+
tcolors = plt.get_cmap(time_colormap)(spiketorch.minmax(t_s))
|
|
1458
|
+
|
|
1459
|
+
ncols = 2
|
|
1460
|
+
nrows = len(features)
|
|
1461
|
+
fig = plt.figure(
|
|
1462
|
+
figsize=(ncols * figscale, (nrows + 0.5) * figscale),
|
|
1463
|
+
layout="constrained",
|
|
1464
|
+
)
|
|
1465
|
+
panels = fig.subfigures(
|
|
1466
|
+
nrows=nrows + 1,
|
|
1467
|
+
height_ratios=[1] + ([2] * nrows),
|
|
1468
|
+
)
|
|
1469
|
+
|
|
1470
|
+
ax_top = panels[0].subplots()
|
|
1471
|
+
ax_top.scatter(t_s, disp, c=dcolors, s=3, lw=0)
|
|
1472
|
+
|
|
1473
|
+
for row, (name, wf) in zip(panels[1:], waveforms.items()):
|
|
1474
|
+
row_axs = row.subplots(ncols=2)
|
|
1475
|
+
row.suptitle(name, fontsize="small")
|
|
1476
|
+
for ax, cs in zip(row_axs, (dcolors, tcolors)):
|
|
1477
|
+
geomplot(
|
|
1478
|
+
waveforms=wf,
|
|
1479
|
+
channels=tchans.numpy(force=True),
|
|
1480
|
+
geom=mix_data.prgeom,
|
|
1481
|
+
colors=cs,
|
|
1482
|
+
subar=True,
|
|
1483
|
+
ax=ax,
|
|
1484
|
+
max_abs_amp=maa,
|
|
1485
|
+
lw=0.5,
|
|
1486
|
+
)
|
|
1487
|
+
ax.axis("off")
|
|
1488
|
+
|
|
1489
|
+
return fig
|